diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 0a81259..f0a8e26 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -707,6 +707,11 @@ // Whether to show the unquoted partition names in query results. HIVE_DECODE_PARTITION_NAME("hive.decode.partition.name", false), + + // ptf partition constants + HIVE_PTF_PARTITION_PERSISTENCE_CLASS("hive.ptf.partition.persistence", + "org.apache.hadoop.hive.ql.exec.PTFPersistence$PartitionedByteBasedList"), + HIVE_PTF_PARTITION_PERSISTENT_SIZE("hive.ptf.partition.persistence.memsize", (int) Math.pow(2, (6 + 10 + 10)) ), // 64MB ; public final String varname; diff --git data/files/flights_tiny.txt data/files/flights_tiny.txt new file mode 100755 index 0000000..4841c92 --- /dev/null +++ data/files/flights_tiny.txt @@ -0,0 +1,137 @@ +BaltimoreNew York20101020-30.01064 +BaltimoreNew York2010102023.01142 +BaltimoreNew York201010206.01599 +ChicagoNew York2010102042.0361 +ChicagoNew York2010102024.0897 +ChicagoNew York2010102015.01531 +ChicagoNew York20101020-6.01610 +ChicagoNew York20101020-2.03198 +BaltimoreNew York2010102117.01064 +BaltimoreNew York20101021105.01142 +BaltimoreNew York2010102128.01599 +ChicagoNew York20101021142.0361 +ChicagoNew York2010102177.0897 +ChicagoNew York2010102153.01531 +ChicagoNew York20101021-5.01610 +ChicagoNew York2010102151.03198 +BaltimoreNew York20101022-12.01064 +BaltimoreNew York2010102254.01142 +BaltimoreNew York2010102218.01599 +ChicagoNew York201010222.0361 +ChicagoNew York2010102224.0897 +ChicagoNew York2010102216.01531 +ChicagoNew York20101022-6.01610 +ChicagoNew York20101022-11.03198 +BaltimoreNew York2010102318.0272 +BaltimoreNew York20101023-10.01805 +BaltimoreNew York201010236.03171 +ChicagoNew York201010233.0384 +ChicagoNew York2010102332.0426 +ChicagoNew York201010231.0650 +ChicagoNew York2010102311.03085 +BaltimoreNew York2010102412.01599 +BaltimoreNew York2010102420.02571 +ChicagoNew York2010102410.0361 +ChicagoNew York20101024113.0897 +ChicagoNew York20101024-5.01531 +ChicagoNew York20101024-17.01610 +ChicagoNew York20101024-3.03198 +BaltimoreNew York20101025-25.01064 +BaltimoreNew York2010102592.01142 +BaltimoreNew York20101025106.01599 +ChicagoNew York2010102531.0361 +ChicagoNew York20101025-1.0897 +ChicagoNew York2010102543.01531 +ChicagoNew York201010256.01610 +ChicagoNew York20101025-16.03198 +BaltimoreNew York20101026-22.01064 +BaltimoreNew York20101026123.01142 +BaltimoreNew York2010102690.01599 +ChicagoNew York2010102612.0361 +ChicagoNew York201010260.0897 +ChicagoNew York2010102629.01531 +ChicagoNew York20101026-17.01610 +ChicagoNew York201010266.03198 +BaltimoreNew York20101027-18.01064 +BaltimoreNew York2010102749.01142 +BaltimoreNew York2010102792.01599 +ChicagoNew York20101027148.0361 +ChicagoNew York20101027-11.0897 +ChicagoNew York2010102770.01531 +ChicagoNew York201010278.01610 +ChicagoNew York2010102721.03198 +BaltimoreNew York20101028-4.01064 +BaltimoreNew York20101028-14.01142 +BaltimoreNew York20101028-14.01599 +ChicagoNew York201010282.0361 +ChicagoNew York201010282.0897 +ChicagoNew York20101028-11.01531 +ChicagoNew York201010283.01610 +ChicagoNew York20101028-18.03198 +BaltimoreNew York20101029-24.01064 +BaltimoreNew York2010102921.01142 +BaltimoreNew York20101029-2.01599 +ChicagoNew York20101029-12.0361 +ChicagoNew York20101029-11.0897 +ChicagoNew York2010102915.01531 +ChicagoNew York20101029-18.01610 +ChicagoNew York20101029-4.03198 +BaltimoreNew York2010103014.0272 +BaltimoreNew York20101030-1.01805 +BaltimoreNew York201010305.03171 +ChicagoNew York20101030-6.0384 +ChicagoNew York20101030-10.0426 +ChicagoNew York20101030-5.0650 +ChicagoNew York20101030-5.03085 +BaltimoreNew York20101031-1.01599 +BaltimoreNew York20101031-14.02571 +ChicagoNew York20101031-25.0361 +ChicagoNew York20101031-18.0897 +ChicagoNew York20101031-4.01531 +ChicagoNew York20101031-22.01610 +ChicagoNew York20101031-15.03198 +ClevelandNew York20101030-23.02018 +ClevelandNew York20101030-12.02932 +ClevelandNew York20101029-4.02630 +ClevelandNew York20101029-19.02646 +ClevelandNew York20101029-12.03014 +ClevelandNew York201010283.02630 +ClevelandNew York20101028-6.02646 +ClevelandNew York201010281.03014 +ClevelandNew York2010102716.02630 +ClevelandNew York2010102727.03014 +ClevelandNew York201010264.02630 +ClevelandNew York20101026-27.02646 +ClevelandNew York20101026-11.02662 +ClevelandNew York2010102613.03014 +ClevelandNew York20101025-4.02630 +ClevelandNew York2010102581.02646 +ClevelandNew York2010102542.03014 +ClevelandNew York201010245.02254 +ClevelandNew York20101024-11.02630 +ClevelandNew York20101024-20.02646 +ClevelandNew York20101024-9.03014 +ClevelandNew York20101023-21.02932 +ClevelandNew York201010221.02630 +ClevelandNew York20101022-25.02646 +ClevelandNew York20101022-3.03014 +ClevelandNew York201010213.02630 +ClevelandNew York2010102129.02646 +ClevelandNew York2010102172.03014 +ClevelandNew York20101020-8.02630 +ClevelandNew York20101020-15.03014 +WashingtonNew York20101023-25.05832 +WashingtonNew York20101023-21.05904 +WashingtonNew York20101023-18.05917 +WashingtonNew York20101030-27.05904 +WashingtonNew York20101030-16.05917 +WashingtonNew York20101020-2.07291 +WashingtonNew York2010102122.07291 +WashingtonNew York20101023-16.07274 +WashingtonNew York20101024-26.07282 +WashingtonNew York201010259.07291 +WashingtonNew York201010264.07291 +WashingtonNew York2010102726.07291 +WashingtonNew York2010102845.07291 +WashingtonNew York201010291.07291 +WashingtonNew York20101031-18.07282 diff --git data/files/part.rc data/files/part.rc new file mode 100755 index 0000000..e39e72e Binary files /dev/null and data/files/part.rc differ diff --git data/files/part.seq data/files/part.seq new file mode 100755 index 0000000..b919eb2 Binary files /dev/null and data/files/part.seq differ diff --git data/files/part_tiny.txt data/files/part_tiny.txt new file mode 100644 index 0000000..f6973fd --- /dev/null +++ data/files/part_tiny.txt @@ -0,0 +1,26 @@ +121152almond antique burnished rose metallicManufacturer#1Brand#14PROMO PLATED TIN2JUMBO BOX1173.15e pinto beans h +121152almond antique burnished rose metallicManufacturer#1Brand#14PROMO PLATED TIN2JUMBO BOX1173.15e pinto beans h +85768almond antique chartreuse lavender yellowManufacturer#1Brand#12LARGE BRUSHED STEEL34SM BAG1753.76refull +110592almond antique salmon chartreuse burlywoodManufacturer#1Brand#15PROMO BURNISHED NICKEL6JUMBO PKG1602.59 to the furiously +86428almond aquamarine burnished black steelManufacturer#1Brand#12STANDARD ANODIZED STEEL28WRAP BAG1414.42arefully +65667almond aquamarine pink moccasin thistleManufacturer#1Brand#12LARGE BURNISHED STEEL42JUMBO CASE1632.66e across the expr +105685almond antique violet chocolate turquoiseManufacturer#2Brand#22MEDIUM ANODIZED COPPER14MED CAN1690.68ly pending requ +191709almond antique violet turquoise frostedManufacturer#2Brand#22ECONOMY POLISHED STEEL40MED BOX1800.7 haggle +146985almond aquamarine midnight light salmonManufacturer#2Brand#23MEDIUM BURNISHED COPPER2SM CASE2031.98s cajole caref +132666almond aquamarine rose maroon antiqueManufacturer#2Brand#24SMALL POLISHED NICKEL25MED BOX1698.66even +195606almond aquamarine sandy cyan gainsboroManufacturer#2Brand#25STANDARD PLATED TIN18SM PKG1701.6ic de +90681almond antique chartreuse khaki whiteManufacturer#3Brand#31MEDIUM BURNISHED TIN17SM CASE1671.68are slyly after the sl +17273almond antique forest lavender goldenrodManufacturer#3Brand#35PROMO ANODIZED TIN14JUMBO CASE1190.27along the +112398almond antique metallic orange dimManufacturer#3Brand#32MEDIUM BURNISHED BRASS19JUMBO JAR1410.39ole car +40982almond antique misty red oliveManufacturer#3Brand#32ECONOMY PLATED COPPER1LG PKG1922.98c foxes can s +144293almond antique olive coral navajoManufacturer#3Brand#34STANDARD POLISHED STEEL45JUMBO CAN1337.29ag furiously about +49671almond antique gainsboro frosted violetManufacturer#4Brand#41SMALL BRUSHED BRASS10SM BOX1620.67ccounts run quick +48427almond antique violet mint lemonManufacturer#4Brand#42PROMO POLISHED STEEL39SM CASE1375.42hely ironic i +45261almond aquamarine floral ivory bisqueManufacturer#4Brand#42SMALL PLATED STEEL27WRAP CASE1206.26careful +17927almond aquamarine yellow dodger mintManufacturer#4Brand#41ECONOMY BRUSHED COPPER7SM PKG1844.92ites. eve +33357almond azure aquamarine papaya violetManufacturer#4Brand#41STANDARD ANODIZED TIN12WRAP CASE1290.35reful +192697almond antique blue firebrick mintManufacturer#5Brand#52MEDIUM BURNISHED TIN31LG DRUM1789.69ickly ir +42669almond antique medium spring khakiManufacturer#5Brand#51STANDARD BURNISHED TIN6MED CAN1611.66sits haggl +155733almond antique sky peru orangeManufacturer#5Brand#53SMALL PLATED BRASS2WRAP DRUM1788.73furiously. bra +15103almond aquamarine dodger light gainsboroManufacturer#5Brand#53ECONOMY BURNISHED STEEL46LG PACK1018.1packages hinder carefu +78486almond azure blanched chiffon midnightManufacturer#5Brand#52LARGE BRUSHED BRASS23MED BAG1464.48hely blith diff --git ql/if/queryplan.thrift ql/if/queryplan.thrift index 4427929..304ff08 100644 --- ql/if/queryplan.thrift +++ ql/if/queryplan.thrift @@ -53,6 +53,7 @@ enum OperatorType { LATERALVIEWFORWARD, HASHTABLESINK, HASHTABLEDUMMY, + PTF, } struct Operator { diff --git ql/src/gen/thrift/gen-cpp/queryplan_types.cpp ql/src/gen/thrift/gen-cpp/queryplan_types.cpp index 184f8a8..3e2bc86 100644 --- ql/src/gen/thrift/gen-cpp/queryplan_types.cpp +++ ql/src/gen/thrift/gen-cpp/queryplan_types.cpp @@ -48,7 +48,8 @@ int _kOperatorTypeValues[] = { OperatorType::LATERALVIEWJOIN, OperatorType::LATERALVIEWFORWARD, OperatorType::HASHTABLESINK, - OperatorType::HASHTABLEDUMMY + OperatorType::HASHTABLEDUMMY, + OperatorType::PTF }; const char* _kOperatorTypeNames[] = { "JOIN", @@ -68,9 +69,10 @@ const char* _kOperatorTypeNames[] = { "LATERALVIEWJOIN", "LATERALVIEWFORWARD", "HASHTABLESINK", - "HASHTABLEDUMMY" + "HASHTABLEDUMMY", + "PTF" }; -const std::map _OperatorType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(18, _kOperatorTypeValues, _kOperatorTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL)); +const std::map _OperatorType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(19, _kOperatorTypeValues, _kOperatorTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL)); int _kTaskTypeValues[] = { TaskType::MAP, diff --git ql/src/gen/thrift/gen-cpp/queryplan_types.h ql/src/gen/thrift/gen-cpp/queryplan_types.h index ed12cc1..ed4fb46 100644 --- ql/src/gen/thrift/gen-cpp/queryplan_types.h +++ ql/src/gen/thrift/gen-cpp/queryplan_types.h @@ -53,7 +53,8 @@ struct OperatorType { LATERALVIEWJOIN = 14, LATERALVIEWFORWARD = 15, HASHTABLESINK = 16, - HASHTABLEDUMMY = 17 + HASHTABLEDUMMY = 17, + PTF = 18 }; }; diff --git ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/Operator.java ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/Operator.java index 47079ce..f1c9e2d 100644 --- ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/Operator.java +++ ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/Operator.java @@ -810,7 +810,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, Operator struct) th for (int _i25 = 0; _i25 < _map24.size; ++_i25) { String _key26; // required - String _val27; // optional + String _val27; // required _key26 = iprot.readString(); _val27 = iprot.readString(); struct.operatorAttributes.put(_key26, _val27); @@ -830,7 +830,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, Operator struct) th for (int _i29 = 0; _i29 < _map28.size; ++_i29) { String _key30; // required - long _val31; // optional + long _val31; // required _key30 = iprot.readString(); _val31 = iprot.readI64(); struct.operatorCounters.put(_key30, _val31); @@ -1003,7 +1003,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, Operator struct) thr for (int _i37 = 0; _i37 < _map36.size; ++_i37) { String _key38; // required - String _val39; // optional + String _val39; // required _key38 = iprot.readString(); _val39 = iprot.readString(); struct.operatorAttributes.put(_key38, _val39); @@ -1018,7 +1018,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, Operator struct) thr for (int _i41 = 0; _i41 < _map40.size; ++_i41) { String _key42; // required - long _val43; // optional + long _val43; // required _key42 = iprot.readString(); _val43 = iprot.readI64(); struct.operatorCounters.put(_key42, _val43); diff --git ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/OperatorType.java ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/OperatorType.java index 7c4c413..f03af2f 100644 --- ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/OperatorType.java +++ ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/OperatorType.java @@ -29,7 +29,8 @@ LATERALVIEWJOIN(14), LATERALVIEWFORWARD(15), HASHTABLESINK(16), - HASHTABLEDUMMY(17); + HASHTABLEDUMMY(17), + PTF(18); private final int value; @@ -86,6 +87,8 @@ public static OperatorType findByValue(int value) { return HASHTABLESINK; case 17: return HASHTABLEDUMMY; + case 18: + return PTF; default: return null; } diff --git ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/Query.java ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/Query.java index 0a10e54..e0d77e8 100644 --- ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/Query.java +++ ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/Query.java @@ -983,7 +983,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, Query struct) throw for (int _i101 = 0; _i101 < _map100.size; ++_i101) { String _key102; // required - String _val103; // optional + String _val103; // required _key102 = iprot.readString(); _val103 = iprot.readString(); struct.queryAttributes.put(_key102, _val103); @@ -1003,7 +1003,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, Query struct) throw for (int _i105 = 0; _i105 < _map104.size; ++_i105) { String _key106; // required - long _val107; // optional + long _val107; // required _key106 = iprot.readString(); _val107 = iprot.readI64(); struct.queryCounters.put(_key106, _val107); @@ -1239,7 +1239,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, Query struct) throws for (int _i118 = 0; _i118 < _map117.size; ++_i118) { String _key119; // required - String _val120; // optional + String _val120; // required _key119 = iprot.readString(); _val120 = iprot.readString(); struct.queryAttributes.put(_key119, _val120); @@ -1254,7 +1254,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, Query struct) throws for (int _i122 = 0; _i122 < _map121.size; ++_i122) { String _key123; // required - long _val124; // optional + long _val124; // required _key123 = iprot.readString(); _val124 = iprot.readI64(); struct.queryCounters.put(_key123, _val124); diff --git ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/Stage.java ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/Stage.java index cf2a52b..c341db2 100644 --- ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/Stage.java +++ ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/Stage.java @@ -911,7 +911,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, Stage struct) throw for (int _i73 = 0; _i73 < _map72.size; ++_i73) { String _key74; // required - String _val75; // optional + String _val75; // required _key74 = iprot.readString(); _val75 = iprot.readString(); struct.stageAttributes.put(_key74, _val75); @@ -931,7 +931,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, Stage struct) throw for (int _i77 = 0; _i77 < _map76.size; ++_i77) { String _key78; // required - long _val79; // optional + long _val79; // required _key78 = iprot.readString(); _val79 = iprot.readI64(); struct.stageCounters.put(_key78, _val79); @@ -1147,7 +1147,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, Stage struct) throws for (int _i90 = 0; _i90 < _map89.size; ++_i90) { String _key91; // required - String _val92; // optional + String _val92; // required _key91 = iprot.readString(); _val92 = iprot.readString(); struct.stageAttributes.put(_key91, _val92); @@ -1162,7 +1162,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, Stage struct) throws for (int _i94 = 0; _i94 < _map93.size; ++_i94) { String _key95; // required - long _val96; // optional + long _val96; // required _key95 = iprot.readString(); _val96 = iprot.readI64(); struct.stageCounters.put(_key95, _val96); diff --git ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/Task.java ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/Task.java index f7bfabf..fc4313f 100644 --- ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/Task.java +++ ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/Task.java @@ -996,7 +996,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, Task struct) throws for (int _i45 = 0; _i45 < _map44.size; ++_i45) { String _key46; // required - String _val47; // optional + String _val47; // required _key46 = iprot.readString(); _val47 = iprot.readString(); struct.taskAttributes.put(_key46, _val47); @@ -1016,7 +1016,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, Task struct) throws for (int _i49 = 0; _i49 < _map48.size; ++_i49) { String _key50; // required - long _val51; // optional + long _val51; // required _key50 = iprot.readString(); _val51 = iprot.readI64(); struct.taskCounters.put(_key50, _val51); @@ -1256,7 +1256,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, Task struct) throws for (int _i62 = 0; _i62 < _map61.size; ++_i62) { String _key63; // required - String _val64; // optional + String _val64; // required _key63 = iprot.readString(); _val64 = iprot.readString(); struct.taskAttributes.put(_key63, _val64); @@ -1271,7 +1271,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, Task struct) throws for (int _i66 = 0; _i66 < _map65.size; ++_i66) { String _key67; // required - long _val68; // optional + long _val68; // required _key67 = iprot.readString(); _val68 = iprot.readI64(); struct.taskCounters.put(_key67, _val68); diff --git ql/src/gen/thrift/gen-php/Types.php ql/src/gen/thrift/gen-php/Types.php index 71b25b7..978dde4 100644 --- ql/src/gen/thrift/gen-php/Types.php +++ ql/src/gen/thrift/gen-php/Types.php @@ -53,6 +53,7 @@ final class OperatorType { const LATERALVIEWFORWARD = 15; const HASHTABLESINK = 16; const HASHTABLEDUMMY = 17; + const PTF = 18; static public $__names = array( 0 => 'JOIN', 1 => 'MAPJOIN', @@ -72,6 +73,7 @@ final class OperatorType { 15 => 'LATERALVIEWFORWARD', 16 => 'HASHTABLESINK', 17 => 'HASHTABLEDUMMY', + 18 => 'PTF', ); } diff --git ql/src/gen/thrift/gen-py/queryplan/ttypes.py ql/src/gen/thrift/gen-py/queryplan/ttypes.py index 0658b61..0a5bab4 100644 --- ql/src/gen/thrift/gen-py/queryplan/ttypes.py +++ ql/src/gen/thrift/gen-py/queryplan/ttypes.py @@ -63,6 +63,7 @@ class OperatorType: LATERALVIEWFORWARD = 15 HASHTABLESINK = 16 HASHTABLEDUMMY = 17 + PTF = 18 _VALUES_TO_NAMES = { 0: "JOIN", @@ -83,6 +84,7 @@ class OperatorType: 15: "LATERALVIEWFORWARD", 16: "HASHTABLESINK", 17: "HASHTABLEDUMMY", + 18: "PTF", } _NAMES_TO_VALUES = { @@ -104,6 +106,7 @@ class OperatorType: "LATERALVIEWFORWARD": 15, "HASHTABLESINK": 16, "HASHTABLEDUMMY": 17, + "PTF": 18, } class TaskType: diff --git ql/src/gen/thrift/gen-rb/queryplan_types.rb ql/src/gen/thrift/gen-rb/queryplan_types.rb index 46d7881..0ec7ed3 100644 --- ql/src/gen/thrift/gen-rb/queryplan_types.rb +++ ql/src/gen/thrift/gen-rb/queryplan_types.rb @@ -39,8 +39,9 @@ module OperatorType LATERALVIEWFORWARD = 15 HASHTABLESINK = 16 HASHTABLEDUMMY = 17 - VALUE_MAP = {0 => "JOIN", 1 => "MAPJOIN", 2 => "EXTRACT", 3 => "FILTER", 4 => "FORWARD", 5 => "GROUPBY", 6 => "LIMIT", 7 => "SCRIPT", 8 => "SELECT", 9 => "TABLESCAN", 10 => "FILESINK", 11 => "REDUCESINK", 12 => "UNION", 13 => "UDTF", 14 => "LATERALVIEWJOIN", 15 => "LATERALVIEWFORWARD", 16 => "HASHTABLESINK", 17 => "HASHTABLEDUMMY"} - VALID_VALUES = Set.new([JOIN, MAPJOIN, EXTRACT, FILTER, FORWARD, GROUPBY, LIMIT, SCRIPT, SELECT, TABLESCAN, FILESINK, REDUCESINK, UNION, UDTF, LATERALVIEWJOIN, LATERALVIEWFORWARD, HASHTABLESINK, HASHTABLEDUMMY]).freeze + PTF = 18 + VALUE_MAP = {0 => "JOIN", 1 => "MAPJOIN", 2 => "EXTRACT", 3 => "FILTER", 4 => "FORWARD", 5 => "GROUPBY", 6 => "LIMIT", 7 => "SCRIPT", 8 => "SELECT", 9 => "TABLESCAN", 10 => "FILESINK", 11 => "REDUCESINK", 12 => "UNION", 13 => "UDTF", 14 => "LATERALVIEWJOIN", 15 => "LATERALVIEWFORWARD", 16 => "HASHTABLESINK", 17 => "HASHTABLEDUMMY", 18 => "PTF"} + VALID_VALUES = Set.new([JOIN, MAPJOIN, EXTRACT, FILTER, FORWARD, GROUPBY, LIMIT, SCRIPT, SELECT, TABLESCAN, FILESINK, REDUCESINK, UNION, UDTF, LATERALVIEWJOIN, LATERALVIEWFORWARD, HASHTABLESINK, HASHTABLEDUMMY, PTF]).freeze end module TaskType diff --git ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java index 83e5042..fc30a3b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java +++ ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java @@ -37,6 +37,7 @@ boolean hasOrderBy = false; boolean hasSortBy = false; boolean hasJoinFollowedByGroupBy = false; + boolean hasPTF = false; // does the query have a using clause boolean usesScript = false; @@ -107,4 +108,12 @@ public boolean hasClusterBy() { public void setHasClusterBy(boolean hasClusterBy) { this.hasClusterBy = hasClusterBy; } + + public boolean hasPTF() { + return hasPTF; + } + + public void setHasPTF(boolean hasPTF) { + this.hasPTF = hasPTF; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java index fc0256c..6cf6480 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java @@ -135,15 +135,23 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFCount; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFCovariance; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFCovarianceSample; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFCumeDist; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFDenseRank; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEWAHBitmap; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFFirstValue; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFHistogramNumeric; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFLastValue; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFMax; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFMin; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFNTile; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFParameterInfo; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFPercentRank; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFPercentileApprox; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFRank; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFResolver; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFResolver2; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFRowNumber; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFStd; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFStdSample; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFSum; @@ -172,6 +180,8 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFInFile; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIndex; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFInstr; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFLeadLag.GenericUDFLag; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFLeadLag.GenericUDFLead; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFLocate; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFMap; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFMapKeys; @@ -214,6 +224,11 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDTFParseUrlTuple; import org.apache.hadoop.hive.ql.udf.generic.GenericUDTFStack; import org.apache.hadoop.hive.ql.udf.generic.SimpleGenericUDAFParameterInfo; +import org.apache.hadoop.hive.ql.udf.ptf.NPath.NPathResolver; +import org.apache.hadoop.hive.ql.udf.ptf.Noop.NoopResolver; +import org.apache.hadoop.hive.ql.udf.ptf.NoopWithMap.NoopWithMapResolver; +import org.apache.hadoop.hive.ql.udf.ptf.TableFunctionResolver; +import org.apache.hadoop.hive.ql.udf.ptf.WindowingTableFunction.WindowingTableFunctionResolver; import org.apache.hadoop.hive.ql.udf.xml.GenericUDFXPath; import org.apache.hadoop.hive.ql.udf.xml.UDFXPathBoolean; import org.apache.hadoop.hive.ql.udf.xml.UDFXPathDouble; @@ -236,6 +251,7 @@ import org.w3c.dom.Element; import org.w3c.dom.NodeList; + /** * FunctionRegistry. */ @@ -247,6 +263,24 @@ * The mapping from expression function names to expression classes. */ static Map mFunctions = Collections.synchronizedMap(new LinkedHashMap()); + + /* + * PTF variables + * */ + + public static final String LEAD_FUNC_NAME = "lead"; + public static final String LAG_FUNC_NAME = "lag"; + + public static final String WINDOWING_TABLE_FUNCTION = "windowingtablefunction"; + public static final String NOOP_TABLE_FUNCTION = "noop"; + public static final String NOOP_MAP_TABLE_FUNCTION = "noopwithmap"; + + static Map tableFunctions = Collections.synchronizedMap(new LinkedHashMap()); + static Map windowFunctions = Collections.synchronizedMap(new LinkedHashMap()); + + public static final ArrayList RANKING_FUNCTIONS = new ArrayList(); + public static final ArrayList NAVIGATION_FUNCTIONS = new ArrayList(); + static { registerUDF("concat", UDFConcat.class, false); registerUDF("substr", UDFSubstr.class, false); @@ -480,6 +514,34 @@ registerGenericUDTF("json_tuple", GenericUDTFJSONTuple.class); registerGenericUDTF("parse_url_tuple", GenericUDTFParseUrlTuple.class); registerGenericUDTF("stack", GenericUDTFStack.class); + + //PTF declarations + registerGenericUDF(true, LEAD_FUNC_NAME, GenericUDFLead.class); + registerGenericUDF(true, LAG_FUNC_NAME, GenericUDFLag.class); + + registerHiveUDAFsAsWindowFunctions(); + registerWindowFunction("rownumber", new GenericUDAFRowNumber()); + registerWindowFunction("rank", new GenericUDAFRank()); + registerWindowFunction("denserank", new GenericUDAFDenseRank()); + registerWindowFunction("percentrank", new GenericUDAFPercentRank()); + registerWindowFunction("cumedist", new GenericUDAFCumeDist()); + registerWindowFunction("ntile", new GenericUDAFNTile()); + registerWindowFunction("first_value", new GenericUDAFFirstValue()); + registerWindowFunction("last_value", new GenericUDAFLastValue()); + + RANKING_FUNCTIONS.add("rank"); + RANKING_FUNCTIONS.add("denserank"); + RANKING_FUNCTIONS.add("percentrank"); + + NAVIGATION_FUNCTIONS.add(LEAD_FUNC_NAME); + NAVIGATION_FUNCTIONS.add(LAG_FUNC_NAME); + NAVIGATION_FUNCTIONS.add("first_value"); + NAVIGATION_FUNCTIONS.add("last_value"); + + registerTableFunction(NOOP_TABLE_FUNCTION, NoopResolver.class); + registerTableFunction(NOOP_MAP_TABLE_FUNCTION, NoopWithMapResolver.class); + registerTableFunction(WINDOWING_TABLE_FUNCTION, WindowingTableFunctionResolver.class); + registerTableFunction("npath", NPathResolver.class); } public static void registerTemporaryUDF(String functionName, @@ -1349,4 +1411,69 @@ public static void registerFunctionsFromPluginJar( private FunctionRegistry() { // prevent instantiation } + + + //---------PTF functions------------ + + public static void registerWindowFunction(String name, GenericUDAFResolver wFn) + { + registerGenericUDAF(true, name, wFn); + FunctionInfo fInfo = getFunctionInfo(name); + WindowFunctionInfo wInfo = new WindowFunctionInfo(fInfo); + windowFunctions.put(name.toLowerCase(), wInfo); + } + + public static boolean isWindowFunction(String name) + { + WindowFunctionInfo wFInfo = windowFunctions.get(name.toLowerCase()); + return wFInfo != null; + } + + public static WindowFunctionInfo getWindowFunctionInfo(String name) + { + return windowFunctions.get(name.toLowerCase()); + } + + static void registerHiveUDAFsAsWindowFunctions() + { + Set fNames = getFunctionNames(); + for(String fName : fNames) + { + FunctionInfo fInfo = getFunctionInfo(fName); + if ( fInfo.isGenericUDAF()) + { + WindowFunctionInfo wInfo = new WindowFunctionInfo(fInfo); + windowFunctions.put(fName, wInfo); + } + } + } + + public static boolean isTableFunction(String name) + { + PTFFunctionInfo tFInfo = tableFunctions.get(name.toLowerCase()); + return tFInfo != null && !tFInfo.isInternal(); + } + + public static TableFunctionResolver getTableFunctionResolver(String name) + { + PTFFunctionInfo tfInfo = tableFunctions.get(name.toLowerCase()); + return (TableFunctionResolver) ReflectionUtils.newInstance(tfInfo.getFunctionResolver(), null); + } + + public static TableFunctionResolver getWindowingTableFunction() + { + return getTableFunctionResolver(WINDOWING_TABLE_FUNCTION); + } + + public static TableFunctionResolver getNoopTableFunction() + { + return getTableFunctionResolver(NOOP_TABLE_FUNCTION); + } + + public static void registerTableFunction(String name, Class tFnCls) + { + PTFFunctionInfo tInfo = new PTFFunctionInfo(name, tFnCls); + tableFunctions.put(name.toLowerCase(), tInfo); + } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java index 69fff0e..8f52456 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hive.ql.plan.LimitDesc; import org.apache.hadoop.hive.ql.plan.MapJoinDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; +import org.apache.hadoop.hive.ql.plan.PTFDesc; import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; import org.apache.hadoop.hive.ql.plan.SMBJoinDesc; import org.apache.hadoop.hive.ql.plan.ScriptDesc; @@ -74,6 +75,7 @@ public OpTuple(Class descClass, Class> opClass) { opvec.add(new OpTuple(FileSinkDesc.class, FileSinkOperator.class)); opvec.add(new OpTuple(CollectDesc.class, CollectOperator.class)); opvec.add(new OpTuple(ScriptDesc.class, ScriptOperator.class)); + opvec.add(new OpTuple(PTFDesc.class, PTFOperator.class)); opvec.add(new OpTuple(ReduceSinkDesc.class, ReduceSinkOperator.class)); opvec.add(new OpTuple(ExtractDesc.class, ExtractOperator.class)); opvec.add(new OpTuple(GroupByDesc.class, GroupByOperator.class)); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/PTFFunctionInfo.java ql/src/java/org/apache/hadoop/hive/ql/exec/PTFFunctionInfo.java new file mode 100644 index 0000000..e62d8ba --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/PTFFunctionInfo.java @@ -0,0 +1,40 @@ +package org.apache.hadoop.hive.ql.exec; + +import org.apache.hadoop.hive.ql.exec.PartitionTableFunctionDescription; +import org.apache.hadoop.hive.ql.udf.ptf.TableFunctionResolver; + +class PTFFunctionInfo +{ + String displayName; + Class functionResolver; + boolean isInternal; + + public PTFFunctionInfo(String displayName, Class tFnCls) + { + super(); + this.displayName = displayName; + this.functionResolver = tFnCls; + isInternal = false; + PartitionTableFunctionDescription def = functionResolver.getAnnotation(PartitionTableFunctionDescription.class); + if ( def != null) + { + isInternal = def.isInternal(); + } + } + + public String getDisplayName() + { + return displayName; + } + + public Class getFunctionResolver() + { + return functionResolver; + } + + public boolean isInternal() + { + return isInternal; + } + +} \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java new file mode 100644 index 0000000..d82b9ef --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java @@ -0,0 +1,396 @@ +package org.apache.hadoop.hive.ql.exec; + +import java.io.ByteArrayInputStream; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Stack; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.exec.PTFPartition.PTFPartitionIterator; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.parse.PTFTranslator; +import org.apache.hadoop.hive.ql.parse.PTFTranslator.PTFDefDeserializer; +import org.apache.hadoop.hive.ql.parse.PTFTranslator.PTFTranslationInfo; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; +import org.apache.hadoop.hive.ql.plan.OperatorDesc; +import org.apache.hadoop.hive.ql.plan.PTFDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.ColumnDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.PTFInputDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.PartitionDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.TableFuncDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.WhereDef; +import org.apache.hadoop.hive.ql.plan.PTFDesc; +import org.apache.hadoop.hive.ql.plan.api.OperatorType; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFLeadLag; +import org.apache.hadoop.hive.ql.udf.ptf.TableFunctionEvaluator; +import org.apache.hadoop.hive.serde2.SerDe; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.Writable; + +public class PTFOperator extends Operator implements Serializable +{ + + private static final long serialVersionUID = 1L; + PTFDef qDef; + PTFPartition inputPart; + boolean isMapOperator; + + transient KeyWrapperFactory keyWrapperFactory; + protected transient KeyWrapper currentKeys; + protected transient KeyWrapper newKeys; + transient HiveConf hiveConf; + + /* + * 1. Find out if the operator is invoked at Map-Side or Reduce-side + * 2. Get the deserialized QueryDef + * 3. Reconstruct the transient variables in QueryDef + * 4. Create input partition to store rows coming from previous operator + */ + @Override + protected void initializeOp(Configuration jobConf) throws HiveException + { + hiveConf = new HiveConf(jobConf, PTFOperator.class); + // if the parent is ExtractOperator, this invocation is from reduce-side + Operator parentOp = getParentOperators().get(0); + if (parentOp instanceof ExtractOperator) + { + isMapOperator = false; + } + else + { + isMapOperator = true; + } + + // use the string from PTFDesc to get deserialized QueryDef + qDef = (PTFDef) PTFUtils + .deserialize(new ByteArrayInputStream(conf.getQueryDefStr() + .getBytes())); + reconstructQueryDef(hiveConf); + inputPart = PTFOperator.createFirstPartitionForChain(qDef, + inputObjInspectors[0], hiveConf, isMapOperator); + + // OI for FileSinkOperator is taken from select-list (reduce-side) + // OI for ReduceSinkOperator is taken from TODO + if (isMapOperator) + { + TableFuncDef tDef = PTFTranslator.getFirstTableFunction(qDef); + outputObjInspector = tDef.getRawInputOI(); + } + else + { + outputObjInspector = qDef.getSelectList().getOI(); + } + + setupKeysWrapper(inputObjInspectors[0]); + + super.initializeOp(jobConf); + } + + @Override + protected void closeOp(boolean abort) throws HiveException + { + super.closeOp(abort); + if(inputPart.size() != 0){ + if (isMapOperator) + { + processMapFunction(); + } + else + { + processInputPartition(); + } + } + } + + @Override + public void processOp(Object row, int tag) throws HiveException + { + if (!isMapOperator ) + { + /* + * checkif current row belongs to the current accumulated Partition: + * - If not: + * - process the current Partition + * - reset input Partition + * - set currentKey to the newKey if it is null or has changed. + */ + newKeys.getNewKey(row, inputPart.getOI()); + boolean keysAreEqual = (currentKeys != null && newKeys != null)? + newKeys.equals(currentKeys) : false; + + if (currentKeys != null && !keysAreEqual) + { + processInputPartition(); + inputPart = PTFOperator.createFirstPartitionForChain(qDef, inputObjInspectors[0], hiveConf, isMapOperator); + } + + if (currentKeys == null || !keysAreEqual) + { + if (currentKeys == null) + { + currentKeys = newKeys.copyKey(); + } + else + { + currentKeys.copyKey(newKeys); + } + } + } + + // add row to current Partition. + inputPart.append(row); + } + + /** + * Initialize the visitor to use the QueryDefDeserializer Use the order + * defined in QueryDefWalker to visit the QueryDef + * + * @param hiveConf + * @throws HiveException + */ + protected void reconstructQueryDef(HiveConf hiveConf) throws HiveException + { + + PTFDefDeserializer qdd = new PTFDefDeserializer(hiveConf, + inputObjInspectors[0]); + PTFTranslator.PTFDefWalker qdw = new PTFTranslator.PTFDefWalker(qdd); + qdw.walk(qDef); + } + + protected void setupKeysWrapper(ObjectInspector inputOI) throws HiveException + { + PartitionDef pDef = PTFTranslator.getFirstTableFunction(qDef).getWindow().getPartDef(); + ArrayList cols = pDef.getColumns(); + int numCols = cols.size(); + ExprNodeEvaluator[] keyFields = new ExprNodeEvaluator[numCols]; + ObjectInspector[] keyOIs = new ObjectInspector[numCols]; + ObjectInspector[] currentKeyOIs = new ObjectInspector[numCols]; + + for(int i=0; i pItr = outPart.iterator(); + while (pItr.hasNext()) + { + Object oRow = pItr.next(); + forward(oRow, outputObjInspector); + } + } + + /** + * @return the name of the operator + */ + @Override + public String getName() { + return getOperatorName(); + } + + static public String getOperatorName() { + return "PTF"; + } + + + @Override + public OperatorType getType() + { + return OperatorType.PTF; + } + + /** + * For all the table functions to be applied to the input + * hive table or query, push them on a stack. + * For each table function popped out of the stack, + * execute the function on the input partition + * and return an output partition. + * @param qDef + * @param part + * @return + * @throws HiveException + */ + private static PTFPartition executeChain(PTFDef qDef, PTFPartition part) + throws HiveException + { + Stack fnDefs = new Stack(); + PTFInputDef iDef = qDef.getInput(); + while (true) + { + if (iDef instanceof TableFuncDef) + { + fnDefs.push((TableFuncDef) iDef); + iDef = ((TableFuncDef) iDef).getInput(); + } + else + { + break; + } + } + + TableFuncDef currFnDef; + while (!fnDefs.isEmpty()) + { + currFnDef = fnDefs.pop(); + part = currFnDef.getFunction().execute(part); + } + return part; + } + + /** + * For each row in the partition: + * 1. evaluate the where condition if applicable. + * 2. evaluate the value for each column retrieved + * from the select list + * 3. Forward the writable value or object based on the + * implementation of the ForwardSink + * @param qDef + * @param oPart + * @param rS + * @throws HiveException + */ + @SuppressWarnings( + { "rawtypes", "unchecked" }) + private static void executeSelectList(PTFDef qDef, PTFPartition oPart, PTFOperator op) + throws HiveException + { + StructObjectInspector selectOI = qDef.getSelectList().getOI(); + StructObjectInspector inputOI = qDef.getInput().getOI(); + int numCols = selectOI.getAllStructFieldRefs().size(); + ArrayList cols = qDef.getSelectList().getColumns(); + int numSelCols = cols == null ? 0 : cols.size(); + Object[] output = new Object[numCols]; + + + WhereDef whDef = qDef.getWhere(); + boolean applyWhere = whDef != null; + Converter whConverter = !applyWhere ? null + : ObjectInspectorConverters + .getConverter( + whDef.getOI(), + PrimitiveObjectInspectorFactory.javaBooleanObjectInspector); + ExprNodeEvaluator whCondEval = !applyWhere ? null : whDef + .getExprEvaluator(); + + Writable value = null; + PTFPartitionIterator pItr = oPart.iterator(); + PTFOperator.connectLeadLagFunctionsToPartition(qDef, pItr); + while (pItr.hasNext()) + { + int colCnt = 0; + Object oRow = pItr.next(); + + if (applyWhere) + { + Object whCond = null; + whCond = whCondEval.evaluate(oRow); + whCond = whConverter.convert(whCond); + if (whCond == null || !((Boolean) whCond).booleanValue()) + { + continue; + } + } + + /* + * Setup the output row columns in the following order + * - the columns in the SelectList processed by the PTF (ie the Select Exprs that have navigation expressions) + * - the columns from the final PTF. + */ + + if ( cols != null ) { + for (ColumnDef cDef : cols) + { + Object newCol = cDef.getExprEvaluator().evaluate(oRow); + output[colCnt++] = newCol; + } + } + + for(; colCnt < numCols; ) { + StructField field = inputOI.getAllStructFieldRefs().get(colCnt - numSelCols); + output[colCnt++] = ObjectInspectorUtils.copyToStandardObject(inputOI.getStructFieldData(oRow, field), + field.getFieldObjectInspector()); + } + + op.forward(output, op.outputObjInspector); + } + } + + /** + * Create a new partition. + * The input OI is used to evaluate rows appended to the partition. + * The serde is determined based on whether the query has a map-phase + * or not. The OI on the serde is used by PTFs to evaluate output of the + * partition. + * @param qDef + * @param oi + * @param hiveConf + * @return + * @throws HiveException + */ + public static PTFPartition createFirstPartitionForChain(PTFDef qDef, ObjectInspector oi, + HiveConf hiveConf, boolean isMapSide) throws HiveException + { + TableFuncDef tabDef = PTFTranslator.getFirstTableFunction(qDef); + TableFunctionEvaluator tEval = tabDef.getFunction(); + String partClassName = tEval.getPartitionClass(); + int partMemSize = tEval.getPartitionMemSize(); + + PTFPartition part = null; + SerDe serde = tabDef.getInput().getSerde(); + part = new PTFPartition(partClassName, partMemSize, serde, + (StructObjectInspector) oi); + return part; + + } + + public static void connectLeadLagFunctionsToPartition(PTFDef qDef, + PTFPartitionIterator pItr) throws HiveException + { + PTFTranslationInfo tInfo = qDef.getTranslationInfo(); + List llFnDescs = tInfo.getLLInfo() + .getLeadLagExprs(); + if (llFnDescs == null) { + return; + } + for (ExprNodeGenericFuncDesc llFnDesc : llFnDescs) + { + GenericUDFLeadLag llFn = (GenericUDFLeadLag) llFnDesc + .getGenericUDF(); + llFn.setpItr(pItr); + } + } + + + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/PTFPartition.java ql/src/java/org/apache/hadoop/hive/ql/exec/PTFPartition.java new file mode 100644 index 0000000..706ca21 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/PTFPartition.java @@ -0,0 +1,262 @@ +package org.apache.hadoop.hive.ql.exec; + +import java.util.ConcurrentModificationException; +import java.util.Iterator; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.ql.exec.PTFPersistence.ByteBasedList; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.SerDe; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.io.Writable; + +/* + * represents a collection of rows that is acted upon by a TableFunction or a WindowFunction. + */ +public class PTFPartition +{ + SerDe serDe; + StructObjectInspector OI; + private ByteBasedList elems; + private Writable wRow; + private int sz; + + public PTFPartition(HiveConf cfg, SerDe serDe, StructObjectInspector oI) throws HiveException + { + String partitionClass = HiveConf.getVar(cfg, ConfVars.HIVE_PTF_PARTITION_PERSISTENCE_CLASS); + int partitionMemSize = HiveConf.getIntVar(cfg, ConfVars.HIVE_PTF_PARTITION_PERSISTENT_SIZE); + init(partitionClass, partitionMemSize, serDe, oI); + } + + public PTFPartition(String partitionClass, int partitionMemSize, SerDe serDe, StructObjectInspector oI) throws HiveException + { + init(partitionClass, partitionMemSize, serDe, oI); + } + + private void init(String partitionClass, int partitionMemSize, SerDe serDe, StructObjectInspector oI) throws HiveException + { + this.serDe = serDe; + OI = oI; + elems = PTFPersistence.createList(partitionClass, partitionMemSize); + sz = 0; + wRow = createWritable(); + } + + public SerDe getSerDe() + { + return serDe; + } + public void setSerDe(SerDe serDe) + { + this.serDe = serDe; + } + public StructObjectInspector getOI() + { + return OI; + } + public void setOI(StructObjectInspector oI) + { + OI = oI; + } + + private Writable createWritable() throws HiveException + { + try + { + return serDe.getSerializedClass().newInstance(); + } + catch(Throwable t) + { + throw new HiveException(t); + } + } + + public Object getAt(int i) throws HiveException + { + try + { + elems.get(i, wRow); + Object o = serDe.deserialize(wRow); + return o; + } + catch(SerDeException se) + { + throw new HiveException(se); + } + } + + public Object getWritableAt(int i) throws HiveException + { + elems.get(i, wRow); + return wRow; + } + + public void append(Writable o) throws HiveException + { + elems.append(o); + sz++; + } + + public void append(Object o) throws HiveException + { + try + { + append(serDe.serialize(o, OI)); + } + catch(SerDeException e) + { + throw new HiveException(e); + } + } + + public int size() + { + return sz; + } + + public PTFPartitionIterator iterator() + { + return new PItr(0, size()); + } + + public PTFPartitionIterator range(int start, int end) + { + assert(start >= 0); + assert(end < size()); + assert(start <= end); + return new PItr(start, end); + } + + class PItr implements PTFPartitionIterator + { + int idx; + final int start; + final int end; + final int createTimeSz; + + PItr(int start, int end) + { + this.idx = start; + this.start = start; + this.end = end; + createTimeSz = PTFPartition.this.size(); + } + + public boolean hasNext() + { + checkForComodification() ; + return idx < end; + } + + public Object next() + { + checkForComodification(); + try + { + return PTFPartition.this.getAt(idx++); + } + catch(HiveException e) + { + throw new RuntimeException(e); + } + } + + public void remove() + { + throw new UnsupportedOperationException(); + } + + final void checkForComodification() + { + if (createTimeSz != PTFPartition.this.size()) { + throw new ConcurrentModificationException(); + } + } + + @Override + public int getIndex() + { + return idx; + } + + private Object getAt(int i) + { + try + { + return PTFPartition.this.getAt(i); + } + catch(HiveException e) + { + throw new RuntimeException(e); + } + } + + @Override + public Object lead(int amt) + { + int i = idx + amt; + i = i >= end ? end - 1 : i; + return getAt(i); + } + + @Override + public Object lag(int amt) + { + int i = idx - amt; + i = i < start ? start : i; + return getAt(i); + } + + @Override + public Object resetToIndex(int idx) + { + if ( idx < start || idx >= end ) + { + return null; + } + Object o = getAt(idx); + this.idx = idx + 1; + return o; + } + + @Override + public PTFPartition getPartition() + { + return PTFPartition.this; + } + + @Override + public void reset() + { + idx = start; + } + }; + + /* + * provide an Iterator on the rows in a Partiton. + * Iterator exposes the index of the next location. + * Client can invoke lead/lag relative to the next location. + */ + public static interface PTFPartitionIterator extends Iterator + { + int getIndex(); + + T lead(int amt); + + T lag(int amt); + + /* + * after a lead and lag call, allow Object associated with SerDe and writable associated with partition to be reset + * to the value for the current Index. + */ + Object resetToIndex(int idx); + + PTFPartition getPartition(); + + void reset(); + } + + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/PTFPersistence.java ql/src/java/org/apache/hadoop/hive/ql/exec/PTFPersistence.java new file mode 100644 index 0000000..bab8bb8 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/PTFPersistence.java @@ -0,0 +1,1054 @@ +package org.apache.hadoop.hive.ql.exec; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.lang.ref.SoftReference; +import java.lang.reflect.Constructor; +import java.nio.ByteBuffer; +import java.nio.IntBuffer; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.ConcurrentModificationException; +import java.util.Iterator; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.Deserializer; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.Serializer; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.io.Writable; + +/* + * contains all the classes to support persisting a PTF partition, + */ +public class PTFPersistence { + + @SuppressWarnings("unchecked") + public static ByteBasedList createList(String clsName, int capacity) throws HiveException + { + try + { + Class cls = (Class) Class.forName(clsName); + Constructor cons = cls.getConstructor(Integer.TYPE); + return cons.newInstance(capacity); + } + catch(Exception e) + { + throw new HiveException(e); + } + } + + public static class ByteBasedList + { + int startOffset; + + /* + * (offset,size) of Writables. + * entry i at position i << 1 + * this array is resizable. + */ + int[] offsetsArray; + + /* + * contains actual bytes of Writables. + * not resizable + */ + byte[] bytes; + int bytesUsed; + + int currentSize; + ReentrantReadWriteLock lock; + volatile long lastModified; + + + public ByteBasedList(int startOffset, int capacity) + { + this.startOffset = startOffset; + bytes = new byte[capacity]; + offsetsArray = new int[INCREMENT_SIZE]; + bytesUsed = 0; + currentSize = 0; + lock = new ReentrantReadWriteLock(); + lastModified = System.nanoTime(); + } + + public ByteBasedList() + { + this(0, MEDIUM_SIZE); + } + + public ByteBasedList(int capacity) + { + this(0, capacity); + } + + /* + * internal api; used by {@link PersistentByteBasedList} to setup BBList from a file. + */ + protected ByteBasedList(File file) + { + lock = new ReentrantReadWriteLock(); + } + + private void ensureCapacity(int wlen) throws ListFullException + { + if ( bytesUsed + wlen > bytes.length) + { + throw new ListFullException(); + } + + if ( (2 * currentSize + 1) > offsetsArray.length ) + { + int[] na = new int[offsetsArray.length + INCREMENT_SIZE]; + System.arraycopy(offsetsArray, 0, na, 0, offsetsArray.length); + offsetsArray = na; + } + } + + private int index(int i) throws HiveException + { + int j = i - startOffset; + j = j << 1; + if ( j > 2 * currentSize ) + { + throw new HiveException(PTFUtils.sprintf("index invalid %d", i)); + } + return j; + } + + private void write(Writable w) throws HiveException, IOException + { + DataOStream dos = PTFPersistence.dos.get(); + ByteArrayOS bos = dos.getUnderlyingStream(); + bos.reset(); + w.write(dos); + ensureCapacity(bos.len()); + int i = currentSize * 2; + System.arraycopy(bos.bytearray(), 0, bytes, bytesUsed, bos.len()); + offsetsArray[i] = bytesUsed; + offsetsArray[i+1] = bos.len(); + currentSize += 1; + bytesUsed += bos.len(); + lastModified = System.nanoTime(); + } + + + public int size() throws HiveException + { + PTFPersistence.lock(lock.readLock()); + try + { + return currentSize; + } + finally + { + lock.readLock().unlock(); + } + } + + public void get(int i, Writable wObj) throws HiveException + { + PTFPersistence.lock(lock.readLock()); + try + { + i = index(i); + DataIStream dis = PTFPersistence.dis.get(); + ByteArrayIS bis = dis.getUnderlyingStream(); + bis.setBuffer(bytes, offsetsArray[i], offsetsArray[i+1]); + wObj.readFields(dis); + } + catch(IOException ie) + { + throw new HiveException(ie); + } + finally + { + lock.readLock().unlock(); + } + } + + public void append(Writable obj) throws HiveException + { + PTFPersistence.lock(lock.writeLock()); + try + { + write(obj); + } + catch(IOException ie) + { + throw new HiveException(ie); + } + finally + { + lock.writeLock().unlock(); + } + + } + + public Object get(int i, Deserializer deserializer, Writable wObj) throws HiveException + { + try + { + get(i, wObj); + return deserializer.deserialize(wObj); + } + catch(SerDeException ie) + { + throw new HiveException(ie); + } + } + + public void append(Object obj, ObjectInspector OI, Serializer serializer) throws HiveException + { + try + { + append(serializer.serialize(obj, OI)); + } + catch(SerDeException ie) + { + throw new HiveException(ie); + } + } + + public Iterator iterator(Writable wObj) throws HiveException + { + return new WIterator(wObj, startOffset); + } + + public Iterator iterator(Deserializer deserializer, Writable wObj) throws HiveException + { + return new OIterator(deserializer, wObj); + } + + public void dump(StringBuilder bldr, Writable wObj) throws IOException, HiveException + { + bldr.append("["); + Iterator wi = iterator(wObj); + while(wi.hasNext()) + { + wObj = wi.next(); + bldr.append(wObj).append(", "); + } + bldr.append("]\n"); + } + + public void dump(StringBuilder bldr, Deserializer deserializer, Writable wObj) throws IOException, HiveException + { + bldr.append("["); + Iterator oi = iterator(deserializer, wObj); + while(oi.hasNext()) + { + bldr.append(oi.next()).append(", "); + } + bldr.append("]\n"); + } + + class WIterator implements Iterator + { + Writable wObj; + long checkTime; + int i; + + WIterator(Writable wObj, int offset) + { + this.wObj = wObj; + checkTime = lastModified; + i = offset; + } + + @Override + public boolean hasNext() + { + return i < currentSize; + } + + @Override + public Writable next() + { + if (checkTime != lastModified) { + throw new ConcurrentModificationException(); + } + try + { + get(i++, wObj); + return wObj; + } + catch(HiveException be) + { + throw new RuntimeException(be); + } + } + + @Override + public void remove() + { + throw new UnsupportedOperationException(); + } + } + + class OIterator implements Iterator + { + Deserializer deserializer; + Iterator wi; + + OIterator(Deserializer deserializer, Writable wObj) throws HiveException + { + wi = iterator(wObj); + this.deserializer = deserializer; + } + + @Override + public boolean hasNext() + { + return wi.hasNext(); + } + + @Override + public Object next() + { + Writable wObj = wi.next(); + try + { + return deserializer.deserialize(wObj); + }catch(SerDeException se) + { + throw new RuntimeException(se); + } + } + + @Override + public void remove() + { + throw new UnsupportedOperationException(); + } + } + + public static class ListFullException extends HiveException + { + private static final long serialVersionUID = 4745303310812778989L; + + public ListFullException() + { + super(); + } + + public ListFullException(String message, Throwable cause) + { + super(message, cause); + } + + public ListFullException(String message) + { + super(message); + } + + public ListFullException(Throwable cause) + { + super(cause); + } + + } + + private static final int INCREMENT_SIZE = (int) Math.pow(2, 16); + + public static final int SMALL_SIZE = (int) Math.pow(2, 6 +10); // 64KB + public static final int MEDIUM_SIZE = (int) Math.pow(2, (10 + 10 + 3)); // 8 MB + public static final int LARGE_SIZE = (int) Math.pow(2, (6 + 10 + 10)); // 64 MB + + } + + public static class PartitionedByteBasedList extends ByteBasedList + { + ArrayList partitions; + ArrayList partitionOffsets; + File dir; + int batchSize; + + public PartitionedByteBasedList(int batchSize) throws HiveException + { + this.batchSize = batchSize; + currentSize = 0; + dir = PartitionedByteBasedList.createTempDir(); + Runtime.getRuntime().addShutdownHook(new ShutdownHook(dir)); + + partitions = new ArrayList(); + partitionOffsets = new ArrayList(); + addPartition(); + } + + public PartitionedByteBasedList() throws HiveException + { + this(ByteBasedList.LARGE_SIZE); + } + + private void addPartition() throws HiveException + { + try + { + if ( partitions.size() > 0 ) + { + int idx = partitions.size() - 1; + ByteBasedList bl = partitions.get(idx); + File f = File.createTempFile("wdw", null, dir); + PersistentByteBasedList.store(bl, f); + partitions.set(idx, new PersistentByteBasedList(f, bl)); + + } + ByteBasedList bl = new ByteBasedList(currentSize, batchSize); + partitions.add(bl); + partitionOffsets.add(currentSize); + } + catch(IOException ie) + { + throw new HiveException(ie); + } + } + + private ByteBasedList getPartition(int i) throws HiveException + { + PTFPersistence.lock(lock.readLock()); + try + { + int numSplits = partitions.size(); + if ( numSplits == 0) { + return partitions.get(0); + } + int start = 0; + int end = numSplits - 1; + + while(start < end) + { + int mid = (start + end + 1) >>> 1; + int val = partitionOffsets.get(mid); + if ( val == i ) + { + return partitions.get(mid); + } + else if ( val < i ) + { + if ( end == mid) + { + return partitions.get(end); + } + start = mid; + } + else + { + end = mid - 1; + } + } + return partitions.get(start); + } + finally + { + lock.readLock().unlock(); + } + } + + @Override + public void get(int i, Writable wObj) throws HiveException + { + ByteBasedList bl = getPartition(i); + bl.get(i, wObj); + } + + @Override + public void append(Writable obj) throws HiveException + { + PTFPersistence.lock(lock.writeLock()); + try + { + partitions.get(partitions.size() -1).append(obj); + currentSize += 1; + lastModified = System.nanoTime(); + } + catch(ListFullException le) + { + addPartition(); + append(obj); + } + finally + { + lock.writeLock().unlock(); + } + + } + + @Override + public Object get(int i, Deserializer deserializer, Writable wObj) throws HiveException + { + ByteBasedList bl = getPartition(i); + return bl.get(i, deserializer, wObj); + } + + @Override + public void append(Object obj, ObjectInspector OI, Serializer serializer) throws HiveException + { + PTFPersistence.lock(lock.writeLock()); + try + { + partitions.get(partitions.size() -1).append(obj, OI, serializer); + currentSize += 1; + lastModified = System.nanoTime(); + } + catch(ListFullException le) + { + addPartition(); + append(obj, OI, serializer); + } + finally + { + lock.writeLock().unlock(); + } + } + + @Override + public Iterator iterator(Writable wObj) throws HiveException + { + return new WIterator(wObj); + } + + class WIterator implements Iterator + { + Writable wObj; + long checkTime; + int i; + Iterator pIter; + + WIterator(Writable wObj) throws HiveException + { + this.wObj = wObj; + checkTime = lastModified; + i = 0; + pIter = partitions.get(i).iterator(wObj); + } + + @Override + public boolean hasNext() + { + if ( pIter.hasNext() ) { + return true; + } + if (checkTime != lastModified) { + throw new ConcurrentModificationException(); + } + try + { + if ( i < partitions.size() ) + { + pIter = partitions.get(i++).iterator(wObj); + return hasNext(); + } + return false; + } + catch(HiveException e) + { + throw new RuntimeException(e); + } + } + + @Override + public Writable next() + { + if (checkTime != lastModified) { + throw new ConcurrentModificationException(); + } + return pIter.next(); + } + + @Override + public void remove() + { + throw new UnsupportedOperationException(); + } + } + + static class ShutdownHook extends Thread + { + File dir; + + public ShutdownHook(File dir) + { + this.dir = dir; + } + + @Override + public void run() + { + try + { + PartitionedByteBasedList.deleteRecursively(dir); + } + catch(IOException ie) + { + } + } + + } + + // copied completely from guavar09 source + /** + * Deletes a file or directory and all contents recursively. + * + *

+ * If the file argument is a symbolic link the link will be deleted but not + * the target of the link. If the argument is a directory, symbolic links + * within the directory will not be followed. + * + * @param file + * the file to delete + * @throws IOException + * if an I/O error occurs + * @see #deleteDirectoryContents + */ + public static void deleteRecursively(File file) throws IOException + { + if (file.isDirectory()) + { + deleteDirectoryContents(file); + } + if (!file.delete()) + { + throw new IOException("Failed to delete " + file); + } + } + + // copied completely from guavar09 source + /** + * Deletes all the files within a directory. Does not delete the directory + * itself. + * + *

+ * If the file argument is a symbolic link or there is a symbolic link in + * the path leading to the directory, this method will do nothing. Symbolic + * links within the directory are not followed. + * + * @param directory + * the directory to delete the contents of + * @throws IllegalArgumentException + * if the argument is not a directory + * @throws IOException + * if an I/O error occurs + * @see #deleteRecursively + */ + public static void deleteDirectoryContents(File directory) + throws IOException + { + /*Preconditions.checkArgument(directory.isDirectory(), + "Not a directory: %s", directory); + */ + if ( !directory.isDirectory()) + { + throw new IOException(PTFUtils.sprintf("Not a directory: %s", directory)); + } + + // Symbolic links will have different canonical and absolute paths + if (!directory.getCanonicalPath().equals(directory.getAbsolutePath())) + { + return; + } + File[] files = directory.listFiles(); + if (files == null) + { + throw new IOException("Error listing files for " + directory); + } + for (File file : files) + { + deleteRecursively(file); + } + } + + // copied completely from guava to remove dependency on guava + /** Maximum loop count when creating temp directories. */ + private static final int TEMP_DIR_ATTEMPTS = 10000; + public static File createTempDir() + { + File baseDir = new File(System.getProperty("java.io.tmpdir")); + String baseName = System.currentTimeMillis() + "-"; + + for (int counter = 0; counter < TEMP_DIR_ATTEMPTS; counter++) + { + File tempDir = new File(baseDir, baseName + counter); + if (tempDir.mkdir()) + { + return tempDir; + } + } + throw new IllegalStateException("Failed to create directory within " + + TEMP_DIR_ATTEMPTS + " attempts (tried " + baseName + "0 to " + + baseName + (TEMP_DIR_ATTEMPTS - 1) + ')'); + } + + } + + static class PersistentByteBasedList extends ByteBasedList + { + private static int headerSize() { return (Integer.SIZE + Integer.SIZE + Integer.SIZE + Long.SIZE) / Byte.SIZE;} + protected static void store(ByteBasedList l, File f) throws IOException + { + /* + * write startOffset:bytesUsed:currentSize:lastModified + */ + int hdrSize = headerSize(); + ByteBuffer buf = ByteBuffer.allocate(hdrSize); + + buf.putInt(l.startOffset); + buf.putInt(l.bytesUsed); + buf.putInt(l.currentSize); + buf.putLong(l.lastModified); + buf.flip(); + + /* + * note: could save this space by using Memory-Mapped I/O and directly writing to the MM buffer. + */ + ByteBuffer offsetB = ByteBuffer.allocate((Integer.SIZE/Byte.SIZE) * 2 * l.currentSize); + IntBuffer iB = offsetB.asIntBuffer(); + iB.put(l.offsetsArray, 0, l.currentSize * 2); + + ByteBuffer bytesB = ByteBuffer.wrap(l.bytes, 0, l.bytesUsed); + + ByteBuffer[] bufs = new ByteBuffer[] { buf, offsetB, bytesB}; + FileOutputStream fos = new FileOutputStream(f); + try + { + FileChannel fc = fos.getChannel(); + while (fc.write(bufs, 0, bufs.length) > 0) { + ; + } + } + finally + { + fos.close(); + } + } + + protected static void load(ByteBasedList l, File f) throws IOException + { + int hdr = headerSize(); + FileInputStream fis = new FileInputStream(f); + try + { + FileChannel fc = fis.getChannel(); + ByteBuffer buf0 = ByteBuffer.allocate(hdr); + while (buf0.hasRemaining()) { + fc.read(buf0); + } + buf0.flip(); + l.startOffset = buf0.getInt(); + l.bytesUsed = buf0.getInt(); + l.currentSize = buf0.getInt(); + l.lastModified = buf0.getLong(); + + /* + * note: could save this space by using Memory-Mapped I/O and directly writing to the MM buffer. + */ + ByteBuffer offsetB = ByteBuffer.allocate((Integer.SIZE/Byte.SIZE) * 2 * l.currentSize); + ByteBuffer bytesB = ByteBuffer.allocate(l.bytesUsed); + ByteBuffer[] bufs = new ByteBuffer[] { offsetB, bytesB }; + while (fc.read(bufs) > 0) { + ; + } + + l.offsetsArray = new int[l.currentSize * 2]; + offsetB.flip(); + IntBuffer iB = offsetB.asIntBuffer(); + iB.get(l.offsetsArray); + l.bytes = bytesB.array(); + } + finally + { + fis.close(); + } + } + + File file; + SoftReference memList; + + protected PersistentByteBasedList(File file, ByteBasedList l) + { + super(file); + this.file = file; + memList = new SoftReference(l); + } + + protected PersistentByteBasedList(File file) + { + this(file, null); + } + + private ByteBasedList getList() throws HiveException + { + PTFPersistence.lock(lock.readLock()); + try + { + ByteBasedList list = memList.get(); + if (list == null) + { + try + { + list = new ByteBasedList(file); + load(list, file); + memList = new SoftReference(list); + } + catch(Exception ie) + { + throw new RuntimeException(ie); + } + } + return list; + } + finally + { + lock.readLock().unlock(); + } + } + + @Override + public int size() throws HiveException + { + return getList().size(); + } + + @Override + public void get(int i, Writable wObj) throws HiveException + { + getList().get(i, wObj); + } + + @Override + public void append(Writable obj) throws HiveException + { + throw new UnsupportedOperationException("Cannot append to a Persisted List"); + } + + @Override + public Object get(int i, Deserializer deserializer, Writable wObj) throws HiveException + { + return getList().get(i, deserializer, wObj); + } + + @Override + public void append(Object obj, ObjectInspector OI, Serializer serializer) throws HiveException + { + throw new UnsupportedOperationException("Cannot append to a Persisted List"); + } + + @Override + public Iterator iterator(Writable wObj) throws HiveException + { + return getList().iterator(wObj); + } + + @Override + public Iterator iterator(Deserializer deserializer, Writable wObj) throws HiveException + { + return getList().iterator(deserializer, wObj); + } + + @Override + public void dump(StringBuilder bldr, Writable wObj) throws IOException, HiveException + { + getList().dump(bldr, wObj); + } + + @Override + public void dump(StringBuilder bldr, Deserializer deserializer, Writable wObj) throws IOException, HiveException + { + getList().dump(bldr, deserializer, wObj); + } + } + + public static class ByteBufferInputStream extends InputStream + { + ByteBuffer buffer; + int mark = -1; + + public void intialize(ByteBuffer buffer) + { + this.buffer = buffer; + } + + public void intialize(ByteBuffer buffer, int off, int len) + { + buffer = buffer.duplicate(); + buffer.position(off); + buffer.limit(off + len); + this.buffer = buffer.slice(); + } + + @Override + public int read() throws IOException + { + return buffer.hasRemaining() ? (buffer.get() & 0xff) : -1; + } + + @Override + public int read(byte b[], int off, int len) throws IOException + { + int remaining = buffer.remaining(); + len= len <= remaining ? len : remaining; + buffer.get(b, off, len); + return len; + } + + @Override + public boolean markSupported() { return true; } + + @Override + public void mark(int readAheadLimit) + { + mark = buffer.position(); + } + + @Override + public void reset() + { + if ( mark == -1 ) { + throw new IllegalStateException(); + } + buffer.position(mark); + mark = -1; + } + } + + public static class ByteBufferOutputStream extends OutputStream + { + ByteBuffer buffer; + + public void intialize(ByteBuffer buffer) + { + this.buffer = buffer; + } + + public void intialize(ByteBuffer buffer, int off, int len) + { + buffer = buffer.duplicate(); + buffer.position(off); + buffer.limit(off + len); + this.buffer = buffer.slice(); + } + + @Override + public void write(int b) throws IOException + { + buffer.put((byte) b); + } + + @Override + public void write(byte b[], int off, int len) + { + int remaining = buffer.remaining(); + if ( len > remaining ) + { + throw new IndexOutOfBoundsException(); + } + buffer.put(b, off, len); + } + } + + public static ThreadLocal bis = new ThreadLocal() + { + @Override + protected ByteArrayIS initialValue() + { + return new ByteArrayIS(); + } + }; + + public static ThreadLocal dis = new ThreadLocal() + { + @Override + protected DataIStream initialValue() + { + return new DataIStream(bis.get()); + } + }; + + public static ThreadLocal bos = new ThreadLocal() + { + @Override + protected ByteArrayOS initialValue() + { + return new ByteArrayOS(); + } + }; + + public static ThreadLocal dos = new ThreadLocal() + { + @Override + protected DataOStream initialValue() + { + return new DataOStream(bos.get()); + } + }; + + + public static class DataIStream extends DataInputStream + { + public DataIStream(ByteArrayIS in) + { + super(in); + } + + public ByteArrayIS getUnderlyingStream() { return (ByteArrayIS) in; } + } + + public static class DataOStream extends DataOutputStream + { + public DataOStream(ByteArrayOS out) + { + super(out); + } + + public ByteArrayOS getUnderlyingStream() { return (ByteArrayOS) out; } + } + + public static class ByteArrayOS extends ByteArrayOutputStream + { + public ByteArrayOS() { super(); } + public ByteArrayOS(int size) { super(size); } + public final byte[] bytearray() { return buf; } + public final int len() { return count; } + + } + + public static class ByteArrayIS extends ByteArrayInputStream + { + public ByteArrayIS() { super(new byte[0]); } + public final byte[] bytearray() { return buf; } + public final void setBuffer(byte[] buf, int offset, int len) + { + this.buf = buf; + this.pos = offset; + this.count = Math.min(offset + len, buf.length); + this.mark = offset; + } + + } + + public static void lock(Lock lock) throws HiveException + { + try + { + lock.lockInterruptibly(); + + } + catch(InterruptedException ie) + { + Thread.currentThread().interrupt(); + throw new HiveException("Operation interrupted", ie); + } + } + + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/PTFUtils.java ql/src/java/org/apache/hadoop/hive/ql/exec/PTFUtils.java new file mode 100644 index 0000000..e795b62 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/PTFUtils.java @@ -0,0 +1,287 @@ +package org.apache.hadoop.hive.ql.exec; + +import java.beans.BeanInfo; +import java.beans.Encoder; +import java.beans.ExceptionListener; +import java.beans.Expression; +import java.beans.IntrospectionException; +import java.beans.Introspector; +import java.beans.PersistenceDelegate; +import java.beans.PropertyDescriptor; +import java.beans.Statement; +import java.beans.XMLDecoder; +import java.beans.XMLEncoder; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PrintStream; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Stack; + +import org.antlr.runtime.CommonToken; +import org.antlr.runtime.tree.BaseTree; +import org.antlr.runtime.tree.CommonTree; +import org.apache.hadoop.hive.ql.exec.Utilities.EnumDelegate; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowFrameSpec.Direction; +import org.apache.hadoop.hive.ql.plan.PTFDef; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; + +public class PTFUtils { + public static String sprintf(String format, Object... values) + { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(outputStream); + out.printf(format, values); + return outputStream.toString(); + } + + public static String toString(List col) + { + StringBuilder buf = new StringBuilder(); + buf.append("["); + boolean first = true; + for (Object o : col) + { + if (first) { + first = false; + } else { + buf.append(", "); + } + buf.append(o.toString()); + } + buf.append("]"); + return buf.toString(); + } + + public static String toString(Map col) + { + StringBuilder buf = new StringBuilder(); + buf.append("["); + boolean first = true; + for (Map.Entry o : col.entrySet()) + { + if (first) { + first = false; + } else { + buf.append(", "); + } + buf.append(o.getKey().toString()).append(" : ") + .append(o.getValue().toString()); + } + buf.append("]"); + return buf.toString(); + } + + public static String unescapeQueryString(String qry) + { + qry = qry.replace("\\\"", "\""); + qry = qry.replace("\\'", "'"); + return qry; + } + + public static class ReverseIterator implements Iterator + { + Stack stack; + + public ReverseIterator(Iterator it) + { + stack = new Stack(); + while (it.hasNext()) + { + stack.push(it.next()); + } + } + + @Override + public boolean hasNext() + { + return !stack.isEmpty(); + } + + @Override + public T next() + { + return stack.pop(); + } + + @Override + public void remove() + { + throw new UnsupportedOperationException(); + } + } + + public static abstract class Predicate + { + public abstract boolean apply(T obj); + }; + + + + /* + * serialization functions + */ + public static void serialize(OutputStream out, Object o) + { + XMLEncoder e = new XMLEncoder(out); + e.setExceptionListener(new EL()); + PTFUtils.addPersistenceDelegates(e); + e.writeObject(o); + e.close(); + } + + public static Object deserialize(InputStream in1) + { + XMLDecoder d = null; + try + { + d = new XMLDecoder(in1, null, null); + return d.readObject(); + } + finally + { + if (null != d) + { + d.close(); + } + } + } + + public static void addPersistenceDelegates(XMLEncoder e) + { + addAntlrPersistenceDelegates(e); + addHivePersistenceDelegates(e); + addEnumDelegates(e); + } + + public static void addEnumDelegates(XMLEncoder e) + { + e.setPersistenceDelegate(Direction.class, new EnumDelegate()); + } + + public static void addAntlrPersistenceDelegates(XMLEncoder e) + { + e.setPersistenceDelegate(ASTNode.class, new PersistenceDelegate() + { + + @Override + protected Expression instantiate(Object oldInstance, Encoder out) + { + return new Expression(oldInstance, oldInstance.getClass(), + "new", new Object[] + { ((ASTNode) oldInstance).getToken() }); + } + }); + e.setPersistenceDelegate(CommonTree.class, new PersistenceDelegate() + { + @Override + protected Expression instantiate(Object oldInstance, Encoder out) + { + return new Expression(oldInstance, oldInstance.getClass(), + "new", new Object[] + { ((CommonTree) oldInstance).getToken() }); + } + }); + e.setPersistenceDelegate(BaseTree.class, new PersistenceDelegate() + { + @Override + protected Expression instantiate(Object oldInstance, Encoder out) + { + return new Expression(oldInstance, oldInstance.getClass(), + "new", new Object[] + {}); + } + + @Override + @SuppressWarnings("rawtypes") + protected void initialize(Class type, Object oldInstance, + Object newInstance, Encoder out) + { + super.initialize(type, oldInstance, newInstance, out); + + BaseTree t = (BaseTree) oldInstance; + + for (int i = 0; i < t.getChildCount(); i++) + { + out.writeStatement(new Statement(oldInstance, "addChild", + new Object[] + { t.getChild(i) })); + } + } + }); + e.setPersistenceDelegate(CommonToken.class, new PersistenceDelegate() + { + @Override + protected Expression instantiate(Object oldInstance, Encoder out) + { + return new Expression(oldInstance, oldInstance.getClass(), + "new", new Object[] + { ((CommonToken) oldInstance).getType(), + ((CommonToken) oldInstance).getText() }); + } + }); + } + + public static void addHivePersistenceDelegates(XMLEncoder e) + { + e.setPersistenceDelegate(PrimitiveTypeInfo.class, + new PersistenceDelegate() + { + @Override + protected Expression instantiate(Object oldInstance, + Encoder out) + { + return new Expression(oldInstance, + TypeInfoFactory.class, "getPrimitiveTypeInfo", + new Object[] + { ((PrimitiveTypeInfo) oldInstance) + .getTypeName() }); + } + }); + } + + static class EL implements ExceptionListener + { + public void exceptionThrown(Exception e) + { + e.printStackTrace(); + throw new RuntimeException("Cannot serialize the query plan", e); + } + } + + public static String serializeQueryDef(PTFDef qdef) + { + String queryDef = null; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + PTFUtils.serialize(out, qdef); + queryDef = out.toString(); + return queryDef; + } + + public static void makeTransient(Class beanClass, String pdName) + { + BeanInfo info; + try + { + info = Introspector.getBeanInfo(beanClass); + PropertyDescriptor[] propertyDescriptors = info + .getPropertyDescriptors(); + for (int i = 0; i < propertyDescriptors.length; ++i) + { + PropertyDescriptor pd = propertyDescriptors[i]; + if (pd.getName().equals(pdName)) + { + pd.setValue("transient", Boolean.TRUE); + } + } + } + catch (IntrospectionException ie) + { + throw new RuntimeException(ie); + } + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/PartitionTableFunctionDescription.java ql/src/java/org/apache/hadoop/hive/ql/exec/PartitionTableFunctionDescription.java new file mode 100644 index 0000000..738db56 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/PartitionTableFunctionDescription.java @@ -0,0 +1,22 @@ +package org.apache.hadoop.hive.ql.exec; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.apache.hadoop.hive.ql.udf.ptf.WindowingTableFunction; + +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE}) +@Documented +public @interface PartitionTableFunctionDescription +{ + Description description (); + + /** + * if true it is not usable in the language. {@link WindowingTableFunction} is the only internal function. + */ + boolean isInternal() default false; +} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionDescription.java ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionDescription.java new file mode 100644 index 0000000..698278a --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionDescription.java @@ -0,0 +1,29 @@ +package org.apache.hadoop.hive.ql.exec; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFResolver2; +import org.apache.hadoop.hive.ql.udf.ptf.WindowingTableFunction; + +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE}) +@Documented +public @interface WindowFunctionDescription +{ + Description description (); + /** + * controls whether this function can be applied to a Window + */ + boolean supportsWindow() default true; + /** + * A WindowFunc is implemented as {@link GenericUDAFResolver2}. It returns only one value. + * If this is true then the function must return a List which is taken to be the column for this function in the Output table returned by the + * {@link WindowingTableFunction}. Otherwise the output is assumed to be a single value, the column of the Output will contain the same value + * for all the rows. + */ + boolean pivotResult() default false; +} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionInfo.java ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionInfo.java new file mode 100644 index 0000000..b8c1582 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionInfo.java @@ -0,0 +1,41 @@ +package org.apache.hadoop.hive.ql.exec; + +import org.apache.hadoop.hive.ql.exec.FunctionInfo; +import org.apache.hadoop.hive.ql.exec.WindowFunctionDescription; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFResolver; + +@SuppressWarnings("deprecation") +public class WindowFunctionInfo +{ + boolean supportsWindow = true; + boolean pivotResult = false; + FunctionInfo fInfo; + + WindowFunctionInfo(FunctionInfo fInfo) + { + assert fInfo.isGenericUDAF(); + this.fInfo = fInfo; + Class wfnCls = fInfo.getGenericUDAFResolver().getClass(); + WindowFunctionDescription def = wfnCls.getAnnotation(WindowFunctionDescription.class); + if ( def != null) + { + supportsWindow = def.supportsWindow(); + pivotResult = def.pivotResult(); + } + } + + public boolean isSupportsWindow() + { + return supportsWindow; + } + + public boolean isPivotResult() + { + return pivotResult; + } + + public FunctionInfo getfInfo() + { + return fInfo; + } +} \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageInfo.java ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageInfo.java index b687a2f..22a8785 100644 --- ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageInfo.java +++ ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageInfo.java @@ -47,7 +47,8 @@ * set operations like union on columns on other tables * e.g. T2.c1 = T1.c1 + T3.c1. * 4. SCRIPT - Indicates that the column is derived from the output - * of a user script through a TRANSFORM, MAP or REDUCE syntax. + * of a user script through a TRANSFORM, MAP or REDUCE syntax + * or from the output of a PTF chain execution. */ public static enum DependencyType { SIMPLE, EXPRESSION, SCRIPT diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java index ccbbd28..51bef04 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java @@ -22,17 +22,17 @@ import java.util.LinkedHashMap; import java.util.Map; -import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.TableScanOperator; -import org.apache.hadoop.hive.ql.exec.SelectOperator; -import org.apache.hadoop.hive.ql.exec.ScriptOperator; +import org.apache.hadoop.hive.ql.exec.CommonJoinOperator; import org.apache.hadoop.hive.ql.exec.GroupByOperator; +import org.apache.hadoop.hive.ql.exec.LateralViewJoinOperator; import org.apache.hadoop.hive.ql.exec.MapJoinOperator; +import org.apache.hadoop.hive.ql.exec.PTFOperator; +import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; +import org.apache.hadoop.hive.ql.exec.ScriptOperator; +import org.apache.hadoop.hive.ql.exec.SelectOperator; +import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.UDTFOperator; import org.apache.hadoop.hive.ql.exec.UnionOperator; -import org.apache.hadoop.hive.ql.exec.CommonJoinOperator; -import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; -import org.apache.hadoop.hive.ql.exec.LateralViewJoinOperator; import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; import org.apache.hadoop.hive.ql.lib.Dispatcher; import org.apache.hadoop.hive.ql.lib.GraphWalker; @@ -82,6 +82,8 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { OpProcFactory.getReduceSinkProc()); opRules.put(new RuleRegExp("R9", LateralViewJoinOperator.getOperatorName() + "%"), OpProcFactory.getLateralViewJoinProc()); + opRules.put(new RuleRegExp("R10", PTFOperator.getOperatorName() + "%"), + OpProcFactory.getTransformProc()); // The dispatcher fires the processor corresponding to the closest matching rule and passes the context along Dispatcher disp = new DefaultRuleDispatcher(OpProcFactory.getDefaultProc(), opRules, lCtx); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/ASTNode.java ql/src/java/org/apache/hadoop/hive/ql/parse/ASTNode.java index a6fe4c8..7dbd96f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/ASTNode.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ASTNode.java @@ -37,10 +37,6 @@ public ASTNode() { } - public ASTNode(ASTNode copy){ - super(copy); - } - /** * Constructor. * @@ -51,6 +47,16 @@ public ASTNode(Token t) { super(t); } + public ASTNode(ASTNode node) { + super(node); + this.origin = node.origin; + } + + @Override + public Tree dupNode() { + return new ASTNode(this); + } + /* * (non-Javadoc) * @@ -95,12 +101,6 @@ public void setOrigin(ASTNodeOrigin origin) { this.origin = origin; } - @Override - public Tree dupNode() { - - return new ASTNode(this); - } - public String dump() { StringBuilder sb = new StringBuilder(); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g index b3a8aca..235fa5b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g @@ -284,6 +284,12 @@ TOK_SKEWED_LOCATIONS; TOK_SKEWED_LOCATION_LIST; TOK_SKEWED_LOCATION_MAP; TOK_STOREDASDIRS; +TOK_PARTITIONINGSPEC; +TOK_PTBLFUNCTION; +TOK_WINDOWDEF; +TOK_WINDOWSPEC; +TOK_WINDOWVALUES; +TOK_WINDOWRANGE; TOK_IGNOREPROTECTION; } @@ -1555,9 +1561,10 @@ regular_body clusterByClause? distributeByClause? sortByClause? + window_clause? limitClause? -> ^(TOK_QUERY fromClause ^(TOK_INSERT insertClause selectClause whereClause? groupByClause? havingClause? orderByClause? clusterByClause? - distributeByClause? sortByClause? limitClause?)) + distributeByClause? sortByClause? window_clause? limitClause?)) | selectStatement ; @@ -1573,9 +1580,10 @@ selectStatement clusterByClause? distributeByClause? sortByClause? + window_clause? limitClause? -> ^(TOK_QUERY fromClause ^(TOK_INSERT ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE)) selectClause whereClause? groupByClause? havingClause? orderByClause? clusterByClause? - distributeByClause? sortByClause? limitClause?)) + distributeByClause? sortByClause? window_clause? limitClause?)) ; @@ -1590,9 +1598,10 @@ body clusterByClause? distributeByClause? sortByClause? + window_clause? limitClause? -> ^(TOK_INSERT insertClause? selectClause whereClause? groupByClause? havingClause? orderByClause? clusterByClause? - distributeByClause? sortByClause? limitClause?) + distributeByClause? sortByClause? window_clause? limitClause?) | selectClause whereClause? @@ -1602,9 +1611,10 @@ body clusterByClause? distributeByClause? sortByClause? + window_clause? limitClause? -> ^(TOK_INSERT ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE)) selectClause whereClause? groupByClause? havingClause? orderByClause? clusterByClause? - distributeByClause? sortByClause? limitClause?) + distributeByClause? sortByClause? window_clause? limitClause?) ; insertClause @@ -1714,7 +1724,10 @@ selectItem @init { msgs.push("selection target"); } @after { msgs.pop(); } : - ( selectExpression ((KW_AS? Identifier) | (KW_AS LPAREN Identifier (COMMA Identifier)* RPAREN))?) -> ^(TOK_SELEXPR selectExpression Identifier*) + ( selectExpression + ((KW_AS? Identifier) | (KW_AS LPAREN Identifier (COMMA Identifier)* RPAREN))? + (KW_OVER LPAREN ws=window_specification RPAREN )? + ) -> ^(TOK_SELEXPR selectExpression Identifier* $ws?) ; trfmClause @@ -1841,7 +1854,7 @@ fromSource @init { msgs.push("from source"); } @after { msgs.pop(); } : - (tableSource | subQuerySource) (lateralView^)* + ((Identifier LPAREN)=> partitionedTableFunction | tableSource | subQuerySource ) (lateralView^)* ; tableBucketSample @@ -1903,6 +1916,95 @@ subQuerySource : LPAREN queryStatementExpression RPAREN Identifier -> ^(TOK_SUBQUERY queryStatementExpression Identifier) ; + +//---------------------- Rules for parsing PTF clauses ----------------------------- +partitioningSpec +@init { msgs.push("partitioningSpec clause"); } +@after { msgs.pop(); } + : + clusterByClause -> ^(TOK_PARTITIONINGSPEC clusterByClause) | + distributeByClause sortByClause? -> ^(TOK_PARTITIONINGSPEC distributeByClause sortByClause?) + ; + +partitionTableFunctionSource +@init { msgs.push("partitionTableFunctionSource clause"); } +@after { msgs.pop(); } + : + subQuerySource | + tableSource | + partitionedTableFunction + ; + +partitionedTableFunction +@init { msgs.push("ptf clause"); } +@after { msgs.pop(); } + : + name=Identifier + LPAREN + ((expression COMMA)=> expression (COMMA expression)* KW_ON)? + ptfsrc=partitionTableFunctionSource partitioningSpec? + RPAREN alias=Identifier? + -> ^(TOK_PTBLFUNCTION $name $alias? partitionTableFunctionSource partitioningSpec? expression*) + ; + +//---------------------- Rules for windowing clauses ------------------------------- +window_clause +@init { msgs.push("window_clause"); } +@after { msgs.pop(); } +: + KW_WINDOW window_defn (COMMA window_defn)* -> ^(KW_WINDOW window_defn+) +; + +window_defn +@init { msgs.push("window_defn"); } +@after { msgs.pop(); } +: + Identifier KW_AS window_specification -> ^(TOK_WINDOWDEF Identifier window_specification) +; + +window_specification +@init { msgs.push("window_specification"); } +@after { msgs.pop(); } +: + Identifier? partitioningSpec? window_frame? -> ^(TOK_WINDOWSPEC Identifier? partitioningSpec? window_frame?) +; + +window_frame : + window_range_expression | + window_value_expression +; + +window_range_expression +@init { msgs.push("window_range_expression"); } +@after { msgs.pop(); } +: + KW_ROWS KW_BETWEEN s=rowsboundary KW_AND end=rowsboundary -> ^(TOK_WINDOWRANGE $s $end) +; + +rowsboundary +@init { msgs.push("rowsboundary"); } +@after { msgs.pop(); } +: + KW_UNBOUNDED (r=KW_PRECEDING|r=KW_FOLLOWING) -> ^($r KW_UNBOUNDED) | + KW_CURRENT KW_ROW -> ^(KW_CURRENT) | + Number (d=KW_PRECEDING | d=KW_FOLLOWING ) -> ^($d Number) +; + +window_value_expression +@init { msgs.push("window_value_expression"); } +@after { msgs.pop(); } +: + KW_RANGE KW_BETWEEN s=valuesboundary KW_AND end=valuesboundary -> ^(TOK_WINDOWVALUES $s $end) +; + +valuesboundary +@init { msgs.push("valuesboundary"); } +@after { msgs.pop(); } +: + KW_UNBOUNDED (r=KW_PRECEDING|r=KW_FOLLOWING) -> ^($r KW_UNBOUNDED) | + KW_CURRENT KW_ROW -> ^(KW_CURRENT) | + rowExp=expression rngExp=Number (d=KW_LESS | d=KW_MORE ) -> ^($d $rowExp $rngExp) +; //----------------------- Rules for parsing whereClause ----------------------------- // where a=b and ... @@ -2001,7 +2103,7 @@ clusterByClause | KW_CLUSTER KW_BY expression - ( COMMA expression )* -> ^(TOK_CLUSTERBY expression+) + ( (COMMA)=> COMMA expression )* -> ^(TOK_CLUSTERBY expression+) ; distributeByClause @@ -2012,7 +2114,7 @@ distributeByClause LPAREN expression (COMMA expression)* RPAREN -> ^(TOK_DISTRIBUTEBY expression+) | KW_DISTRIBUTE KW_BY - expression (COMMA expression)* -> ^(TOK_DISTRIBUTEBY expression+) + expression ((COMMA)=> COMMA expression)* -> ^(TOK_DISTRIBUTEBY expression+) ; sortByClause @@ -2025,7 +2127,7 @@ sortByClause | KW_SORT KW_BY columnRefOrder - ( COMMA columnRefOrder)* -> ^(TOK_SORTBY columnRefOrder+) + ( (COMMA)=> COMMA columnRefOrder)* -> ^(TOK_SORTBY columnRefOrder+) ; // fun(par1, par2, par3) @@ -2038,8 +2140,9 @@ function ( (star=STAR) | (dist=KW_DISTINCT)? (expression (COMMA expression)*)? - ) - RPAREN -> {$star != null}? ^(TOK_FUNCTIONSTAR functionName) + ) + RPAREN + -> {$star != null}? ^(TOK_FUNCTIONSTAR functionName) -> {$dist == null}? ^(TOK_FUNCTION functionName (expression+)?) -> ^(TOK_FUNCTIONDI functionName (expression+)?) ; @@ -2604,6 +2707,14 @@ KW_ROLLUP: 'ROLLUP'; KW_CUBE: 'CUBE'; KW_DIRECTORIES: 'DIRECTORIES'; KW_FOR: 'FOR'; +KW_WINDOW: 'WINDOW'; +KW_UNBOUNDED: 'UNBOUNDED'; +KW_PRECEDING: 'PRECEDING'; +KW_FOLLOWING: 'FOLLOWING'; +KW_CURRENT: 'CURRENT'; +KW_LESS: 'LESS'; +KW_MORE: 'MORE'; +KW_OVER: 'OVER'; KW_GROUPING: 'GROUPING'; KW_SETS: 'SETS'; KW_TRUNCATE: 'TRUNCATE'; diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/PTFSpec.java ql/src/java/org/apache/hadoop/hive/ql/parse/PTFSpec.java new file mode 100644 index 0000000..ed28a05 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/parse/PTFSpec.java @@ -0,0 +1,1378 @@ +package org.apache.hadoop.hive.ql.parse; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.antlr.runtime.tree.CommonTree; +import org.apache.hadoop.hive.ql.exec.FunctionRegistry; +import org.apache.hadoop.hive.ql.exec.PTFUtils; + +public class PTFSpec +{ + PTFInputSpec input; + SelectSpec selectList; + ASTNode whereExpr; + Map windowSpecs; + + public PTFInputSpec getInput() + { + return input; + } + public void setInput(PTFInputSpec input) + { + this.input = input; + } + public ASTNode getWhereExpr() + { + return whereExpr; + } + public void setWhereExpr(CommonTree whereExpr) + { + this.whereExpr = (ASTNode) whereExpr; + } + public Map getWindowSpecs() + { + windowSpecs = windowSpecs == null ? new HashMap() : windowSpecs; + return windowSpecs; + } + public void setWindowSpecs(Map windowSpecs) + { + this.windowSpecs = windowSpecs; + } + + public void addWindowSpec(String name, WindowSpec wdwSpec) + { + windowSpecs = windowSpecs == null ? new HashMap() : windowSpecs; + windowSpecs.put(name, wdwSpec); + } + + public WindowSpec getWindowSpec(String name) + { + return windowSpecs == null ? null : windowSpecs.get(name); + } + + public SelectSpec getSelectList() + { + return selectList; + } + public void setSelectList(SelectSpec selectList) + { + this.selectList = selectList; + } + + + public static String NL = System.getProperty("line.separator"); + @Override + public String toString() + { + StringBuilder buf = new StringBuilder(); + + buf.append(selectList).append(NL); + buf.append("from ").append(input).append(NL); + if ( whereExpr != null ) + { + buf.append("where ").append(whereExpr.toStringTree()).append(NL); + } + + if ( windowSpecs != null) + { + boolean first = true; + buf.append("window "); + for(Map.Entry wentry : windowSpecs.entrySet()) + { + if ( first ) { + first = false; + } else { + buf.append(","); + } + buf.append(NL).append(" ").append(wentry.getKey()).append(" as ").append(wentry.getValue()); + } + buf.append(NL); + } + + return buf.toString(); + } + + /* + * QueryInputSpec class + */ + public static abstract class PTFInputSpec + { + ASTNode astNode; + PartitionSpec partition; + OrderSpec order; + + public PartitionSpec getPartition() + { + return partition; + } + public void setPartition(PartitionSpec partition) + { + this.partition = partition; + } + public OrderSpec getOrder() + { + return order; + } + public void setOrder(OrderSpec order) + { + this.order = order; + } + + public ASTNode getAstNode() { + return astNode; + } + public void setAstNode(ASTNode astNode) { + this.astNode = astNode; + } + /* + * is the starting point of this Input Chain a Hive Query or Table. + */ + public abstract boolean sourcedFromHive(); + } + + public static enum Order + { + ASC, + DESC; + } + + + /* + * HiveTableSpec class + */ + public static class PTFTableOrSubQueryInputSpec extends PTFInputSpec + { + String dbName; + String tableName; + + public PTFTableOrSubQueryInputSpec() {} + + public PTFTableOrSubQueryInputSpec(String dbName, String tableName) + { + super(); + this.dbName = dbName; + this.tableName = tableName; + } + + public String getDbName() + { + return dbName; + } + + public void setDbName(String dbName) + { + this.dbName = dbName; + } + + public String getTableName() + { + return tableName; + } + + public void setTableName(String tableName) + { + this.tableName = tableName; + } + + @Override + public boolean sourcedFromHive() + { + return true; + } + + @Override + public String toString() + { + StringBuilder buf = new StringBuilder(); + + if ( dbName != null) { + buf.append(dbName).append("."); + } + buf.append(tableName); + + if ( partition != null ) + { + buf.append(" ").append(partition); + if ( order != null ) { + buf.append(" ").append(order); + } + } + + return buf.toString(); + } + + } + + + /** + * To support multi-PTF operator chain + * PTFComponentQuerySpec. + * + */ + public static class PTFComponentQuerySpec extends PTFInputSpec + { + @Override + public boolean sourcedFromHive() { + return true; + } + + @Override + public String toString() + { + return "PTFOp up the chain..."; + } + } + + /* + * TableFunc Spec + */ + public static class TableFuncSpec extends PTFInputSpec + { + String name; + String alias; + ArrayList args; + PTFInputSpec input; + + /* + * This is only set for WindowingTableFunction. + * We allow for the Partitioning Spec to be inferred from a WindowingFunction or Window Clause as a convenience. + * This is so that the language is compliant with standard SQL. But at runtime we only allow all functions to have the + * same Partitioning Spec. So as a further convenience functions w/o a Partitioning Spec can inherit this from the default. + * The rules for setting the details are the following: + * 1. On encountering the first UDAF with a WindowingSpec we set the WindowingTablFunc's default Window Spec to this. + * We also set the Table Func's Partition & Order spec based on the WindowSpec. + * 2. On encountering a Distribute/Cluster clause we set the WindowingTablFunc's Partition spec. We clear the default Window Spec. + * There is no default in this case; the Partitioning spec is being specified explicitly. + * 3. On encountering a Sort clause: + * a. we set the WindowTableFunc's OrderSpec. + * b. If the default WindowSpec is not null & it has no OrderSpec, we set its OrderSpec. + * 4. On encountering the first Window Clause, if the WindowingTablFunc has no PartitionSpec we set it from the Window Clause. + * If the TblFunc has no OrderSpec, we set it from the Window Clause. + * If the TblFunc has a OrderSpec and the Window Clause doesn't then we set the OrderSpec on the Window Clause. + */ + WindowSpec defaultWindowSpec; + + public TableFuncSpec() {} + public TableFuncSpec(String name) + { + super(); + this.name = name; + } + + public String getName() + { + return name; + } + + public void setName(String name) + { + this.name = name; + } + + public String getAlias() { + return alias; + } + + public void setAlias(String alias) { + this.alias = alias; + } + + public ArrayList getArgs() + { + return args; + } + + public void setArgs(ArrayList args) + { + this.args = args; + } + + public void addArg(CommonTree arg) + { + args = args == null ? new ArrayList() : args; + args.add((ASTNode)arg); + } + + public PTFInputSpec getInput() + { + return input; + } + public void setInput(PTFInputSpec input) + { + this.input = input; + } + + protected WindowSpec getDefaultWindowSpec() { + return defaultWindowSpec; + } + protected void setDefaultWindowSpec(WindowSpec defaultWindowSpec) { + this.defaultWindowSpec = defaultWindowSpec; + } + + public void inferDefaultWindowingSpec(WindowFunctionSpec wFnSpec) { + if ( !getName().equals(FunctionRegistry.WINDOWING_TABLE_FUNCTION)) { + return; + } + + WindowSpec wSpec = wFnSpec == null ? null : wFnSpec.getWindowSpec(); + if ( getDefaultWindowSpec() == null && wSpec != null && wSpec.getPartition() != null ) { + setDefaultWindowSpec(wSpec); + setPartition(wSpec.getPartition()); + setOrder(wSpec.getOrder()); + } + } + + public void inferDefaultWindowingSpec(WindowSpec wSpec) { + if ( !getName().equals(FunctionRegistry.WINDOWING_TABLE_FUNCTION)) { + return; + } + + if ( wSpec == null || wSpec.getPartition() == null || getPartition() != null ) { + return; + } + + setDefaultWindowSpec(wSpec); + setPartition(wSpec.getPartition()); + + if ( getOrder() == null ) { + setOrder(wSpec.getOrder()); + } + else { + if ( wSpec.getOrder() == null ) { + wSpec.setOrder(getOrder()); + } + } + + } + + public void inferDefaultOrderSpec(OrderSpec oSpec) { + if ( !getName().equals(FunctionRegistry.WINDOWING_TABLE_FUNCTION)) { + return; + } + setOrder(oSpec); + if ( getDefaultWindowSpec() != null ) { + WindowSpec dws = getDefaultWindowSpec(); + if ( dws.getOrder() == null ) { + dws.setOrder(oSpec); + } + } + } + + @Override + public boolean sourcedFromHive() + { + return input.sourcedFromHive(); + } + + @Override + public String toString() + { + StringBuilder buf = new StringBuilder(); + + buf.append(name).append("("); + buf.append(PTFSpec.NL).append(" ").append(input); + if ( args != null ) + { + buf.append(PTFSpec.NL).append(" "); + boolean first = true; + for(CommonTree arg : args) + { + if ( first) { + first = false; + } else { + buf.append(", "); + } + buf.append(arg.toStringTree()); + } + } + buf.append(PTFSpec.NL).append(" )"); + if ( partition != null ) + { + buf.append(" ").append(partition); + if ( order != null ) { + buf.append(" ").append(order); + } + } + + return buf.toString(); + } + } + + /* + * Window Function Spec + */ + public static class WindowFunctionSpec + { + String name; + boolean isStar; + boolean isDistinct; + ArrayList args; + WindowSpec windowSpec; + String alias; + ASTNode expression; + + + public String getName() + { + return name; + } + + public void setName(String name) + { + this.name = name; + } + + public boolean isStar() + { + return isStar; + } + + public void setStar(boolean isStar) + { + this.isStar = isStar; + } + + public boolean isDistinct() + { + return isDistinct; + } + + public void setDistinct(boolean isDistinct) + { + this.isDistinct = isDistinct; + } + + public ArrayList getArgs() + { + args = args == null ? new ArrayList() : args; + return args; + } + + public void setArgs(ArrayList args) + { + this.args = args; + } + + public void addArg(CommonTree arg) + { + args = args == null ? new ArrayList() : args; + args.add((ASTNode)arg); + } + + public WindowSpec getWindowSpec() + { + return windowSpec; + } + + public void setWindowSpec(WindowSpec windowSpec) + { + this.windowSpec = windowSpec; + } + + public String getAlias() + { + return alias; + } + + public void setAlias(String alias) + { + this.alias = alias; + } + + public ASTNode getExpression() + { + return expression; + } + + public void setExpression(ASTNode expression) + { + this.expression = expression; + } + + @Override + public String toString() + { + StringBuilder buf = new StringBuilder(); + buf.append(name).append("("); + if (isStar ) + { + buf.append("*"); + } + else + { + if ( isDistinct ) + { + buf.append("distinct "); + } + if ( args != null ) + { + boolean first = true; + for(CommonTree arg : args) + { + if ( first) { + first = false; + } else { + buf.append(", "); + } + buf.append(arg.toStringTree()); + } + } + } + + buf.append(")"); + + if ( windowSpec != null ) + { + buf.append(" ").append(windowSpec.toString()); + } + + if ( alias != null ) + { + buf.append(" as ").append(alias); + } + + return buf.toString(); + } + + } + + /* + * PartitionSpec class + */ + public static class PartitionSpec + { + ArrayList columns; + + public ArrayList getColumns() + { + return columns; + } + + public void setColumns(ArrayList columns) + { + this.columns = columns; + } + + public void addColumn(ColumnSpec c) + { + columns = columns == null ? new ArrayList() : columns; + columns.add(c); + } + + @Override + public int hashCode() + { + final int prime = 31; + int result = 1; + result = prime * result + ((columns == null) ? 0 : columns.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + PartitionSpec other = (PartitionSpec) obj; + if (columns == null) + { + if (other.columns != null) { + return false; + } + } + else if (!columns.equals(other.columns)) { + return false; + } + return true; + } + + @Override + public String toString() + { + return PTFUtils.sprintf("partitionColumns=%s",PTFUtils.toString(columns)); + } + } + + /* + * OrderSpec + */ + public static class OrderSpec + { + ArrayList columns; + + public OrderSpec() {} + + public OrderSpec(PartitionSpec pSpec) + { + for(ColumnSpec cSpec : pSpec.getColumns()) + { + addColumn(new OrderColumnSpec(cSpec)); + } + } + + public ArrayList getColumns() + { + return columns; + } + + public void setColumns(ArrayList columns) + { + this.columns = columns; + } + + public void addColumn(OrderColumnSpec c) + { + columns = columns == null ? new ArrayList() : columns; + columns.add(c); + } + + @Override + public int hashCode() + { + final int prime = 31; + int result = 1; + result = prime * result + ((columns == null) ? 0 : columns.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + OrderSpec other = (OrderSpec) obj; + if (columns == null) + { + if (other.columns != null) { + return false; + } + } + else if (!columns.equals(other.columns)) { + return false; + } + return true; + } + + @Override + public String toString() + { + return PTFUtils.sprintf("orderColumns=%s",PTFUtils.toString(columns)); + } + } + + /* + * ColumnSpec + */ + public static class ColumnSpec + { + String tableName; + String columnName; + ASTNode expression; + + public ColumnSpec() {} + + public ColumnSpec(String tableName, String columnName) + { + super(); + setTableName(tableName); + setColumnName(columnName); + } + + public ColumnSpec(ColumnSpec cSpec) + { + this(cSpec.getTableName(), cSpec.getColumnName()); + expression = cSpec.getExpression(); + } + + public String getTableName() + { + return tableName; + } + + public void setTableName(String tableName) + { + this.tableName = tableName; + } + + public String getColumnName() + { + return columnName; + } + + public void setColumnName(String columnName) + { + this.columnName = columnName; + } + + public ASTNode getExpression() { + return expression; + } + + public void setExpression(ASTNode expression) { + this.expression = expression; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((columnName == null) ? 0 : columnName.hashCode()); + result = prime * result + ((expression == null) ? 0 : expression.hashCode()); + result = prime * result + ((tableName == null) ? 0 : tableName.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + ColumnSpec other = (ColumnSpec) obj; + if (columnName == null) { + if (other.columnName != null) { + return false; + } + } else if (!columnName.equals(other.columnName)) { + return false; + } + if (expression == null) { + if (other.expression != null) { + return false; + } + } else if (!expression.toStringTree().equals(other.expression.toStringTree())) { + return false; + } + if (tableName == null) { + if (other.tableName != null) { + return false; + } + } else if (!tableName.equals(other.tableName)) { + return false; + } + return true; + } + + @Override + public String toString() + { + /* if ( tableName != null ) { + return PTFUtils.sprintf("%s.%s", tableName, columnName); + } + return columnName; + */ + return expression.toStringTree(); + } + + } + + /* + * OrderColumnSpec class + */ + public static class OrderColumnSpec extends ColumnSpec + { + Order order; + + public OrderColumnSpec() {} + + public OrderColumnSpec(ColumnSpec cSpec) + { + super(cSpec); + order = Order.ASC; + } + + public OrderColumnSpec(String tableName, String columnName, Order order) + { + super(tableName, columnName); + this.order = order; + } + + public Order getOrder() + { + return order; + } + + public void setOrder(Order order) + { + this.order = order; + } + + @Override + public int hashCode() + { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + ((order == null) ? 0 : order.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (!super.equals(obj)) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + OrderColumnSpec other = (OrderColumnSpec) obj; + if (order != other.order) { + return false; + } + return true; + } + + @Override + public String toString() + { + return PTFUtils.sprintf("%s %s", super.toString(), order); + } + } + + /* + * Window Spec class + */ + public static class WindowSpec + { + String sourceId; + PartitionSpec partition; + OrderSpec order; + WindowFrameSpec window; + + public WindowSpec() {} + + public WindowSpec(String sourceId, PartitionSpec partition, + OrderSpec order, WindowFrameSpec window) + { + super(); + this.sourceId = sourceId; + this.partition = partition; + this.order = order; + this.window = window; + } + + public String getSourceId() + { + return sourceId; + } + + public void setSourceId(String sourceId) + { + this.sourceId = sourceId; + } + + public PartitionSpec getPartition() + { + return partition; + } + + public void setPartition(PartitionSpec partition) + { + this.partition = partition; + } + + public OrderSpec getOrder() + { + return order; + } + + public void setOrder(OrderSpec order) + { + this.order = order; + } + + public WindowFrameSpec getWindow() + { + return window; + } + + public void setWindow(WindowFrameSpec window) + { + this.window = window; + } + + @Override + public String toString() + { + StringBuilder buf = new StringBuilder(); + if (sourceId != null) { + buf.append(PTFUtils.sprintf("%s ", sourceId)); + } + if (partition != null) { + buf.append(PTFUtils.sprintf("%s ", partition)); + } + if (order != null) { + buf.append(PTFUtils.sprintf("%s ", order)); + } + if (window != null) { + buf.append(PTFUtils.sprintf("%s ", window)); + } + return buf.toString(); + } + } + + /* + * Window Frame classes + */ + public static class WindowFrameSpec + { + BoundarySpec start; + BoundarySpec end; + + public WindowFrameSpec() {} + + public WindowFrameSpec(BoundarySpec start, BoundarySpec end) + { + super(); + this.start = start; + this.end = end; + } + + public BoundarySpec getStart() + { + return start; + } + + public void setStart(BoundarySpec start) + { + this.start = start; + } + + public BoundarySpec getEnd() + { + return end; + } + + public void setEnd(BoundarySpec end) + { + this.end = end; + } + + @Override + public String toString() + { + return PTFUtils.sprintf( "window(start=%s, end=%s)", start, end); + } + + public static enum Direction + { + PRECEDING, + CURRENT, + FOLLOWING + }; + + public abstract static class BoundarySpec implements Comparable + { + public static int UNBOUNDED_AMOUNT = Integer.MAX_VALUE; + + public abstract Direction getDirection(); + + } + + public static class RangeBoundarySpec extends BoundarySpec + { + + Direction direction; + int amt; + + public RangeBoundarySpec() {} + + public RangeBoundarySpec(Direction direction, int amt) + { + super(); + this.direction = direction; + this.amt = amt; + } + + @Override + public Direction getDirection() + { + return direction; + } + + public void setDirection(Direction direction) + { + this.direction = direction; + } + + public int getAmt() + { + return amt; + } + + public void setAmt(int amt) + { + this.amt = amt; + } + + @Override + public String toString() + { + return PTFUtils.sprintf( "range(%s %s)", (amt == UNBOUNDED_AMOUNT ? "Unbounded" : amt), direction); + } + + public int compareTo(BoundarySpec other) + { + int c = direction.compareTo(other.getDirection()); + if ( c != 0) { + return c; + } + RangeBoundarySpec rb = (RangeBoundarySpec) other; + return amt - rb.amt; + } + + } + + public static class CurrentRowSpec extends BoundarySpec + { + public CurrentRowSpec() {} + + @Override + public String toString() + { + return PTFUtils.sprintf( "currentRow"); + } + + @Override + public Direction getDirection() { return Direction.CURRENT; } + + public int compareTo(BoundarySpec other) + { + return getDirection().compareTo(other.getDirection()); + } + + } + + public static class ValueBoundarySpec extends BoundarySpec + { + Direction direction; + ASTNode expression; + int amt; + + public ValueBoundarySpec() {} + + public ValueBoundarySpec(Direction direction, CommonTree expression, + int amt) + { + super(); + this.direction = direction; + this.expression = (ASTNode) expression; + this.amt = amt; + } + + @Override + public Direction getDirection() + { + return direction; + } + + public void setDirection(Direction direction) + { + this.direction = direction; + } + + public ASTNode getExpression() + { + return expression; + } + + public void setExpression(CommonTree expression) + { + this.expression = (ASTNode) expression; + } + + public void setExpression(ASTNode expression) + { + this.expression = expression; + } + + public int getAmt() + { + return amt; + } + + public void setAmt(int amt) + { + this.amt = amt; + } + + @Override + public String toString() + { + return PTFUtils.sprintf( "value(%s %s %s)", expression.toStringTree(), amt, direction); + } + + public int compareTo(BoundarySpec other) + { + int c = direction.compareTo(other.getDirection()); + if ( c != 0) { + return c; + } + ValueBoundarySpec vb = (ValueBoundarySpec) other; + return amt - vb.amt; + } + + } + + } + + /* + * Select Spec + */ + public static class SelectSpec implements Iterable + { + ArrayList expressions; + ArrayList windowFuncs; + ArrayList isWindowFn; + ArrayList aliases; + LinkedHashMap aliasToAST; + + public ArrayList getExpressions() + { + return expressions; + } + + public void addExpression(CommonTree expr, String alias) + { + isWindowFn = isWindowFn == null ? new ArrayList() : isWindowFn; + expressions = expressions == null ? new ArrayList() : expressions; + aliases = aliases == null ? new ArrayList() : aliases; + isWindowFn.add(false); + expressions.add((ASTNode) expr); + aliases.add(alias); + aliasToAST = aliasToAST == null ? new LinkedHashMap() : aliasToAST; + /* + * maintain alias mapping in lowercase. why? + * - colAlias mainatined as lowercase in RR + * - when we build an OI, fieldnames are converted to lowercase. + * So we get a match when looking for adding col expression mapping in the Select RR. + */ + aliasToAST.put(alias.toLowerCase(), (ASTNode) expr); + } + + public ArrayList getWindowFuncs() + { + return windowFuncs; + } + + public void addWindowFunc(WindowFunctionSpec wFn, String alias) + { + isWindowFn = isWindowFn == null ? new ArrayList() : isWindowFn; + windowFuncs = windowFuncs == null ? new ArrayList() : windowFuncs; + aliases = aliases == null ? new ArrayList() : aliases; + isWindowFn.add(true); + windowFuncs.add(wFn); + wFn.setAlias(alias); + aliases.add(alias); + aliasToAST = aliasToAST == null ? new LinkedHashMap() : aliasToAST; + /* + * maintain alias mapping in lowercase. why? + * - colAlias mainatined as lowercase in RR + * - when we build an OI, fieldnames are converted to lowercase. + * So we get a match when looking for adding col expression mapping in the Select RR. + */ + aliasToAST.put(alias.toLowerCase(), (ASTNode) wFn.getExpression()); + } + + public ArrayList getIsWindowFn() + { + return isWindowFn; + } + + public class It implements Iterator + { + Iterator exprIt; + Iterator wnFnIt; + Iterator isWnfnIt; + + It() + { + if (SelectSpec.this.isWindowFn != null) + { + isWnfnIt = SelectSpec.this.isWindowFn.iterator(); + if ( SelectSpec.this.windowFuncs != null ) { + wnFnIt = SelectSpec.this.windowFuncs.iterator(); + } + if ( SelectSpec.this.expressions != null ) { + exprIt = SelectSpec.this.expressions.iterator(); + } + } + } + + @Override + public boolean hasNext() + { + return isWnfnIt != null && isWnfnIt.hasNext(); + } + + @Override + public Object next() + { + if ( hasNext() ) + { + boolean isWFn = isWnfnIt.next(); + if ( isWFn ) { + return wnFnIt.next(); + } else { + return exprIt.next(); + } + } + return null; + } + + @Override + public void remove() + { + throw new UnsupportedOperationException(); + } + } + + public Iterator iterator() + { + return new It(); + } + + @Override + public String toString() + { + StringBuilder buf = new StringBuilder(); + buf.append("select "); + boolean first = true; + Iterator aIt = aliases.iterator(); + for(Object o : this) + { + if ( first ) { + first = false; + } else { + buf.append(","); + } + buf.append(PTFSpec.NL).append(" "); + if ( o instanceof CommonTree) + { + buf.append(((CommonTree)o).toStringTree()); + } + else + { + buf.append(o.toString()); + } + String alias = aIt.next(); + if (alias != null && (o instanceof CommonTree) ) + { + buf.append(" as ").append(alias); + } + } + return buf.toString(); + } + + public void setExpressions(ArrayList expressions) + { + this.expressions = expressions; + } + + public void setWindowFuncs(ArrayList windowFuncs) + { + this.windowFuncs = windowFuncs; + } + + public void setIsWindowFn(ArrayList isWindowFn) + { + this.isWindowFn = isWindowFn; + } + + public void setAliases(ArrayList aliases) + { + this.aliases = aliases; + } + + public ArrayList getAliases() + { + return aliases; + } + + public Iterator getColumnListAndAlias() + { + return new ColumnListAndAliasItr(); + } + + public LinkedHashMap getAliasToAST() { + return aliasToAST; + } + + public void setAliasToAST(LinkedHashMap aliasToAST) { + this.aliasToAST = aliasToAST; + } + + class ColumnListAndAliasItr implements Iterator + { + int cnt; + int exprIdx; + int wdwFnIdx; + int idx; + + ColumnListAndAliasItr() + { + cnt = SelectSpec.this.isWindowFn.size(); + idx = 0; + exprIdx = 0; + wdwFnIdx = 0; + } + + @Override + public boolean hasNext() + { + return idx < cnt; + } + + @Override + public Object next() + { + boolean isWnFn = SelectSpec.this.isWindowFn.get(idx); + Object alias = SelectSpec.this.aliases.get(idx); + idx++; + + if ( isWnFn ) + { + return new Object[] { isWnFn, alias, SelectSpec.this.windowFuncs.get(wdwFnIdx++)}; + } + else + { + return new Object[] { isWnFn, alias, SelectSpec.this.expressions.get(exprIdx++)}; + } + } + + @Override + public void remove() + { + throw new UnsupportedOperationException(); + } + + } + + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java new file mode 100644 index 0000000..bb28b4c --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java @@ -0,0 +1,2722 @@ +package org.apache.hadoop.hive.ql.parse; + +import static org.apache.hadoop.hive.ql.exec.PTFUtils.sprintf; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.antlr.runtime.Token; +import org.antlr.runtime.tree.CommonTreeAdaptor; +import org.antlr.runtime.tree.Tree; +import org.antlr.runtime.tree.TreeAdaptor; +import org.antlr.runtime.tree.TreeWizard; +import org.antlr.runtime.tree.TreeWizard.ContextVisitor; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.ErrorMsg; +import org.apache.hadoop.hive.ql.exec.ColumnInfo; +import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; +import org.apache.hadoop.hive.ql.exec.FunctionRegistry; +import org.apache.hadoop.hive.ql.exec.PTFUtils.Predicate; +import org.apache.hadoop.hive.ql.exec.PTFUtils.ReverseIterator; +import org.apache.hadoop.hive.ql.exec.WindowFunctionInfo; +import org.apache.hadoop.hive.ql.lib.Node; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.parse.PTFSpec.ColumnSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.OrderColumnSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.OrderSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.PTFComponentQuerySpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.PTFInputSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.PTFTableOrSubQueryInputSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.PartitionSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.SelectSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.TableFuncSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowFrameSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowFrameSpec.BoundarySpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowFrameSpec.CurrentRowSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowFrameSpec.RangeBoundarySpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowFrameSpec.ValueBoundarySpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowFunctionSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowSpec; +import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; +import org.apache.hadoop.hive.ql.plan.PTFDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.ArgDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.ColumnDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.OrderColumnDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.OrderDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.PTFComponentQueryDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.PTFInputDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.PTFTableOrSubQueryInputDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.PartitionDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.SelectDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.TableFuncDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.WhereDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.WindowDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.WindowFrameDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.WindowFrameDef.BoundaryDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.WindowFrameDef.CurrentRowDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.WindowFrameDef.RangeBoundaryDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.WindowFrameDef.ValueBoundaryDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.WindowFunctionDef; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFLeadLag; +import org.apache.hadoop.hive.ql.udf.ptf.TableFunctionEvaluator; +import org.apache.hadoop.hive.ql.udf.ptf.TableFunctionResolver; +import org.apache.hadoop.hive.serde2.SerDe; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.SerDeUtils; +import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; + +public class PTFTranslator +{ + public PTFDef translate(PTFSpec qSpec, HiveConf qCfg, RowResolver rr, UnparseTranslator unparseT) throws SemanticException + { + PTFDef qry = new PTFDef(); + qry.setSpec(qSpec); + + PTFTranslationInfo transInfo = new PTFTranslationInfo(); + transInfo.setHiveCfg(qCfg); + transInfo.unparseT = unparseT; + qry.setTranslationInfo(transInfo); + + PTFTranslator.translateInput(qry, rr); + PTFTranslator.translateWhere(qry); + PTFTranslator.translateOutput(qry); + + return qry; + } + + /* + * Input translation methods + */ + private static void translateInput(PTFDef qDef, RowResolver rr) throws SemanticException + { + PTFSpec spec = qDef.getSpec(); + + /* + * validate that input chain ends in a Hive Query or TAble. + */ + if (!spec.getInput().sourcedFromHive()) + { + throw new SemanticException("Translation not supported for HdfsLocation based queries"); + } + + PTFTranslationInfo tInfo = qDef.getTranslationInfo(); + PTFSpec qSpec = qDef.getSpec(); + Iterator it = PTFTranslator.iterateInputSpecs(qSpec, true); + PTFInputDef currentIDef = null; + int inputNum = 0; + while (it.hasNext()) + { + PTFInputSpec nextSpec = it.next(); + if (nextSpec instanceof PTFTableOrSubQueryInputSpec) + { + currentIDef = PTFTranslator.translate(qDef, (PTFTableOrSubQueryInputSpec) nextSpec, + (PTFTableOrSubQueryInputDef) null, rr); + } + else if (nextSpec instanceof PTFComponentQuerySpec) + { + currentIDef = PTFTranslator.translate(qDef, (PTFComponentQuerySpec) nextSpec, + (PTFComponentQueryDef) null, rr); + } + else + { + currentIDef = translate(qDef, (TableFuncSpec) nextSpec, currentIDef); + } + String alias = getTableAlias(qDef, inputNum, currentIDef); + currentIDef.setAlias(alias); + tInfo.addInput(currentIDef, inputNum == 0 ? rr : null); + inputNum++; + } + + qDef.setInput(currentIDef); + } + + + /* + * todo: revisit computing the alias. For a HiveTableDef it probably only needs to be the + * tablename. + */ + private static String getTableAlias(PTFDef qDef, int inputNum, PTFInputDef inputDef) + throws SemanticException + { + if (inputDef instanceof PTFTableOrSubQueryInputDef) + { + PTFTableOrSubQueryInputDef hTbldef = (PTFTableOrSubQueryInputDef) inputDef; + String db = ((PTFTableOrSubQueryInputSpec) hTbldef.getSpec()).getDbName(); + String tableName = ((PTFTableOrSubQueryInputSpec) hTbldef.getSpec()).getTableName(); + return db + "." + tableName; + } + else if (inputDef instanceof PTFComponentQueryDef) + { + String alias = "ptf_" + inputNum ; + return alias; + } + else if (inputDef instanceof TableFuncDef) + { + String alias = ((TableFuncDef) inputDef).getTableFuncSpec().getAlias(); + alias = alias == null ? "ptf_" + inputNum : alias; + return alias; + } + throw new SemanticException(sprintf("Internal Error: attempt to translate %s", + inputDef.getSpec())); + } + + private static PTFComponentQueryDef translate(PTFDef qDef, + PTFComponentQuerySpec spec, + PTFComponentQueryDef def, RowResolver rr) throws SemanticException + { + def = def == null ? new PTFComponentQueryDef() : def; + def.setSpec(spec); + ObjectInspector oi = PTFTranslator.getInputOI(rr); + Map serdePropsMap = new LinkedHashMap(); + SerDe serde = null; + + try { + serde = PTFTranslator.createLazyBinarySerDe( + qDef.getTranslationInfo().getHiveCfg(), + (StructObjectInspector) oi, serdePropsMap); + } + catch(SerDeException se) { + throw new SemanticException(se); + } + def.setSerde(serde); + def.setCompSerdeClassName(serde.getClass().getName()); + def.setOI((StructObjectInspector) oi); + return def; + + } + + + private static PTFTableOrSubQueryInputDef translate(PTFDef qDef, + PTFTableOrSubQueryInputSpec spec, + PTFTableOrSubQueryInputDef def, RowResolver rr) throws SemanticException + { + def = def == null ? new PTFTableOrSubQueryInputDef() : def; + def.setSpec(spec); + ObjectInspector oi = PTFTranslator.getInputOI(rr); + Map serdePropsMap = new LinkedHashMap(); + SerDe serde = null; + + try { + serde = PTFTranslator.createLazyBinarySerDe( + qDef.getTranslationInfo().getHiveCfg(), + (StructObjectInspector) oi, serdePropsMap); + } + catch(SerDeException se) { + throw new SemanticException(se); + } + def.setSerde(serde); + def.setTableSerdeProps(serdePropsMap); + def.setTableSerdeClassName(serde.getClass().getName()); + def.setOI((StructObjectInspector) oi); + return def; + + } + + /* + *
    + *
  1. Get the TableFunctionResolver for this Function from the FunctionRegistry. + *
  2. Create the TableFuncDef object. + *
  3. Get the InputInfo for the input to this function. + *
  4. Translate the Arguments to this Function in the Context of the InputInfo. + *
  5. ask the TableFunctionResolver to create a TableFunctionEvaluator based on the Args passed + * in. + *
  6. ask the TableFunctionEvaluator to setup the Map-side ObjectInspector. Gives a chance to + * functions that + * reshape the Input before it is partitioned to define the Shape after raw data is transformed. + *
  7. Setup the Window Definition for this Function. The Window Definition is resolved wrt to + * the InputDef's + * Shape or the MapOI, for Functions that reshape the raw input. + *
  8. ask the TableFunctionEvaluator to setup the Output ObjectInspector for this Function. + *
  9. setup a Serde for the Output partition based on the OutputOI. + *
+ */ + private static TableFuncDef translate(PTFDef qDef, TableFuncSpec tSpec, PTFInputDef inputDef) + throws SemanticException + { + PTFTranslationInfo tInfo = qDef.getTranslationInfo(); + + TableFunctionResolver tFn = FunctionRegistry.getTableFunctionResolver(tSpec.getName()); + if (tFn == null) + { + throw new SemanticException(sprintf("Unknown Table Function %s", tSpec.getName())); + } + + TableFuncDef tDef = new TableFuncDef(); + tDef.setSpec(tSpec); + tDef.setInput(inputDef); + PTFInputInfo iInfo = tInfo.getInputInfo(inputDef); + + /* + * translate args + */ + ArrayList args = tSpec.getArgs(); + if (args != null) + { + for (ASTNode expr : args) + { + ArgDef argDef = null; + try { + argDef = translateTableFunctionArg(qDef, tDef, iInfo, expr); + } + catch(HiveException he) { + throw new SemanticException(he); + } + tDef.addArg(argDef); + } + } + + tFn.initialize(qDef, tDef); + TableFunctionEvaluator tEval = tFn.getEvaluator(); + tDef.setFunction(tEval); + tDef.setCarryForwardNames(tFn.carryForwardNames()); + tFn.setupRawInputOI(); + tDef.setRawInputColumnNames(tFn.getRawInputColumnNames()); + tDef.setWindow(PTFTranslator.translateWindow(qDef, tDef)); + tFn.setupOutputOI(); + tDef.setOutputColumnNames(tFn.getOutputColumnNames()); + try { + PTFTranslator.setupSerdeAndOI(tDef, inputDef, tInfo, tEval, true); + } + catch(SerDeException se) { + throw new SemanticException(se); + } + + return tDef; + } + + private static ArgDef translateTableFunctionArg(PTFDef qDef, TableFuncDef tDef, + PTFInputInfo iInfo, ASTNode arg) throws HiveException + { + return PTFTranslator.buildArgDef(qDef, iInfo, arg); + } + + + /* + * Where Clause translation. + */ + + public static void translateWhere(PTFDef qDef) throws SemanticException + { + PTFTranslationInfo tInfo = qDef.getTranslationInfo(); + PTFSpec spec = qDef.getSpec(); + + ASTNode wExpr = (ASTNode) spec.getWhereExpr(); + + if (wExpr == null) { + return; + } + + wExpr = (ASTNode) wExpr.getChild(0); + + WhereDef whDef = new WhereDef(); + whDef.setExpression(wExpr); + + PTFInputDef iDef = qDef.getInput(); + PTFInputInfo iInfo = tInfo.getInputInfo(iDef); + + ExprNodeDesc exprNode = PTFTranslator.buildExprNode(wExpr, iInfo.getTypeCheckCtx()); + ExprNodeEvaluator exprEval = null; + ObjectInspector oi = null; + try { + exprEval = WindowingExprNodeEvaluatorFactory.get(tInfo, exprNode); + oi = PTFTranslator.initExprNodeEvaluator(qDef, exprNode, exprEval, iInfo); + } + catch(HiveException he) { + throw new SemanticException(he); + } + try + { + ObjectInspectorConverters.getConverter(oi, + PrimitiveObjectInspectorFactory.javaBooleanObjectInspector); + } catch (Throwable t) + { + throw new SemanticException("Where Expr must be convertible to a boolean value", t); + } + + whDef.setExprNode(exprNode); + whDef.setExprEvaluator(exprEval); + whDef.setOI(oi); + + qDef.setWhere(whDef); + } + + /* + * Output Translation methods. + */ + + private static void translateOutput(PTFDef qDef) throws SemanticException + { + translateSelectExprs(qDef); + setupSelectRRAndOI(qDef); + } + + private static void translateSelectExprs(PTFDef qDef) throws SemanticException + { + PTFTranslationInfo tInfo = qDef.getTranslationInfo(); + PTFInputDef iDef = qDef.getInput(); + PTFInputInfo iInfo = tInfo.getInputInfo(iDef); + SelectSpec selectSpec = qDef.getSpec().getSelectList(); + + if (selectSpec == null) { + return; + } + + SelectDef selectDef = qDef.getSelectList(); + Iterator selectExprsAndAliases = selectSpec.getColumnListAndAlias(); + int i = 0; + ColumnDef cDef = null; + + + while (selectExprsAndAliases.hasNext()) + { + Object[] o = (Object[]) selectExprsAndAliases.next(); + boolean isWnFn = ((Boolean) o[0]).booleanValue(); + + if (!isWnFn) + { + cDef = translateSelectExpr(qDef, iInfo, i++, (String) o[1], (ASTNode) o[2]); + selectDef.addColumn(cDef); + } + } + } + + + private static ColumnDef translateSelectExpr(PTFDef qDef, PTFInputInfo iInfo, int colIdx, + String alias, ASTNode expr) + throws SemanticException + { + ColumnDef cDef = new ColumnDef((ColumnSpec) null); + ExprNodeDesc exprNode = PTFTranslator.buildExprNode(expr, iInfo.getTypeCheckCtx()); + ExprNodeEvaluator exprEval = null; + ObjectInspector oi = null; + try { + exprEval = WindowingExprNodeEvaluatorFactory.get(qDef.getTranslationInfo(), exprNode); + oi = PTFTranslator.initExprNodeEvaluator(qDef, exprNode, exprEval, iInfo); + } + catch(HiveException he) { + throw new SemanticException(he); + } + + cDef.setExpression(expr); + cDef.setExprNode(exprNode); + cDef.setExprEvaluator(exprEval); + cDef.setOI(oi); + + cDef.setAlias(getAlias(alias, expr, colIdx)); + + return cDef; + } + + private static String getAlias(String alias, ASTNode expr, int columnIdx) + { + if (alias != null) { + return alias; + } + + if (expr.getToken().getType() == HiveParser.TOK_TABLE_OR_COL) + { + Tree child = expr.getChild(0); + return child.getText(); + } + + return sprintf("col_%i", columnIdx); + } + + /* + * Setup based on the OI of the final PTF + expressions in the SelectList that are handled by the + * Partition mechanism. + * For expressions that are handled by the PTFOp : navigation expressions in SelectList or + * windowing clauses + * add the mapping from ASTNode to ColumnInfo so that the SelectOp doesn't try to evaluate these. + */ + static void setupSelectRRAndOI(PTFDef qDef) throws SemanticException { + PTFTranslationInfo tInfo = qDef.getTranslationInfo(); + PTFInputDef iDef = qDef.getInput(); + PTFInputInfo iInfo = tInfo.getInputInfo(iDef); + SelectDef selectDef = qDef.getSelectList(); + RowResolver inputRR = iInfo.getRowResolver(); + SelectSpec selectSpec = qDef.getSpec().getSelectList(); + LinkedHashMap aliasToAST = selectSpec == null ? null : selectSpec + .getAliasToAST(); + boolean isWindowPTF = ((TableFuncDef) qDef.getInput()).getName() == FunctionRegistry.WINDOWING_TABLE_FUNCTION; + RowResolver rr = new RowResolver(); + + // should I just set it to the iDef.getAlias()? + // if the ptf invocation didn't have an alias, the alias would be set to an internally generated + // alias. + String outAlias = ((TableFuncDef) qDef.getInput()).getTableFuncSpec().getAlias(); + + /* + * Give the Columns internalNames based on position. + * The Columns are added in the following order: + * - the columns in the SelectList processed by the PTF (ie the Select Exprs that have navigation expressions) + * - the columns from the final PTF. + * Why? + * - during translation the input contains Virtual columns that are not present during runtime + * - this messes with the Column Numbers (and hence internal Names) if we add the columns in a different order. + */ + + int pos = 0; + + if (selectDef.getColumns() != null) { + for (ColumnDef cDef : selectDef.getColumns()) { + String internalName = HiveConf.getColumnInternalName(pos++); + ColumnInfo cInfo = new ColumnInfo(internalName, + TypeInfoUtils.getTypeInfoFromObjectInspector(cDef.getOI()), + outAlias, + false); + rr.putExpression(cDef.getExpression(), cInfo); + } + } + + for (ColumnInfo inpCInfo : inputRR.getColumnInfos()) { + String internalName = HiveConf.getColumnInternalName(pos++); + ColumnInfo cInfo = new ColumnInfo(inpCInfo); + cInfo.setInternalName(internalName); + String colAlias = cInfo.getAlias(); + + if (outAlias != null) { + cInfo.setTabAlias(outAlias); + } + + String[] tabColAlias = inputRR.reverseLookup(inpCInfo.getInternalName()); + if (tabColAlias != null) { + colAlias = tabColAlias[1]; + } + if (isWindowPTF && aliasToAST.keySet().contains(colAlias)) { + rr.putExpression(aliasToAST.get(colAlias), cInfo); + } + else { + ASTNode inExpr = null; + inExpr = PTFTranslator.getASTNode(inpCInfo, inputRR); + if ( inExpr != null ) { + rr.putExpression(inExpr, cInfo); + } + else { + rr.put(cInfo.getTabAlias(), colAlias, cInfo); + } + } + } + + selectDef.setRowResolver(rr); + selectDef.setOI((StructObjectInspector) PTFTranslator.getInputOI(rr)); + } + + /* + * Window Function Translation methods + */ + public static WindowFunctionDef translate(PTFDef qDef, TableFuncDef windowTableFnDef, + WindowFunctionSpec wFnSpec) throws SemanticException + { + PTFTranslationInfo tInfo = qDef.getTranslationInfo(); + PTFInputInfo iInfo = tInfo.getInputInfo(windowTableFnDef.getInput()); + + WindowFunctionDef wFnDef = new WindowFunctionDef(); + wFnDef.setSpec(wFnSpec); + + /* + * translate args + */ + ArrayList args = wFnSpec.getArgs(); + if (args != null) + { + for (ASTNode expr : args) + { + ArgDef argDef = null; + try { + argDef = translateWindowFunctionArg(qDef, windowTableFnDef, iInfo, expr); + } + catch(HiveException he) { + throw new SemanticException(he); + } + wFnDef.addArg(argDef); + } + } + + if (RANKING_FUNCS.contains(wFnSpec.getName())) + { + setupRankingArgs(qDef, windowTableFnDef, wFnDef, wFnSpec); + } + + WindowDef wDef = translateWindowSpec(qDef, iInfo, wFnSpec); + wFnDef.setWindow(wDef); + validateWindowDefForWFn(windowTableFnDef, wFnDef); + + try { + setupEvaluator(wFnDef); + } + catch(HiveException he) { + throw new SemanticException(he); + } + + return wFnDef; + } + + static void setupEvaluator(WindowFunctionDef wFnDef) throws HiveException + { + WindowFunctionSpec wSpec = wFnDef.getSpec(); + ArrayList args = wFnDef.getArgs(); + ArrayList argOIs = getWritableObjectInspector(args); + GenericUDAFEvaluator wFnEval = org.apache.hadoop.hive.ql.exec.FunctionRegistry + .getGenericUDAFEvaluator(wSpec.getName(), argOIs, wSpec.isDistinct(), wSpec.isStar()); + ObjectInspector[] funcArgOIs = null; + + if (args != null) + { + funcArgOIs = new ObjectInspector[args.size()]; + int i = 0; + for (ArgDef arg : args) + { + funcArgOIs[i++] = arg.getOI(); + } + } + + ObjectInspector OI = wFnEval.init(GenericUDAFEvaluator.Mode.COMPLETE, funcArgOIs); + + wFnDef.setEvaluator(wFnEval); + wFnDef.setOI(OI); + } + + private static ArgDef translateWindowFunctionArg(PTFDef qDef, TableFuncDef tDef, + PTFInputInfo iInfo, ASTNode arg) throws HiveException + { + return PTFTranslator.buildArgDef(qDef, iInfo, arg); + } + + private static ArrayList getWritableObjectInspector(ArrayList args) + { + ArrayList result = new ArrayList(); + if (args != null) + { + for (ArgDef arg : args) + { + result.add(arg.getOI()); + } + } + return result; + } + + public static final ArrayList RANKING_FUNCS = new ArrayList(); + static + { + RANKING_FUNCS.add("rank"); + RANKING_FUNCS.add("denserank"); + RANKING_FUNCS.add("percentrank"); + RANKING_FUNCS.add("cumedist"); + }; + + private static void setupRankingArgs(PTFDef qDef, TableFuncDef windowTableFnDef, + WindowFunctionDef wFnDef, WindowFunctionSpec wSpec) throws SemanticException + { + if (wSpec.getArgs().size() > 0) + { + throw new SemanticException("Ranking Functions can take no arguments"); + } + + PTFInputDef inpDef = windowTableFnDef.getInput(); + PTFInputInfo inpInfo = qDef.getTranslationInfo().getInputInfo(inpDef); + OrderDef oDef = getTableFuncOrderDef(windowTableFnDef); + ArrayList oCols = oDef.getColumns(); + for (OrderColumnDef oCol : oCols) + { + try { + wFnDef.addArg(PTFTranslator.buildArgDef(qDef, inpInfo, oCol.getExpression())); + } + catch(HiveException he) { + throw new SemanticException(he); + } + } + } + + private static OrderDef getTableFuncOrderDef(TableFuncDef tblFnDef) throws SemanticException + { + if (tblFnDef.getWindow() != null) + { + return tblFnDef.getWindow().getOrderDef(); + } + PTFInputDef iDef = tblFnDef.getInput(); + if (iDef instanceof TableFuncDef) + { + return getTableFuncOrderDef((TableFuncDef) iDef); + } + throw new SemanticException("No Order by specification on Function: " + tblFnDef.getSpec()); + } + + private static WindowDef translateWindowSpec(PTFDef qDef, PTFInputInfo iInfo, + WindowFunctionSpec wFnSpec) throws SemanticException + { + WindowSpec wSpec = wFnSpec.getWindowSpec(); + + if (wSpec == null) { + return null; + } + + WindowFunctionInfo wFnInfo = FunctionRegistry.getWindowFunctionInfo(wFnSpec.getName()); + String desc = wFnSpec.toString(); + + if (wSpec != null && !wFnInfo.isSupportsWindow() && wSpec.getWindow() != null) + { + throw new SemanticException(sprintf("Function %s doesn't support windowing", desc)); + } + return PTFTranslator.translateWindowSpecOnInput(qDef, wSpec, iInfo, desc); + } + + private static void validateWindowDefForWFn(TableFuncDef tFnDef, WindowFunctionDef wFnDef) + throws SemanticException + { + WindowDef tWindow = tFnDef.getWindow(); + WindowDef fWindow = wFnDef.getWindow(); + + PartitionDef tPart = tWindow == null ? null : tWindow.getPartDef(); + PartitionDef fPart = fWindow == null ? null : fWindow.getPartDef(); + + if (!PTFTranslator.isCompatible(tPart, fPart)) + { + throw new SemanticException( + sprintf("Window Function '%s' has an incompatible partition clause", + wFnDef.getSpec())); + } + + OrderDef tOrder = tWindow == null ? null : tWindow.getOrderDef(); + OrderDef fOrder = fWindow == null ? null : fWindow.getOrderDef(); + if (!PTFTranslator.isCompatible(tOrder, fOrder)) + { + throw new SemanticException( + sprintf("Window Function '%s' has an incompatible order clause", wFnDef.getSpec())); + } + } + + public static void addInputColumnsToList(PTFDef qDef, TableFuncDef windowTableFnDef, + ArrayList fieldNames, ArrayList fieldOIs) + { + PTFTranslationInfo tInfo = qDef.getTranslationInfo(); + PTFInputInfo iInfo = tInfo.getInputInfo(windowTableFnDef.getInput()); + + StructObjectInspector OI = (StructObjectInspector) iInfo.getOI(); + for (StructField f : OI.getAllStructFieldRefs()) + { + fieldNames.add(f.getFieldName()); + fieldOIs.add(f.getFieldObjectInspector()); + } + } + + /* + * Partitioning spec (Partition, Order and Window ) translation methods + */ + /* + * compute the Description to use for the Input. + * get the inputInfo for the input: if the function has a MapPhase use the Map Inputfo. + * invoke translateWindowSpecOnInput on WdwSpec of TblFunc + * If TableFunc is the FunctionRegistry.WINDOWING_TABLE_FUNCTION: + * - + */ + static WindowDef translateWindow(PTFDef qDef, TableFuncDef tFnDef) throws SemanticException + { + PTFTranslationInfo tInfo = qDef.getTranslationInfo(); + TableFuncSpec tFnSpec = tFnDef.getTableFuncSpec(); + + /* + * for now the Language only allows explicit specification of Partition & Order clauses. + * Easy to allow references to a Global Window Spec. + */ + WindowSpec wSpec = new WindowSpec(); + + wSpec.setPartition(tFnSpec.getPartition()); + wSpec.setOrder(tFnSpec.getOrder()); + PTFInputDef iDef = tFnDef.getInput(); + + if (wSpec.getPartition() == null) + { + return null; + } + + String desc = getInputDescription(qDef, tFnDef); + TableFunctionEvaluator tFn = tFnDef.getFunction(); + PTFInputInfo iInfo = null; + if (tFn.isTransformsRawInput()) + { + iInfo = tInfo.getMapInputInfo(tFnDef); + } + else + { + iInfo = tInfo.getInputInfo(iDef); + } + + return translateWindowSpecOnInput(qDef, wSpec, iInfo, desc); + } + + /* + *
    + *
  1. If wSpec points to a source WindowSpec. Validate that it is valid. If it hasn't been + * already translated then translate it. + *
  2. Start with an empty WdwDef or a cloned WdwDef from the source WdwDef. + *
  3. translate the PartitionSpec if it exists. Replace the existing PDef with this; also remove + * the OrderDef. + *
  4. translate the OrderSpec if it exists. Replace existing OrderDef with this. + *
  5. add in Partition Columns if not in OrderDef already. + *
  6. translate the WindowSpec if it exists. Replace existing WdwDef with it. + *
  7. If name is non-null add this def to TranslationInfo::nameToWdwDef map. + *
+ */ + static WindowDef translateWindowSpecOnInput(PTFDef qDef, WindowSpec wSpec, PTFInputInfo iInfo, + String inputDesc) throws SemanticException + { + PTFSpec qSpec = qDef.getSpec(); + WindowDef wDef; + + fillInWindowSpec(qSpec, wSpec.getSourceId(), wSpec); + wDef = new WindowDef(wSpec); + + PartitionSpec pSpec = wSpec.getPartition(); + OrderSpec oSpec = wSpec.getOrder(); + WindowFrameSpec wFrameSpec = wSpec.getWindow(); + PartitionDef pDef = translatePartition(qDef, iInfo, pSpec); + OrderDef oDef = translateOrder(qDef, inputDesc, iInfo, oSpec, pDef); + WindowFrameDef wdwDef = translateWindowFrame(qDef, wFrameSpec, iInfo); + + wDef.setPartDef(pDef); + wDef.setOrderDef(oDef); + wDef.setWindow(wdwDef); + + return wDef; + } + + private static void fillInWindowSpec(PTFSpec qSpec, String sourceId, WindowSpec destWSpec) + throws SemanticException + { + if (sourceId != null) + { + WindowSpec sourceWSpec = qSpec.getWindowSpecs().get(sourceId); + if (sourceWSpec == null || sourceWSpec.equals(destWSpec)) + { + throw new SemanticException(sprintf("Window Spec %s refers to an unknown source " , + destWSpec)); + } + + if (destWSpec.getPartition() == null) + { + destWSpec.setPartition(sourceWSpec.getPartition()); + } + + if (destWSpec.getOrder() == null) + { + destWSpec.setOrder(sourceWSpec.getOrder()); + } + + if (destWSpec.getWindow() == null) + { + destWSpec.setWindow(sourceWSpec.getWindow()); + } + + fillInWindowSpec(qSpec, sourceWSpec.getSourceId(), destWSpec); + } + } + + private static PartitionDef translatePartition(PTFDef qDef, PTFInputInfo iInfo, PartitionSpec spec) + throws SemanticException + { + if (spec == null || spec.getColumns() == null || spec.getColumns().size() == 0) { + return null; + } + + PartitionDef pDef = new PartitionDef(spec); + for (ColumnSpec colSpec : spec.getColumns()) + { + ColumnDef cDef = translatePartitionColumn(qDef, iInfo, colSpec); + pDef.addColumn(cDef); + } + return pDef; + } + + private static OrderDef translateOrder(PTFDef qDef, String inputDesc, PTFInputInfo iInfo, + OrderSpec spec, PartitionDef pDef) throws SemanticException + { + + if (spec == null || spec.getColumns() == null || spec.getColumns().size() == 0) + { + if (pDef == null) { + return null; + } + return new OrderDef(pDef); + } + + if (pDef == null) + { + throw new SemanticException(sprintf( + "Input %s cannot have an Order spec w/o a Partition spec", inputDesc)); + } + + OrderDef oDef = new OrderDef(spec); + for (OrderColumnSpec colSpec : spec.getColumns()) + { + OrderColumnDef cDef = translateOrderColumn(qDef, iInfo, colSpec); + oDef.addColumn(cDef); + } + + /* + * either all partition columns must be in Order list or none must be specified. + * If none are specified then add them all. + */ + int numOfPartColumns = 0; + List orderCols = oDef.getColumns(); + List partCols = pDef.getColumns(); + int chkSize = partCols.size(); + chkSize = chkSize > orderCols.size() ? orderCols.size() : chkSize; + for (int i = 0; i < chkSize; i++) + { + if (orderCols.get(i).getSpec().getExpression().toStringTree() + .equals(partCols.get(i).getSpec().getExpression().toStringTree())) + { + numOfPartColumns++; + } else { + break; + } + } + + if (numOfPartColumns != 0 && numOfPartColumns != partCols.size()) + { + throw new SemanticException( + sprintf( + "For Input %s:n all partition columns must be in order clause or none should be specified", + inputDesc)); + } + + ArrayList combinedOrderCols = new ArrayList(); + if (numOfPartColumns == 0) + { + for (ColumnDef cDef : partCols) + { + OrderColumnDef ocDef = new OrderColumnDef(cDef); + combinedOrderCols.add(ocDef); + } + combinedOrderCols.addAll(orderCols); + oDef.setColumns(combinedOrderCols); + } + + return oDef; + } + + private static OrderColumnDef translateOrderColumn(PTFDef qDef, PTFInputInfo iInfo, + OrderColumnSpec oSpec) throws SemanticException + { + OrderColumnDef ocDef = new OrderColumnDef(oSpec); + translateColumn(qDef, ocDef, iInfo, oSpec); + PTFTranslator.validateComparable(ocDef.getOI(), + sprintf("Partition Column %s is not comparable", oSpec)); + return ocDef; + } + + + private static ColumnDef translatePartitionColumn(PTFDef qDef, PTFInputInfo iInfo, + ColumnSpec cSpec) throws SemanticException + { + ColumnDef cDef = new ColumnDef(cSpec); + translateColumn(qDef, cDef, iInfo, cSpec); + PTFTranslator.validateComparable(cDef.getOI(), + sprintf("Partition Column %s is not comparable", cSpec)); + return cDef; + } + + private static void translateColumn(PTFDef qDef, ColumnDef cDef, PTFInputInfo iInfo, + ColumnSpec cSpec) throws SemanticException + { + String colTabName = cSpec.getTableName(); + if (colTabName != null && !colTabName.equals(iInfo.getAlias())) + { + throw new SemanticException(sprintf("Unknown Table Reference in column", cSpec)); + } + + ASTNode expr = cSpec.getExpression(); + ExprNodeDesc exprNode = PTFTranslator.buildExprNode(expr, iInfo.getTypeCheckCtx()); + ExprNodeEvaluator exprEval = null; + ObjectInspector oi = null; + + try { + exprEval = WindowingExprNodeEvaluatorFactory.get(qDef.getTranslationInfo(), exprNode); + oi = PTFTranslator.initExprNodeEvaluator(qDef, exprNode, exprEval, iInfo); + } + catch(HiveException he) { + throw new SemanticException(he); + } + + cDef.setExpression(expr); + cDef.setExprNode(exprNode); + cDef.setExprEvaluator(exprEval); + cDef.setOI(oi); + cDef.setAlias(cSpec.getColumnName()); + } + + + private static WindowFrameDef translateWindowFrame(PTFDef qDef, WindowFrameSpec wfSpec, + PTFInputInfo iInfo) throws SemanticException + { + if (wfSpec == null) + { + return null; + } + + BoundarySpec s = wfSpec.getStart(); + BoundarySpec e = wfSpec.getEnd(); + WindowFrameDef wfDef = new WindowFrameDef(wfSpec); + + wfDef.setStart(translateBoundary(qDef, s, iInfo)); + wfDef.setEnd(translateBoundary(qDef, e, iInfo)); + + int cmp = s.compareTo(e); + if (cmp > 0) + { + throw new SemanticException(sprintf( + "Window range invalid, start boundary is greater than end boundary: %s", wfSpec)); + } + return wfDef; + } + + private static BoundaryDef translateBoundary(PTFDef qDef, BoundarySpec bndSpec, PTFInputInfo iInfo) + throws SemanticException + { + if (bndSpec instanceof ValueBoundarySpec) + { + ValueBoundarySpec vBndSpec = (ValueBoundarySpec) bndSpec; + ValueBoundaryDef vbDef = new ValueBoundaryDef(vBndSpec); + PTFTranslator.validateNoLeadLagInValueBoundarySpec(vBndSpec.getExpression()); + ExprNodeDesc exprNode = PTFTranslator.buildExprNode(vBndSpec.getExpression(), + iInfo.getTypeCheckCtx()); + vbDef.setExprNode(exprNode); + ExprNodeEvaluator exprEval = null; + ObjectInspector OI = null; + try { + exprEval = WindowingExprNodeEvaluatorFactory.get(qDef.getTranslationInfo(), exprNode); + OI = PTFTranslator.initExprNodeEvaluator(qDef, exprNode, exprEval, iInfo); + } + catch(HiveException he) { + throw new SemanticException(he); + } + PTFTranslator.validateValueBoundaryExprType(OI); + vbDef.setExprEvaluator(exprEval); + vbDef.setOI(OI); + return vbDef; + } + else if (bndSpec instanceof RangeBoundarySpec) + { + RangeBoundarySpec rBndSpec = (RangeBoundarySpec) bndSpec; + RangeBoundaryDef rbDef = new RangeBoundaryDef(rBndSpec); + return rbDef; + } + else if (bndSpec instanceof CurrentRowSpec) + { + CurrentRowSpec cBndSpec = (CurrentRowSpec) bndSpec; + CurrentRowDef cbDef = new CurrentRowDef(cBndSpec); + return cbDef; + } + throw new SemanticException("Unknown Boundary: " + bndSpec); + } + + private static String getInputDescription(PTFDef qDef, TableFuncDef tDef) + { + if (qDef.getInput() == tDef && + (tDef.getName().equals(FunctionRegistry.NOOP_TABLE_FUNCTION) || + tDef.getName().equals(FunctionRegistry.WINDOWING_TABLE_FUNCTION))) + { + return "Query"; + } + return sprintf("TableFunction %s[alias:%s]", tDef.getName(), tDef.getAlias()); + + } + + /* + * If the cInfo is for an ASTNode, this function returns the ASTNode that it is for. + */ + public static ASTNode getASTNode(ColumnInfo cInfo, RowResolver rr) throws SemanticException { + for(Map.Entry entry : rr.getExpressionMap().entrySet()) { + ASTNode expr = entry.getValue(); + if ( rr.getExpression(expr).equals(cInfo)) { + return expr; + } + } + return null; + } + + /* + * The information captured during translation of a QuerySpec into a QueryDef + */ + public static class PTFTranslationInfo + { + private static final Log LOG = LogFactory.getLog("com.sap.hadoop.windowing"); + + HiveConf hCfg; + + /* + * A map from a QueryInput to the WindowDefns defined on it. + * In the future this will enable (Partition, Order) specification for a QueryInput to refer to + * a Window definition. + * This will require that we accept window definitions on DataSets other then the input to the + * Windowing Table Function. + */ + Map> windowDefMap; + + /* + * A map from a QueryInput to its Shape. + */ + Map inputInfoMap; + + /* + * InputInfos for table functions that rehape the input map-side. + */ + Map mapReshapeInfoMap; + + LeadLagInfo llInfo; + + UnparseTranslator unparseT; + + public HiveConf getHiveCfg() + { + return hCfg; + } + + public void setHiveCfg(HiveConf hCfg) + { + this.hCfg = hCfg; + } + + void addInput(PTFInputDef input, RowResolver rr) throws SemanticException + { + inputInfoMap = inputInfoMap == null ? new HashMap() : inputInfoMap; + + RowResolver prevRR = null; + if (input instanceof TableFuncDef) { + PTFInputDef prevInpDef = ((TableFuncDef) input).getInput(); + prevRR = inputInfoMap.get(prevInpDef.getAlias()).rr; + } + + PTFInputInfo iInfo = new PTFInputInfo(this, input, null, rr, prevRR, unparseT); + input.setInputInfo(iInfo); + inputInfoMap.put(input.getAlias(), iInfo); + } + + public PTFInputInfo getMapInputInfo(TableFuncDef tDef) throws SemanticException + { + TableFunctionEvaluator tFn = tDef.getFunction(); + if (!tFn.isTransformsRawInput()) + { + return null; + } + mapReshapeInfoMap = mapReshapeInfoMap == null ? new HashMap() + : mapReshapeInfoMap; + PTFInputInfo ii = mapReshapeInfoMap.get(tDef.getAlias()); + if (ii == null) + { + PTFInputDef prevInpDef = tDef.getInput(); + RowResolver prevRR = inputInfoMap.get(prevInpDef.getAlias()).rr; + ii = new PTFInputInfo(this, tDef, tFn.getRawInputOI(), null, prevRR, unparseT); + tDef.setRawInputInfo(ii); + mapReshapeInfoMap.put(tDef.getAlias(), ii); + } + return ii; + } + + public PTFInputInfo getInputInfo(PTFInputDef input) + { + return inputInfoMap.get(input.getAlias()); + } + + public LeadLagInfo getLLInfo() + { + llInfo = llInfo == null ? new LeadLagInfo() : llInfo; + return llInfo; + } + + public static RowResolver buildRowResolver(String tabAlias, + StructObjectInspector rowObjectInspector, + ArrayList outputColNames, + RowResolver inputRowResolver) throws SemanticException { + if ( outputColNames != null ) { + return buildRowResolver(tabAlias, rowObjectInspector, outputColNames); + } + else { + return buildRowResolver(tabAlias, rowObjectInspector, inputRowResolver); + } + } + + protected static RowResolver buildRowResolver(String tabAlias, + StructObjectInspector rowObjectInspector, + ArrayList outputColNames) throws SemanticException { + RowResolver rwsch = new RowResolver(); + List fields = rowObjectInspector.getAllStructFieldRefs(); + for (int i = 0; i < fields.size(); i++) + { + ColumnInfo colInfo = new ColumnInfo(fields.get(i).getFieldName(), + TypeInfoUtils.getTypeInfoFromObjectInspector(fields.get(i) + .getFieldObjectInspector()), + tabAlias, + false); + rwsch.put(tabAlias, outputColNames.get(i), colInfo); + } + return rwsch; + } + + /* + * Construct a Row Resolver from a PTF OI. + * For WindowTablFunction and Noop functions the PTF's Input RowResolver is also passed in. + * This way for any columns that are in the Input Row Resolver we carry forward their + * internalname, alias and table Alias. + */ + protected static RowResolver buildRowResolver(String tabAlias, + StructObjectInspector rowObjectInspector, + RowResolver inputRowResolver) throws SemanticException + { + LOG.info("QueryTranslationInfo::getRowResolver invoked on ObjectInspector"); + RowResolver rwsch = new RowResolver(); + List fields = rowObjectInspector.getAllStructFieldRefs(); + for (int i = 0; i < fields.size(); i++) + { + StructField field = fields.get(i); + String internalName = field.getFieldName(); + String[] tabColAlias = inputRowResolver == null ? null : inputRowResolver + .reverseLookup(internalName); + String colTabAlias = tabColAlias == null ? tabAlias : tabColAlias[0]; + String colAlias = tabColAlias == null ? null : tabColAlias[1]; + ColumnInfo inpColInfo; + ColumnInfo colInfo; + + if ( tabColAlias != null ) { + inpColInfo = inputRowResolver.get(colTabAlias, colAlias); + } + else { + /* + * for the Virtual columns: + * - the internalName is UPPER Case and the alias is lower case + * - since we put them in an OI, the fieldnames became lower cased. + * - so we look in the inputRR for the fieldName as an alias. + */ + inpColInfo = inputRowResolver == null ? null : inputRowResolver + .get(tabAlias, internalName); + colAlias = inpColInfo != null ? inpColInfo.getInternalName() : colAlias; + } + + if ( inpColInfo != null ) { + colInfo = new ColumnInfo(inpColInfo); + } + else { + colInfo = new ColumnInfo(fields.get(i).getFieldName(), + TypeInfoUtils.getTypeInfoFromObjectInspector(fields.get(i) + .getFieldObjectInspector()), + tabAlias, + false); + colAlias = colInfo.getInternalName(); + } + + ASTNode expr = inputRowResolver == null ? null :PTFTranslator.getASTNode(inpColInfo, inputRowResolver); + + if ( expr != null ) { + rwsch.putExpression(expr, colInfo); + } + else { + rwsch.put(colTabAlias, colAlias, colInfo); + } + } + return rwsch; + } + } + + public static class PTFInputInfo + { + private final boolean forMapPhase; + private final PTFInputDef inpDef; + private final StructObjectInspector OI; + private final RowResolver rr; + private final TypeCheckCtx tCtx; + + PTFInputInfo(PTFTranslationInfo tInfo, PTFInputDef input, + StructObjectInspector mapOI, RowResolver inputRR, + RowResolver prevInputRR, + UnparseTranslator unparseT) + throws SemanticException + { + this.inpDef = input; + this.forMapPhase = mapOI != null; + OI = forMapPhase ? mapOI : inpDef.getOI(); + + if ( inputRR != null ) { + rr = inputRR; + } + else if ( input instanceof TableFuncDef ){ + TableFuncDef tFnDef = (TableFuncDef) input; + boolean carryFwdNames = tFnDef.isCarryForwardNames(); + ArrayList outputColNames = forMapPhase ? tFnDef.getRawInputColumnNames() : tFnDef.getOutputColumnNames(); + rr = PTFTranslationInfo.buildRowResolver(null, OI, outputColNames, carryFwdNames ? prevInputRR : null); + } + else { + /* + * this is for the start of a Chain; so prevInputRR must be null. + */ + rr = PTFTranslationInfo.buildRowResolver(null, OI, null, prevInputRR); + } + tCtx = new TypeCheckCtx(rr); + tCtx.setUnparseTranslator(unparseT); + } + + public RowResolver getRowResolver() + { + return rr; + } + + public TypeCheckCtx getTypeCheckCtx() + { + return tCtx; + } + + public String getAlias() + { + return inpDef.getAlias(); + } + + public PTFInputDef getInputDef() + { + return inpDef; + } + + public ObjectInspector getOI() + { + return OI; + } + } + + public static class LeadLagInfo + { + List leadLagExprs; + Map> mapTopExprToLLFunExprs; + + private void addLeadLagExpr(ExprNodeGenericFuncDesc llFunc) + { + leadLagExprs = leadLagExprs == null ? new ArrayList() : leadLagExprs; + leadLagExprs.add(llFunc); + } + + public List getLeadLagExprs() + { + return leadLagExprs; + } + + public void addLLFuncExprForTopExpr(ExprNodeDesc topExpr, ExprNodeGenericFuncDesc llFuncExpr) + { + addLeadLagExpr(llFuncExpr); + mapTopExprToLLFunExprs = mapTopExprToLLFunExprs == null ? + new HashMap>() : mapTopExprToLLFunExprs; + List funcList = mapTopExprToLLFunExprs.get(topExpr); + if (funcList == null) + { + funcList = new ArrayList(); + mapTopExprToLLFunExprs.put(topExpr, funcList); + } + funcList.add(llFuncExpr); + } + + public List getLLFuncExprsInTopExpr(ExprNodeDesc topExpr) + { + if (mapTopExprToLLFunExprs == null) { + return null; + } + return mapTopExprToLLFunExprs.get(topExpr); + } + } + + public static ArrayList componentize(PTFSpec ptfSpec) throws SemanticException { + ArrayList componentPTFSpecs = new ArrayList(); + Iterator it = PTFTranslator.iterateTableFuncSpecs(ptfSpec, true); + TableFuncSpec prevInSpec = it.next(); + + PartitionSpec partSpec = prevInSpec.getPartition(); + OrderSpec orderSpec = prevInSpec.getOrder(); + + if ( partSpec == null ) { + //oops this should have been caught before trying to componentize + throw new SemanticException("No Partitioning specification specified at start of a PTFChain"); + } + if ( orderSpec == null ) { + orderSpec = new OrderSpec(partSpec); + prevInSpec.setOrder(orderSpec); + } + + while (it.hasNext()) { + TableFuncSpec currentSpec = it.next(); + + String fnName = currentSpec.getName(); + if ( !FunctionRegistry.isTableFunction(fnName)) { + throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg(fnName)); + } + boolean transformsRawInput = FunctionRegistry.getTableFunctionResolver(fnName).transformsRawInput(); + + /* + * if the current table function has no partition info specified: inherit it from the PTF up the chain. + */ + if ( currentSpec.getPartition() == null ) { + currentSpec.setPartition(partSpec); + if ( currentSpec.getOrder() == null ) { + currentSpec.setOrder(orderSpec); + } + } + /* + * If the current table function has no order info specified; + */ + if ( currentSpec.getOrder() == null ) { + currentSpec.setOrder(new OrderSpec(currentSpec.getPartition())); + } + + if(!currentSpec.getPartition().equals(partSpec) || + !currentSpec.getOrder().equals(orderSpec) || + transformsRawInput ) { + PTFSpec cQSpec = new PTFSpec(); + cQSpec.setInput(prevInSpec); + componentPTFSpecs.add(cQSpec); + PTFComponentQuerySpec cQInSpec= new PTFComponentQuerySpec(); + currentSpec.setInput(cQInSpec); + } + prevInSpec = currentSpec; + partSpec = prevInSpec.getPartition(); + orderSpec = prevInSpec.getOrder(); + } + + componentPTFSpecs.add(ptfSpec); + + return componentPTFSpecs; + + } + + /* + * Utility classes and methods + */ + + // Function Chain Iterators + public static Predicate IsTableFunc = new Predicate() { + @Override + public boolean apply(PTFInputSpec obj) { + return obj != null && obj instanceof TableFuncSpec; + } + }; + + public static class QueryInputSpecIterator implements Iterator + { + PTFSpec qSpec; + PTFInputSpec nextInput; + + public QueryInputSpecIterator(PTFSpec qSpec) + { + this.qSpec = qSpec; + nextInput = qSpec.getInput(); + } + + @Override + public boolean hasNext() + { + return nextInput != null; + } + + @Override + public PTFInputSpec next() + { + PTFInputSpec curr = nextInput; + if (curr instanceof TableFuncSpec) + { + TableFuncSpec tFunc = (TableFuncSpec) curr; + nextInput = tFunc.getInput(); + } + else + { + nextInput = null; + } + return curr; + } + + @Override + public void remove() + { + throw new UnsupportedOperationException(); + } + } + + public static class ReverseQueryInputSpecIterator extends ReverseIterator + { + ReverseQueryInputSpecIterator(PTFSpec qSpec) + { + super(new QueryInputSpecIterator(qSpec)); + } + } + + public static class TableFunctionSpecIterator implements Iterator + { + QueryInputSpecIterator qSpecIt; + TableFuncSpec nextInput; + + public TableFunctionSpecIterator(PTFSpec qSpec) + { + qSpecIt = new QueryInputSpecIterator(qSpec); + } + + @Override + public boolean hasNext() + { + if (qSpecIt.hasNext()) + { + PTFInputSpec iSpec = qSpecIt.next(); + if (iSpec instanceof TableFuncSpec) + { + nextInput = (TableFuncSpec) iSpec; + return true; + } + } + return false; + } + + @Override + public TableFuncSpec next() + { + return nextInput; + } + + @Override + public void remove() + { + throw new UnsupportedOperationException(); + } + } + + public static class ReverseTableFunctionSpecIterator extends ReverseIterator + { + ReverseTableFunctionSpecIterator(PTFSpec qSpec) + { + super(new TableFunctionSpecIterator(qSpec)); + } + } + + public static class QueryInputDefIterator implements Iterator + { + PTFDef qDef; + PTFInputDef nextInput; + + public QueryInputDefIterator(PTFDef qDef) + { + this.qDef = qDef; + nextInput = qDef.getInput(); + } + + @Override + public boolean hasNext() + { + return nextInput != null; + } + + @Override + public PTFInputDef next() + { + PTFInputDef curr = nextInput; + if (curr instanceof TableFuncDef) + { + TableFuncDef tFunc = (TableFuncDef) curr; + nextInput = tFunc.getInput(); + } + else + { + nextInput = null; + } + return curr; + } + + @Override + public void remove() + { + throw new UnsupportedOperationException(); + } + } + + public static class ReverseQueryInputDefIterator extends ReverseIterator + { + ReverseQueryInputDefIterator(PTFDef qDef) + { + super(new QueryInputDefIterator(qDef)); + } + } + + static Iterator iterateInputSpecs(PTFSpec qSpec, + boolean reverse) + { + return reverse ? new ReverseQueryInputSpecIterator(qSpec) + : new QueryInputSpecIterator(qSpec); + } + + public static Iterator iterateTableFuncSpecs( + PTFSpec qSpec, boolean reverse) + { + return reverse ? new ReverseTableFunctionSpecIterator(qSpec) + : new TableFunctionSpecIterator(qSpec); + } + + public static Iterator iterateInputDefs(PTFDef qDef, + boolean reverse) + { + return reverse ? new ReverseQueryInputDefIterator(qDef) + : new QueryInputDefIterator(qDef); + } + + public static PTFTableOrSubQueryInputSpec getHiveTableSpec(PTFSpec qSpec) { + Iterator it = PTFTranslator.iterateInputSpecs(qSpec, true); + PTFInputSpec qInSpec = it.next(); + return (PTFTableOrSubQueryInputSpec) qInSpec; + } + + public static ExprNodeDesc buildExprNode(ASTNode expr, + TypeCheckCtx typeCheckCtx) throws SemanticException + { + // todo: use SemanticAnalyzer::genExprNodeDesc + // currently SA not available to PTFTranslator. + HashMap map = TypeCheckProcFactory + .genExprNode(expr, typeCheckCtx); + ExprNodeDesc desc = (ExprNodeDesc) map.get(expr); + if (desc == null) { + String errMsg = typeCheckCtx.getError(); + if ( errMsg == null) { + errMsg = "Error in parsing "; + } + throw new SemanticException(errMsg); + } + return desc; + } + + public static ObjectInspector initExprNodeEvaluator(PTFDef qDef, + ExprNodeDesc exprNode, ExprNodeEvaluator exprEval, PTFInputInfo iInfo) + throws HiveException + { + ObjectInspector OI; + OI = exprEval.initialize(iInfo.getOI()); + + /* + * if there are any LeadLag functions in this Expression Tree: - setup a + * duplicate Evaluator for the 1st arg of the LLFuncDesc - initialize it + * using the InputInfo provided for this Expr tree - set the duplicate + * evaluator on the LLUDF instance. + */ + LeadLagInfo llInfo = qDef.getTranslationInfo().getLLInfo(); + List llFuncExprs = llInfo + .getLLFuncExprsInTopExpr(exprNode); + if (llFuncExprs != null) + { + for (ExprNodeGenericFuncDesc llFuncExpr : llFuncExprs) + { + ExprNodeDesc firstArg = llFuncExpr.getChildren().get(0); + ExprNodeEvaluator dupExprEval = WindowingExprNodeEvaluatorFactory + .get(qDef.getTranslationInfo(), firstArg); + dupExprEval.initialize(iInfo.getOI()); + GenericUDFLeadLag llFn = (GenericUDFLeadLag) llFuncExpr + .getGenericUDF(); + llFn.setExprEvaluator(dupExprEval); + } + } + + return OI; + } + + public static ArgDef buildArgDef(PTFDef qDef, PTFInputInfo iInfo, ASTNode arg) + throws HiveException + { + ArgDef argDef = new ArgDef(); + + ExprNodeDesc exprNode = PTFTranslator.buildExprNode(arg, + iInfo.getTypeCheckCtx()); + ExprNodeEvaluator exprEval = WindowingExprNodeEvaluatorFactory.get( + qDef.getTranslationInfo(), exprNode); + ObjectInspector oi = initExprNodeEvaluator(qDef, exprNode, exprEval, + iInfo); + + argDef.setExpression(arg); + argDef.setExprNode(exprNode); + argDef.setExprEvaluator(exprEval); + argDef.setOI(oi); + return argDef; + } + + public static ArrayList[] getTypeMap( + StructObjectInspector oi) + { + StructTypeInfo t = (StructTypeInfo) TypeInfoUtils + .getTypeInfoFromObjectInspector(oi); + ArrayList fnames = t.getAllStructFieldNames(); + ArrayList fields = t.getAllStructFieldTypeInfos(); + return new ArrayList[] + { fnames, fields }; + } + + public static SerDe createLazyBinarySerDe(Configuration cfg, + StructObjectInspector oi) throws SerDeException + { + return createLazyBinarySerDe(cfg, oi, null); + } + + public static SerDe createLazyBinarySerDe(Configuration cfg, + StructObjectInspector oi, Map serdePropsMap) throws SerDeException + { + serdePropsMap = serdePropsMap == null ? new LinkedHashMap() : serdePropsMap; + + addOIPropertiestoSerDePropsMap(oi, serdePropsMap); + + SerDe serDe = new LazyBinarySerDe(); + Properties p = new Properties(); + p.setProperty(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMNS, + serdePropsMap.get(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMNS)); + p.setProperty( + org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMN_TYPES, + serdePropsMap.get(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMN_TYPES)); + serDe.initialize(cfg, p); + return serDe; + } + + @SuppressWarnings({"unchecked"}) + private static void addOIPropertiestoSerDePropsMap(StructObjectInspector OI, Map serdePropsMap) { + + if ( serdePropsMap == null ) { + return; + } + + ArrayList[] tInfo = getTypeMap(OI); + + ArrayList columnNames = (ArrayList) tInfo[0]; + ArrayList fields = (ArrayList) tInfo[1]; + StringBuilder cNames = new StringBuilder(); + StringBuilder cTypes = new StringBuilder(); + + for (int i = 0; i < fields.size(); i++) + { + cNames.append(i > 0 ? "," : ""); + cTypes.append(i > 0 ? "," : ""); + cNames.append(columnNames.get(i)); + cTypes.append(fields.get(i).getTypeName()); + } + + serdePropsMap.put(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMNS, cNames.toString()); + serdePropsMap.put(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMN_TYPES, cTypes.toString()); + } + + /* + * @remove + * used for validating that OI columnNames are the same during runtime. + */ + @SuppressWarnings("unused") + private static void checkSame(Map transProps, Map runtimeProps) throws SerDeException { + String transNames = transProps.get(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMNS); + String runtimeNames = runtimeProps.get(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMNS); + + if ( !transNames.startsWith(runtimeNames) ) { + /* + * Why is this false? + * There is one case where the translation names don't match the runtime names. + * If a UDAF expression has an alias and is used in the having clause (see test 52.) + * it is added to the ReduceSink Operator's RowResolver twice with the same name. + * But at runtime each instance of this column is given a different internal Name. + * This doesn't seem to affect the child Operators of ReduceSink because their + * ColumnExprNodeDesc's can still get to the one instance of the column in the RowResolver. + */ + if ( false ) { + throw new SerDeException("Runtime colNames changed: " + transNames + " -> " + runtimeNames ); + } + } + + String transTypes = transProps.get(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMN_TYPES); + String runtimeTypes = runtimeProps.get(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMN_TYPES); + + /* + * Why is this false? + * - see above + * @revisit NPath tpath TypeInformation has alias names in Struct during translation; internalNames during runtime. + */ + if ( !transTypes.startsWith(runtimeTypes)) { + if ( false ) { + throw new SerDeException("Runtime colTypes changed: " + transTypes + " -> " + runtimeTypes ); + } + } + } + + @SuppressWarnings("unchecked") + public static Properties buildSerDePropertiesFromOI(StructObjectInspector OI) { + Properties p = new Properties(); + ArrayList[] tInfo = getTypeMap(OI); + + ArrayList columnNames = (ArrayList) tInfo[0]; + ArrayList fields = (ArrayList) tInfo[1]; + StringBuilder cNames = new StringBuilder(); + StringBuilder cTypes = new StringBuilder(); + for (int i = 0; i < fields.size(); i++) + { + cNames.append(i > 0 ? "," : ""); + cTypes.append(i > 0 ? "," : ""); + cNames.append(columnNames.get(i)); + cTypes.append(fields.get(i).getTypeName()); + } + p.setProperty(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMNS, + cNames.toString()); + p.setProperty( + org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMN_TYPES, + cTypes.toString()); + + return p; + } + + + public static ASTNode buildASTNode(String colName) + { + TreeWizard tw = new TreeWizard(adaptor, HiveParser.tokenNames); + Object o = tw.create(sprintf("(TOK_TABLE_OR_COL Identifier[%s])", colName)); + return (ASTNode) o; + } + + public static void validateComparable(ObjectInspector OI, String errMsg) + throws SemanticException + { + if (!ObjectInspectorUtils.compareSupported(OI)) + { + throw new SemanticException(errMsg); + } + } + + /* + * - equal if there columnName & tableName match - a null tableName is + * interpreted as matching the other tableName + */ + public static boolean isEqual(ColumnSpec spec1, ColumnSpec spec2) + { + if (spec1 == null && spec2 == null) { + return false; + } + if (spec1 == null && spec2 != null) { + return false; + } + if (spec1 != null && spec2 == null) { + return false; + } + + if (!spec1.getExpression().toStringTree().equals(spec2.getExpression().toStringTree())) { + return false; + } + + String t1 = spec1.getTableName(); + String t2 = spec2.getTableName(); + if (t1 == null || t2 == null) { + return true; + } + + return t1.equals(t2); + } + + public static boolean isEqual(ColumnDef def1, ColumnDef def2) + { + if (def1 == null && def2 == null) { + return false; + } + if (def1 == null && def2 != null) { + return false; + } + if (def1 != null && def2 == null) { + return false; + } + + return isEqual(def1.getSpec(), def2.getSpec()); + } + + /** + * For NOOP table functions, the serde is the same as that on the input + * hive table,; for other table functions it is the lazy binary serde. + * If the query has a map-phase, the map oi is set to be the oi on the + * lazy binary serde unless the table function is a NOOP_MAP_TABLE_FUNCTION + * (in which case it is set to the oi on the serde of the input hive + * table definition). + * @param tDef + * @param inputDef + * @param tInfo + * @param tEval + * @throws SerDeException + */ + public static void setupSerdeAndOI(TableFuncDef tDef, + PTFInputDef inputDef, PTFTranslationInfo tInfo, + TableFunctionEvaluator tEval, boolean translationTime) throws SerDeException + { + /* + * setup the SerDe. + */ + SerDe serde = null; + HashMap serDePropsMap = new HashMap(); + // treat Noop Function special because it just hands the input Partition + // to the next function in the chain. + if (tDef.getName().equals(FunctionRegistry.NOOP_TABLE_FUNCTION) + || tDef.getName().equals( + FunctionRegistry.NOOP_MAP_TABLE_FUNCTION)) + { + serde = inputDef.getSerde(); + addOIPropertiestoSerDePropsMap((StructObjectInspector) serde.getObjectInspector(), serDePropsMap); + } + else + { + serde = PTFTranslator.createLazyBinarySerDe(tInfo.getHiveCfg(), + tEval.getOutputOI(), serDePropsMap); + } + tDef.setSerde(serde); + if ( translationTime ) { + tDef.setOutputSerdeProps(serDePropsMap); + } + else { + checkSame(tDef.getOutputSerdeProps(), serDePropsMap); + } + tDef.setOI((StructObjectInspector) serde.getObjectInspector()); + + if (tEval.isTransformsRawInput()) + { + serDePropsMap.clear(); + if (tDef.getName().equals(FunctionRegistry.NOOP_MAP_TABLE_FUNCTION)) + { + serde = inputDef.getSerde(); + addOIPropertiestoSerDePropsMap((StructObjectInspector) serde.getObjectInspector(), serDePropsMap); + } + else + { + serde = PTFTranslator.createLazyBinarySerDe( + tInfo.getHiveCfg(), tEval.getRawInputOI(), serDePropsMap); + } + tDef.setRawInputOI((StructObjectInspector) serde + .getObjectInspector()); + if ( translationTime ) { + tDef.setRawInputSerdeProps(serDePropsMap); + } + else { + checkSame(tDef.getRawInputSerdeProps(), serDePropsMap); + } + } + + } + + /** + * Returns true if the query needs a map-side reshape. PTFOperator is added + * on the map-side before ReduceSinkOperator in this scenario. + * + * @param qdef + * @return + * @throws SemanticException + */ + public static boolean addPTFMapOperator(PTFDef qdef) throws SemanticException { + boolean hasMap = false; + TableFuncDef tabDef = PTFTranslator.getFirstTableFunction(qdef); + TableFunctionEvaluator tEval = tabDef.getFunction(); + if (tEval.isTransformsRawInput()) { + hasMap = true; + } + return hasMap; + + } + + /** + * For each column on the input RR, construct a StructField for it + * OI is constructed using the list of input column names and + * their corresponding OIs. + * @param rr + * @return + */ + public static ObjectInspector getInputOI(RowResolver rr) { + StructObjectInspector oi; + ArrayList colLists = rr.getColumnInfos(); + ArrayList structFieldNames = new ArrayList(); + ArrayList structFieldObjectInspectors = new ArrayList(); + for (ColumnInfo columnInfo : colLists) { + String colName = columnInfo.getInternalName(); + ObjectInspector colOI = columnInfo.getObjectInspector(); + structFieldNames.add(colName); + structFieldObjectInspectors.add(colOI); + } + oi = ObjectInspectorFactory.getStandardStructObjectInspector(structFieldNames, + structFieldObjectInspectors); + return oi; + + } + + public static ExprNodeColumnDesc getStringColumn(String columnName) + { + return new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, + columnName, "", false); + } + + /* + * A Window Function's partition clause must exactly match that of the + * associated tableFn. + */ + public static boolean isCompatible(PartitionDef tFnPartDef, + PartitionDef wFnPartDef) + { + if (tFnPartDef == null && wFnPartDef == null) { + return true; + } + if (tFnPartDef == null && wFnPartDef != null) { + return false; + } + if (tFnPartDef != null && wFnPartDef == null) { + return true; + } + + ArrayList cols1 = tFnPartDef.getColumns(); + ArrayList cols2 = wFnPartDef.getColumns(); + if (cols1.size() != cols2.size()) { + return false; + } + for (int i = 0; i < cols1.size(); i++) + { + boolean e = isEqual(cols1.get(i), cols2.get(i)); + if (!e) { + return false; + } + } + return true; + } + + public static boolean isEqual(OrderColumnSpec spec1, OrderColumnSpec spec2) + { + if (spec1 == null && spec2 == null) { + return false; + } + if (spec1 == null && spec2 != null) { + return false; + } + if (spec1 != null && spec2 == null) { + return false; + } + + if (!spec1.getExpression().toStringTree().equals(spec2.getExpression().toStringTree())) { + return false; + } + if (!spec1.getOrder().equals(spec2.getOrder())) { + return false; + } + + String t1 = spec1.getTableName(); + String t2 = spec2.getTableName(); + if (t1 == null || t2 == null) { + return true; + } + + return t1.equals(t2); + } + + public static boolean isEqual(OrderColumnDef def1, OrderColumnDef def2) + { + if (def1 == null && def2 == null) { + return false; + } + if (def1 == null && def2 != null) { + return false; + } + if (def1 != null && def2 == null) { + return false; + } + + return isEqual((OrderColumnSpec) def1.getSpec(), + (OrderColumnSpec) def2.getSpec()); + } + + public static boolean isCompatible(OrderDef tFnOrderDef, OrderDef wFnOrderDef) + { + if (tFnOrderDef == null && wFnOrderDef == null) { + return true; + } + if (tFnOrderDef == null && wFnOrderDef != null) { + return false; + } + if (tFnOrderDef != null && wFnOrderDef == null) { + return true; + } + + ArrayList cols1 = tFnOrderDef.getColumns(); + ArrayList cols2 = wFnOrderDef.getColumns(); + if (cols1.size() != cols2.size()) { + return false; + } + for (int i = 0; i < cols1.size(); i++) + { + boolean e = isEqual(cols1.get(i), cols2.get(i)); + if (!e) { + return false; + } + } + return true; + } + + public static void validateValueBoundaryExprType(ObjectInspector OI) + throws SemanticException + { + if (!OI.getCategory().equals(Category.PRIMITIVE)) + { + throw new SemanticException( + "Value Boundary expression must be of primitve type"); + } + + PrimitiveObjectInspector pOI = (PrimitiveObjectInspector) OI; + PrimitiveCategory pC = pOI.getPrimitiveCategory(); + + switch (pC) + { + case BYTE: + case DOUBLE: + case FLOAT: + case INT: + case LONG: + case SHORT: + case TIMESTAMP: + break; + default: + throw new SemanticException( + sprintf("Primitve type %s not supported in Value Boundary expression", + pC)); + } + + } + + /** + * Iterate the list of the query input definitions in reverse order + * Return the first table function definition in the chain. + * This table function is the first one to be executed on the + * input hive table. + * @param qDef + * @return + */ + public static TableFuncDef getFirstTableFunction(PTFDef qDef){ + TableFuncDef tabDef = null; + Iterator it = PTFTranslator.iterateInputDefs(qDef, true); + while(it.hasNext()){ + PTFInputDef qIn = it.next(); + if(qIn instanceof TableFuncDef){ + tabDef = (TableFuncDef) qIn; + break; + } + } + return tabDef; + + } + + public static int compare(Object[] o1, ObjectInspector[] oi1, Object[] o2, + ObjectInspector[] oi2) + { + int c = 0; + for (int i = 0; i < oi1.length; i++) + { + c = ObjectInspectorUtils.compare(o1[i], oi1[i], o2[i], oi2[i]); + if (c != 0) { + return c; + } + } + return c; + } + + public static Object[] copyToStandardObject(Object[] o, + ObjectInspector[] oi, + ObjectInspectorCopyOption objectInspectorOption) + { + Object[] out = new Object[o.length]; + for (int i = 0; i < oi.length; i++) + { + out[i] = ObjectInspectorUtils.copyToStandardObject(o[i], oi[i], + objectInspectorOption); + } + return out; + } + + /** + * Copied from Hive ParserDriver. + */ + public static final TreeAdaptor adaptor = new CommonTreeAdaptor() + { + /** + * Creates an ASTNode for the given token. The ASTNode is a wrapper + * around antlr's CommonTree class that implements the Node interface. + * + * @param payload + * The token. + * @return Object (which is actually an ASTNode) for the token. + */ + @Override + public Object create(Token payload) + { + return new ASTNode(payload); + } + }; + + /* + * Utility to visit all nodes in an AST tree. + */ + public static void visit(Object t, ContextVisitor visitor) { + _visit(t, null, 0, visitor); + } + + /** Do the recursive work for visit */ + private static void _visit(Object t, Object parent, int childIndex, ContextVisitor visitor) { + if ( t==null ) { + return; + } + visitor.visit(t, parent, childIndex, null); + int n = adaptor.getChildCount(t); + for (int i=0; iPTF1->PTF2->WINDOWF + * @param qDef + * @throws HiveException + */ + protected void walkInputChain(PTFDef qDef) throws HiveException + { + Iterator it = PTFTranslator.iterateInputDefs(qDef, true); + while(it.hasNext()) + { + PTFInputDef nextDef = it.next(); + if (nextDef instanceof PTFTableOrSubQueryInputDef) + { + visitor.visit((PTFTableOrSubQueryInputDef) nextDef); + } + else if (nextDef instanceof PTFComponentQueryDef) + { + visitor.visit((PTFComponentQueryDef) nextDef); + } + else + { + walk(qDef, (TableFuncDef) nextDef); + } + } + } + + /** + * 1. Reconstruct the InputInfo during previsit + * 2. setup OIs and Evaluators on the ArgDefs + * 3. setup OIs and Evaluators on the columns in the + * PartitionDef, OrderDef and WindowFrameDef in the + * WindowFunction definitions + * 4. walk the functions on the select list to + * setup the OI and GenericUDAFEvaluators + * + * @param qDef + * @param tblFunc + * @throws HiveException + */ + protected void walk(PTFDef qDef, TableFuncDef tblFunc) throws HiveException + { + // 1. visit the Args; these are resolved based on the shape of the Input to the function. + walk(tblFunc.getArgs()); + + // 2. allow visitor to establish input + visitor.preVisit(tblFunc); + + //3. walk the window objects; this resolved based on the rawInputTransformation, if it has happened. + walk(tblFunc.getWindow()); + + //4. for WindowTable Func walk the Window Functions on the Select List + String fName = tblFunc.getName(); + if ( fName.equals(FunctionRegistry.WINDOWING_TABLE_FUNCTION)) + { + SelectDef select = qDef.getSelectList(); + ArrayList wFns = select.getWindowFuncs(); + for(WindowFunctionDef wFn : wFns) + { + walk(wFn.getWindow()); + walk(wFn.getArgs()); + visitor.visit(wFn); + } + } + + //5. revisit tblFunc + visitor.visit(tblFunc); + } + + /** + * Setup the OIs and evaluators on ArgDefs + * @param args + * @throws HiveException + */ + protected void walk(ArrayList args) throws HiveException + { + if ( args != null ) + { + for(ArgDef arg :args) + { + visitor.visit(arg); + } + } + } + + /** + * Visit the partition columns and order columns + * Visit the window frame definitions + * @param window + * @throws HiveException + */ + protected void walk(WindowDef window) throws HiveException + { + if ( window == null ) { + return; + } + + PartitionDef pDef = window.getPartDef(); + if(pDef != null){ + ArrayList cols = pDef.getColumns(); + for(ColumnDef col : cols) + { + visitor.visit(col); + } + visitor.visit(pDef); + } + + + OrderDef oDef = window.getOrderDef(); + if(oDef != null){ + ArrayList ocols = oDef.getColumns(); + for(OrderColumnDef ocol : ocols) + { + visitor.visit(ocol); + } + visitor.visit(oDef); + } + + WindowFrameDef wFrmDef = window.getWindow(); + if ( wFrmDef != null) + { + walk(wFrmDef.getStart()); + walk(wFrmDef.getEnd()); + visitor.visit(wFrmDef); + } + visitor.visit(window); + } + + /** + * Visit all the implementations of + * BoundaryDef + * @param boundary + * @throws HiveException + */ + protected void walk(BoundaryDef boundary) throws HiveException + { + if ( boundary instanceof ValueBoundaryDef ) + { + visitor.visit((ValueBoundaryDef)boundary); + } + else if ( boundary instanceof RangeBoundaryDef) + { + visitor.visit((RangeBoundaryDef)boundary); + } + else if ( boundary instanceof CurrentRowDef) + { + visitor.visit((CurrentRowDef)boundary); + } + } + + /** + * Visit all the columns in the select list + * to setup their OIs and evaluators + * @param select + * @throws HiveException + */ + protected void walk(SelectDef select) throws HiveException + { + ArrayList cols = select.getColumns(); + if ( cols != null ) + { + for(ColumnDef col : cols) + { + visitor.visit(col); + } + } + visitor.visit(select); + } + + } + + /* + * An implementation of the {@link QueryDefVisitor} to reconstruct + * the OIs, serdes and evaluators on the QueryDef. + * This follows the same order used in the translation logic. + */ + public static class PTFDefDeserializer + { + HiveConf hConf; + PTFDef qDef; + PTFInputDef qInDef; + PTFInputInfo inputInfo; + PTFTranslationInfo tInfo; + ObjectInspector inputOI; + TableFunctionResolver currentTFnResolver; + + // TODO get rid of this dependency + static + { + FunctionRegistry.getWindowFunctionInfo("rank"); + } + + public PTFDefDeserializer(HiveConf hc, ObjectInspector inputOI) + { + this.hConf = hc; + this.inputOI = inputOI; + } + + public PTFDefDeserializer(HiveConf hc) + { + this.hConf = hc; + } + + /* + * Create new instance for the translation info and set the hiveConf on it + */ + + public void initialize(PTFDef queryDef) + { + qDef = queryDef; + tInfo = new PTFTranslationInfo(); + tInfo.setHiveCfg(hConf); + qDef.setTranslationInfo(tInfo); + + } + + public void visit(PTFComponentQueryDef compInputDef) throws HiveException + { + this.qInDef = compInputDef; + String serDeClassName = compInputDef.getCompSerdeClassName(); + + try + { + SerDe serDe = (SerDe) SerDeUtils.lookupDeserializer(serDeClassName); + serDe.initialize(hConf, PTFTranslator.buildSerDePropertiesFromOI((StructObjectInspector)inputOI)); + compInputDef.setSerde(serDe); + compInputDef.setOI((StructObjectInspector)inputOI); + } + catch (SerDeException se) + { + throw new HiveException(se); + } + + tInfo.addInput(compInputDef, null); + inputInfo = tInfo.getInputInfo(compInputDef); + + } + + + /* + * 1. Use the passed in InputOI to setup the SerDe and OI for the HiveTableDef. + * The runtime OI maybe different from the translation time oI (missing virtual columns). + * 2. We add the hive table definition to the input + * map on the query translation info. + */ + public void visit(PTFTableOrSubQueryInputDef hiveTable) throws HiveException + { + this.qInDef = hiveTable; + + String serDeClassName = hiveTable.getTableSerdeClassName(); + Properties serDeProps = new Properties(); + Map serdePropsMap = hiveTable.getTableSerdeProps(); + for (String serdeName : serdePropsMap.keySet()) + { + serDeProps.setProperty(serdeName, serdePropsMap.get(serdeName)); + } + + try + { + SerDe serDe = (SerDe) SerDeUtils.lookupDeserializer(serDeClassName); + //serDe.initialize(hConf, serDeProps); + serDe.initialize(hConf, PTFTranslator.buildSerDePropertiesFromOI((StructObjectInspector)inputOI)); + hiveTable.setSerde(serDe); + //hiveTable.setOI((StructObjectInspector) serDe.getObjectInspector()); + hiveTable.setOI((StructObjectInspector)inputOI); + } + catch (SerDeException se) + { + throw new HiveException(se); + } + + tInfo.addInput(hiveTable, null); + inputInfo = tInfo.getInputInfo(hiveTable); + } + + /* + * If the query has a map phase, the inputInfo is retrieved from the map + * output info of the table function definition. This is constructed using + * the map output oi of the table function definition. If the query does not + * have a map phase, the inputInfo is retrieved from the QueryInputDef + * (either HiveTableDef or HiveQueryDef) of the query. + */ + public void preVisit(TableFuncDef tblFuncDef) throws HiveException + { + TableFunctionEvaluator tEval = tblFuncDef.getFunction(); + currentTFnResolver = FunctionRegistry.getTableFunctionResolver(tEval.getTableDef().getName()); + currentTFnResolver.initialize(qDef, tblFuncDef, tEval); + if (tEval.isTransformsRawInput()) + { + currentTFnResolver.initializeRawInputOI(); + inputInfo = qDef.getTranslationInfo().getMapInputInfo(tblFuncDef); + } + else + { + inputInfo = qDef.getTranslationInfo().getInputInfo(qInDef); + } + } + + /* + * 1. Invoke setupOI on the TableFunctionEvaluator + * 2. Setup serde and OI on the table function definition + * 3. Add the table function definition to input map + * on the query translation info + * 4. Reset the inputInfo to the one associated + * with this table function definition. + */ + public void visit(TableFuncDef tblFuncDef) throws HiveException + { + TableFunctionEvaluator tEval = tblFuncDef.getFunction(); + currentTFnResolver.initializeOutputOI(); + try { + PTFTranslator.setupSerdeAndOI(tblFuncDef, qInDef, tInfo, tEval, false); + } + catch(SerDeException se) { + throw new SemanticException(se); + } + tInfo.addInput(tblFuncDef, null); + inputInfo = qDef.getTranslationInfo().getInputInfo(tblFuncDef); + } + + /* + * Recreate the ExprEvaluator, OI using the current inputInfo This is the + * inputInfo on the first InputDef in chain if the query does not have a map + * phase; else it is the mapInputInfo on the table function definition + */ + public void visit(ArgDef arg) throws HiveException + { + ExprNodeDesc exprNodeDesc = arg.getExprNode(); + ExprNodeEvaluator exprEval = WindowingExprNodeEvaluatorFactory.get( + tInfo, exprNodeDesc); + ObjectInspector oi = PTFTranslator.initExprNodeEvaluator(qDef, + exprNodeDesc, exprEval, inputInfo); + + arg.setExprEvaluator(exprEval); + arg.setOI(oi); + + } + + /* + * Recreate ExprNodeEvaluator, OI using InputInfo of first InputDef in + * chain. + */ + public void visit(ColumnDef column) throws HiveException + { + ExprNodeEvaluator exprEval = WindowingExprNodeEvaluatorFactory.get( + tInfo, column.getExprNode()); + ObjectInspector oi = PTFTranslator.initExprNodeEvaluator(qDef, + column.getExprNode(), exprEval, inputInfo); + column.setExprEvaluator(exprEval); + column.setOI(oi); + } + + /* + * Same as visit on {@link ColumnDef} + */ + public void visit(OrderColumnDef column) throws HiveException + { + visit((ColumnDef) column); + } + + /* + * Recreate the ExprEvaluator, OI using the current inputInfo This is the + * inputInfo on the first InputDef in chain if the query does not have a map + * phase; else it is the mapInputInfo on the table function definition + */ + public void visit(ValueBoundaryDef boundary) throws HiveException + { + ExprNodeEvaluator exprEval = WindowingExprNodeEvaluatorFactory.get( + tInfo, boundary.getExprNode()); + ObjectInspector oi = PTFTranslator.initExprNodeEvaluator(qDef, + boundary.getExprNode(), exprEval, inputInfo); + boundary.setExprEvaluator(exprEval); + boundary.setOI(oi); + } + + /* + * Setup the evaluators and OIs - Recreate the GenericUDAFEvaluator and use + * this and OIs on function arguments to reconstruct the OI on the window + * function definition + */ + public void visit(WindowFunctionDef wFn) throws HiveException + { + PTFTranslator.setupEvaluator(wFn); + } + + /* + * Recreate ExprNodeEvaluator, OI using InputInfo of first InputDef in + * chain. + */ + public void visit(WhereDef where) throws HiveException + { + ExprNodeEvaluator exprEval = WindowingExprNodeEvaluatorFactory.get( + tInfo, where.getExprNode()); + ObjectInspector oi = PTFTranslator.initExprNodeEvaluator(qDef, + where.getExprNode(), exprEval, inputInfo); + where.setExprEvaluator(exprEval); + where.setOI(oi); + } + + /* + * Recreate OI on select list. + */ + public void visit(SelectDef select) throws HiveException + { + PTFTranslator.setupSelectRRAndOI(qDef); + } + + public void finish() throws HiveException + { + } + + public void visit(WindowDef window) throws HiveException + { + } + + public void visit(PartitionDef partition) throws HiveException + { + } + + public void visit(OrderDef order) throws HiveException + { + } + + public void visit(WindowFrameDef windowFrame) throws HiveException + { + } + + public void visit(CurrentRowDef boundary) throws HiveException + { + } + + public void visit(RangeBoundaryDef boundary) throws HiveException + { + } + + } + + +} + diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java index 9bd1c92..8f98f2c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java @@ -457,4 +457,44 @@ public ASTNode parse(String command, Context ctx) throws ParseException { return (ASTNode) r.getTree(); } + + + /* + * parse a String as a Select List. This allows table functions to be passed expression Strings + * that are translated in + * the context they define at invocation time. Currently used by NPath to allow users to specify + * what output they want. + * NPath allows expressions n 'tpath' a column that represents the matched set of rows. This + * column doesn't exist in + * the input schema and hence the Result Expression cannot be analyzed by the regular Hive + * translation process. + */ + public ASTNode parseSelect(String command, Context ctx) throws ParseException { + LOG.info("Parsing command: " + command); + + HiveLexerX lexer = new HiveLexerX(new ANTLRNoCaseStringStream(command)); + TokenRewriteStream tokens = new TokenRewriteStream(lexer); + if (ctx != null) { + ctx.setTokenRewriteStream(tokens); + } + HiveParserX parser = new HiveParserX(tokens); + parser.setTreeAdaptor(adaptor); + HiveParser.selectClause_return r = null; + try { + r = parser.selectClause(); + } catch (RecognitionException e) { + e.printStackTrace(); + throw new ParseException(parser.getErrors()); + } + + if (lexer.getErrors().size() == 0 && parser.getErrors().size() == 0) { + LOG.info("Parse Completed"); + } else if (lexer.getErrors().size() != 0) { + throw new ParseException(lexer.getErrors()); + } else { + throw new ParseException(parser.getErrors()); + } + + return (ASTNode) r.getTree(); + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java index 7dd0426..02bc364 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java @@ -52,6 +52,16 @@ private boolean isAnalyzeRewrite; private CreateTableDesc tblDesc = null; // table descriptor of the final + // used by PTFs + /* + * This map maintains the PTFSpec for each PTF chain invocation in this QB. + */ + private HashMap ptfNodeToSpec; + /* + * the PTFSpec used for windowing clauses in this QB. + */ + private HashMap destToWindowingPTFSpec; + // results public void print(String msg) { @@ -76,6 +86,8 @@ public QB(String outer_id, String alias, boolean isSubQ) { } qbp = new QBParseInfo(alias, isSubQ); qbm = new QBMetaData(); + ptfNodeToSpec = new HashMap(); + destToWindowingPTFSpec = new HashMap(); id = getAppendedAliasFromId(outer_id, alias); } @@ -246,4 +258,35 @@ public boolean isAnalyzeRewrite() { public void setAnalyzeRewrite(boolean isAnalyzeRewrite) { this.isAnalyzeRewrite = isAnalyzeRewrite; } -} + + public PTFSpec getPtfSpec(ASTNode node) { + return ptfNodeToSpec == null ? null : ptfNodeToSpec.get(node); + } + + public void addPTFNodeToSpec(ASTNode node, PTFSpec spec) { + ptfNodeToSpec = ptfNodeToSpec == null ? new HashMap() : ptfNodeToSpec; + ptfNodeToSpec.put(node, spec); + } + + public HashMap getPTFNodeToSpec() { + return ptfNodeToSpec; + } + + public PTFSpec getWindowingPTFSpec(String dest) { + return destToWindowingPTFSpec.get(dest); + } + + public void addDestToWindowingPTFSpec(String dest, PTFSpec windowingPTFSpec) { + destToWindowingPTFSpec.put(dest, windowingPTFSpec); + } + + public boolean hasWindowingPTFSpec(String dest) { + return destToWindowingPTFSpec.get(dest) != null; + } + + public HashMap getAllWindowingPTFSpecs() { + return destToWindowingPTFSpec; + } + + +} \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java index b317f49..3e57b10 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java @@ -65,6 +65,7 @@ private String partName; // used for column statistics private boolean isTblLvl; // used for column statistics + /** * ClusterBy is a short name for both DistributeBy and SortBy. */ @@ -95,6 +96,10 @@ private final LinkedHashMap> destToAggregationExprs; private final HashMap> destToDistinctFuncExprs; + // used by Windowing + private final LinkedHashMap> destToWindowingExprs; + + @SuppressWarnings("unused") private static final Log LOG = LogFactory.getLog(QBParseInfo.class.getName()); @@ -118,6 +123,7 @@ public QBParseInfo(String alias, boolean isSubQ) { destGroupingSets = new HashSet(); destToAggregationExprs = new LinkedHashMap>(); + destToWindowingExprs = new LinkedHashMap>(); destToDistinctFuncExprs = new HashMap>(); this.alias = alias; @@ -130,6 +136,14 @@ public QBParseInfo(String alias, boolean isSubQ) { } + /* + * If a QB is such that the aggregation expressions need to be handled by + * the Windowing PTF; we invoke this function to clear the AggExprs on the dest. + */ + public void clearAggregationExprsForClause(String clause) { + destToAggregationExprs.get(clause).clear(); + } + public void setAggregationExprsForClause(String clause, LinkedHashMap aggregationTrees) { destToAggregationExprs.put(clause, aggregationTrees); @@ -157,6 +171,26 @@ public boolean isInsertIntoTable(String dbName, String table) { return destToAggregationExprs.get(clause); } + public void addWindowingExprsForClause(String clause, + LinkedHashMap windowingTrees) { + if (destToWindowingExprs.containsKey(clause)) { + destToWindowingExprs.get(clause).putAll(windowingTrees); + } else { + destToWindowingExprs.put(clause, windowingTrees); + } + } + + public HashMap getWindowingExprsForClause(String clause) { + return destToWindowingExprs.get(clause); + } + + public void clearDistinctFuncExprsForClause(String clause) { + List l = destToDistinctFuncExprs.get(clause); + if ( l != null ) { + l.clear(); + } + } + public void setDistinctFuncExprsForClause(String clause, List ast) { destToDistinctFuncExprs.put(clause, ast); } @@ -563,3 +597,5 @@ public void setColType(List colType) { this.colType = colType; } } + + diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 97c7038..aeec893 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -29,12 +30,15 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.Stack; import java.util.TreeSet; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; import org.antlr.runtime.tree.BaseTree; import org.antlr.runtime.tree.Tree; +import org.antlr.runtime.tree.TreeWizard; +import org.antlr.runtime.tree.TreeWizard.ContextVisitor; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.Path; @@ -68,6 +72,8 @@ import org.apache.hadoop.hive.ql.exec.MapRedTask; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.OperatorFactory; +import org.apache.hadoop.hive.ql.exec.PTFOperator; +import org.apache.hadoop.hive.ql.exec.PTFUtils; import org.apache.hadoop.hive.ql.exec.RecordReader; import org.apache.hadoop.hive.ql.exec.RecordWriter; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; @@ -115,6 +121,21 @@ import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalOptimizer; import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.tableSpec.SpecType; +import org.apache.hadoop.hive.ql.parse.PTFSpec.ColumnSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.OrderColumnSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.OrderSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.PTFInputSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.PTFTableOrSubQueryInputSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.PartitionSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.SelectSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.TableFuncSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowFrameSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowFrameSpec.BoundarySpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowFrameSpec.Direction; +import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowFrameSpec.RangeBoundarySpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowFrameSpec.ValueBoundarySpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowFunctionSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowSpec; import org.apache.hadoop.hive.ql.plan.AggregationDesc; import org.apache.hadoop.hive.ql.plan.ColumnStatsDesc; import org.apache.hadoop.hive.ql.plan.ColumnStatsWork; @@ -149,6 +170,14 @@ import org.apache.hadoop.hive.ql.plan.MapredWork; import org.apache.hadoop.hive.ql.plan.MoveWork; import org.apache.hadoop.hive.ql.plan.OperatorDesc; +import org.apache.hadoop.hive.ql.plan.PTFDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.ColumnDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.OrderColumnDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.PTFComponentQueryDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.PTFInputDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.PTFTableOrSubQueryInputDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.TableFuncDef; +import org.apache.hadoop.hive.ql.plan.PTFDesc; import org.apache.hadoop.hive.ql.plan.PlanUtils; import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; import org.apache.hadoop.hive.ql.plan.ScriptDesc; @@ -164,6 +193,7 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFHash; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr; import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF; +import org.apache.hadoop.hive.ql.udf.ptf.TableFunctionEvaluator; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe; @@ -260,6 +290,7 @@ public SemanticAnalyzer(HiveConf conf) throws SemanticException { HiveConf.ConfVars.HIVE_AUTOGEN_COLUMNALIAS_PREFIX_INCLUDEFUNCNAME); queryProperties = new QueryProperties(); opToPartToSkewedPruner = new HashMap>(); + destinationNodeToPTFChainNode = new HashMap(); } @Override @@ -319,7 +350,13 @@ public void doPhase1QBExpr(ASTNode ast, QBExpr qbexpr, String id, String alias) switch (ast.getToken().getType()) { case HiveParser.TOK_QUERY: { QB qb = new QB(id, alias, true); - doPhase1(ast, qb, initPhase1Ctx()); + Phase1Ctx ctx_1 = initPhase1Ctx(); + doPhase1(ast, qb, ctx_1); + ensurePTFSpecOnQB(qb, ctx_1.dest); + ensurePTFChainHasPartitioning(qb.getWindowingPTFSpec(ctx_1.dest), + generateErrorMessage(ast, + "No partition specification associated with start of PTF chain ")); + qbexpr.setOpcode(QBExpr.Opcode.NULLOP); qbexpr.setQB(qb); } @@ -345,13 +382,19 @@ public void doPhase1QBExpr(ASTNode ast, QBExpr qbexpr, String id, String alias) } private LinkedHashMap doPhase1GetAggregationsFromSelect( - ASTNode selExpr) { + ASTNode selExpr, QB qb, String dest) { // Iterate over the selects search for aggregation Trees. // Use String as keys to eliminate duplicate trees. LinkedHashMap aggregationTrees = new LinkedHashMap(); for (int i = 0; i < selExpr.getChildCount(); ++i) { - ASTNode sel = (ASTNode) selExpr.getChild(i).getChild(0); - doPhase1GetAllAggregations(sel, aggregationTrees); + ASTNode sel = (ASTNode) selExpr.getChild(i); + if(queryProperties.hasPTF() && qb.getWindowingPTFSpec(dest) != null ){ + PTFSpec qSpec = qb.getWindowingPTFSpec(dest); + if(qSpec.getSelectList().getAliasToAST().values().contains(sel.getChild(0))){ + continue; + } + } + doPhase1GetAllAggregations((ASTNode) sel.getChild(0), aggregationTrees); } return aggregationTrees; } @@ -637,6 +680,16 @@ private void processJoin(QB qb, ASTNode join) throws SemanticException { processTable(qb, child); } else if (child.getToken().getType() == HiveParser.TOK_SUBQUERY) { processSubQuery(qb, child); + } else if (child.getToken().getType() == HiveParser.TOK_PTBLFUNCTION) { + queryProperties.setHasPTF(true); + processPTF(qb, child); + PTFSpec ptfSpec = qb.getPtfSpec(child); + String inputAlias = ptfSpec == null ? null : ((TableFuncSpec)ptfSpec.getInput()).getAlias();; + if ( inputAlias == null ) { + throw new SemanticException(generateErrorMessage(child, + "PTF invocation in a Join must have an alias")); + } + } else if (child.getToken().getType() == HiveParser.TOK_LATERAL_VIEW) { // SELECT * FROM src1 LATERAL VIEW udtf() AS myTable JOIN src2 ... // is not supported. Instead, the lateral view must be in a subquery @@ -728,7 +781,9 @@ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1) qbp.setHints((ASTNode) ast.getChild(0)); } - LinkedHashMap aggregations = doPhase1GetAggregationsFromSelect(ast); + handleWindowingExprsInSelectList(qb, ctx_1, ast); + + LinkedHashMap aggregations = doPhase1GetAggregationsFromSelect(ast, qb, ctx_1.dest); doPhase1GetColumnAliasesFromSelect(ast, qbp); qbp.setAggregationExprsForClause(ctx_1.dest, aggregations); qbp.setDistinctFuncExprsForClause(ctx_1.dest, @@ -759,6 +814,18 @@ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1) } qbp.setDestForClause(ctx_1.dest, (ASTNode) ast.getChild(0)); + + /* + * If this node has an associated ptf; + * then set the QuerySpec of the QB to the QSpec of the ptf. + */ + ASTNode ptfNode = getAssociatedPTFNode(ast); + if ( ptfNode != null ) + { + PTFSpec qSpec = qb.getPtfSpec(ptfNode); + qb.addDestToWindowingPTFSpec(ctx_1.dest, qSpec); + } + break; case HiveParser.TOK_FROM: @@ -780,22 +847,37 @@ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1) queryProperties.setHasJoin(true); processJoin(qb, frm); qbp.setJoinExpr(frm); + }else if(frm.getToken().getType() == HiveParser.TOK_PTBLFUNCTION){ + queryProperties.setHasPTF(true); + processPTF(qb, frm); } break; case HiveParser.TOK_CLUSTERBY: - // Get the clusterby aliases - these are aliased to the entries in the - // select list - queryProperties.setHasClusterBy(true); - qbp.setClusterByExprForClause(ctx_1.dest, ast); + + // if there is no GroupBy Clause, but there are still Agg Exprs remaining in the select list + // - move them to qb.windowingQSpec. + if ( qbp.getAggregationExprsForClause(ctx_1.dest) != null && + !queryProperties.hasGroupBy()) { + moveaggregationExprsToPTFSpec(qb, ctx_1.dest); + } + + // if QB has WindowingClauses + // - associate DistributeBy with qb.windowingQSpec + if (qb.hasWindowingPTFSpec(ctx_1.dest) ) + { + handleClusterOrDistributeByForWindowing(qb, ctx_1, ast); + } + else { + // Get the clusterby aliases - these are aliased to the entries in the + // select list + queryProperties.setHasClusterBy(true); + qbp.setClusterByExprForClause(ctx_1.dest, ast); + } break; case HiveParser.TOK_DISTRIBUTEBY: - // Get the distribute by aliases - these are aliased to the entries in - // the - // select list - queryProperties.setHasDistributeBy(true); - qbp.setDistributeByExprForClause(ctx_1.dest, ast); + if (qbp.getClusterByForClause(ctx_1.dest) != null) { throw new SemanticException(generateErrorMessage(ast, ErrorMsg.CLUSTERBY_DISTRIBUTEBY_CONFLICT.getMsg())); @@ -803,13 +885,32 @@ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1) throw new SemanticException(generateErrorMessage(ast, ErrorMsg.ORDERBY_DISTRIBUTEBY_CONFLICT.getMsg())); } + + // if there is no GroupBy Clause, but there are still Agg Exprs remaining in the select list + // - move them to qb.windowingQSpec. + if ( qbp.getAggregationExprsForClause(ctx_1.dest) != null && + !queryProperties.hasGroupBy()) { + moveaggregationExprsToPTFSpec(qb, ctx_1.dest); + } + + // if QB has WindowingClauses + // - associate DistributeBy with qb.windowingQSpec + if (qb.hasWindowingPTFSpec(ctx_1.dest) ) + { + handleClusterOrDistributeByForWindowing(qb, ctx_1, ast); + } + else { + + // Get the distribute by aliases - these are aliased to the entries in + // the + // select list + queryProperties.setHasDistributeBy(true); + qbp.setDistributeByExprForClause(ctx_1.dest, ast); + } break; case HiveParser.TOK_SORTBY: - // Get the sort by aliases - these are aliased to the entries in the - // select list - queryProperties.setHasSortBy(true); - qbp.setSortByExprForClause(ctx_1.dest, ast); + if (qbp.getClusterByForClause(ctx_1.dest) != null) { throw new SemanticException(generateErrorMessage(ast, ErrorMsg.CLUSTERBY_SORTBY_CONFLICT.getMsg())); @@ -818,6 +919,19 @@ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1) ErrorMsg.ORDERBY_SORTBY_CONFLICT.getMsg())); } + // if QB has WindowingClauses + // - associate SortBy with qb.windowingSpec. + if (qb.hasWindowingPTFSpec(ctx_1.dest) ) + { + handleSortByForWindowing(qb, ctx_1, ast); + } + else { + // Get the sort by aliases - these are aliased to the entries in the + // select list + queryProperties.setHasSortBy(true); + qbp.setSortByExprForClause(ctx_1.dest, ast); + } + break; case HiveParser.TOK_ORDERBY: @@ -859,8 +973,36 @@ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1) break; case HiveParser.TOK_HAVING: - qbp.setHavingExprForClause(ctx_1.dest, ast); - qbp.addAggregationExprsForClause(ctx_1.dest, doPhase1GetAggregationsFromSelect(ast)); + boolean hasLLArgs = false; + TreeWizard tw = new TreeWizard(ParseDriver.adaptor, HiveParser.tokenNames); + CheckLeadLagInSelectExprs checkLLFunctions = new CheckLeadLagInSelectExprs(); + tw.visit(ast, HiveParser.TOK_FUNCTION, checkLLFunctions); + if ( checkLLFunctions.isError() ) { + throw new SemanticException(generateErrorMessage(ast, checkLLFunctions.getErrString())); + } + hasLLArgs = checkLLFunctions.isHasWindowingExprs(); + + // if QB has WindowingClauses and no GroupBy clause + // - associate Having with qb.windowingSpec. + if (qb.hasWindowingPTFSpec(ctx_1.dest) && qbp.getGroupByForClause(ctx_1.dest) == null) + { + handleWindowingHavingClause(qb, ctx_1, ast); + } else if(hasLLArgs){ + throw new SemanticException(generateErrorMessage(ast, + "Query has no windowing or group by clause: " + + "Unsupported place for having")); + } else{ + qbp.setHavingExprForClause(ctx_1.dest, ast); + qbp.addAggregationExprsForClause(ctx_1.dest, doPhase1GetAggregationsFromSelect(ast, qb, ctx_1.dest)); + } + break; + + case HiveParser.KW_WINDOW: + if (!qb.hasWindowingPTFSpec(ctx_1.dest) ) { + throw new SemanticException(generateErrorMessage(ast, + "Query has no Cluster/Distribute By; but has a Window definition")); + } + handleQueryWindowClauses(qb, ctx_1, ast); break; case HiveParser.TOK_LIMIT: @@ -2157,12 +2299,20 @@ private int setBit(int bitmap, int bitIdx) { List result = new ArrayList(selectExprs == null ? 0 : selectExprs.getChildCount()); if (selectExprs != null) { + HashMap windowingExprs = parseInfo.getWindowingExprsForClause(dest); + for (int i = 0; i < selectExprs.getChildCount(); ++i) { if (((ASTNode) selectExprs.getChild(i)).getToken().getType() == HiveParser.TOK_HINTLIST) { continue; } // table.column AS alias ASTNode grpbyExpr = (ASTNode) selectExprs.getChild(i).getChild(0); + /* + * If this is handled by Windowing then ignore it. + */ + if (windowingExprs != null && windowingExprs.containsKey(grpbyExpr.toStringTree())) { + continue; + } result.add(grpbyExpr); } } @@ -2189,7 +2339,9 @@ private int setBit(int bitmap, int bitIdx) { String tabAlias = null; String[] colRef = new String[2]; - if (selExpr.getChildCount() == 2) { + //for queries with a windowing expressions, the selexpr may have a third child + if (selExpr.getChildCount() == 2 || + (selExpr.getChildCount() == 3 && selExpr.getChild(2).getType() == HiveParser.TOK_WINDOWSPEC)) { // return zz for "xx + yy AS zz" colAlias = unescapeIdentifier(selExpr.getChild(1).getText()); colRef[0] = tabAlias; @@ -2266,6 +2418,7 @@ private static boolean isRegex(String pattern) { return false; } + private Operator genSelectPlan(String dest, QB qb, Operator input) throws SemanticException { ASTNode selExprList = qb.getParseInfo().getSelForClause(dest); @@ -2403,11 +2556,14 @@ private static boolean isRegex(String pattern) { // child can be EXPR AS ALIAS, or EXPR. ASTNode child = (ASTNode) exprList.getChild(i); boolean hasAsClause = (!isInTransform) && (child.getChildCount() == 2); + boolean isWindowSpec = child.getChildCount() == 3 ? + (child.getChild(2).getType() == HiveParser.TOK_WINDOWSPEC) : + false; // EXPR AS (ALIAS,...) parses, but is only allowed for UDTF's // This check is not needed and invalid when there is a transform b/c the // AST's are slightly different. - if (!isInTransform && !isUDTF && child.getChildCount() > 2) { + if (!isWindowSpec && !isInTransform && !isUDTF && child.getChildCount() > 2) { throw new SemanticException(generateErrorMessage( (ASTNode) child.getChild(2), ErrorMsg.INVALID_AS.getMsg())); @@ -6232,12 +6388,18 @@ private QBJoinTree genJoinTree(QB qb, ASTNode joinParseTree, ASTNode right = (ASTNode) joinParseTree.getChild(1); if ((left.getToken().getType() == HiveParser.TOK_TABREF) - || (left.getToken().getType() == HiveParser.TOK_SUBQUERY)) { + || (left.getToken().getType() == HiveParser.TOK_SUBQUERY) + || (left.getToken().getType() == HiveParser.TOK_PTBLFUNCTION)) { String tableName = getUnescapedUnqualifiedTableName((ASTNode) left.getChild(0)) .toLowerCase(); String alias = left.getChildCount() == 1 ? tableName : unescapeIdentifier(left.getChild(left.getChildCount() - 1) - .getText().toLowerCase()); + .getText().toLowerCase()); + // ptf node form is: ^(TOK_PTBLFUNCTION $name $alias? partitionTableFunctionSource partitioningSpec? expression*) + // guranteed to have an lias here: check done in processJoin + alias = (left.getToken().getType() == HiveParser.TOK_PTBLFUNCTION) ? + unescapeIdentifier(left.getChild(1).getText().toLowerCase()) : + alias; joinTree.setLeftAlias(alias); String[] leftAliases = new String[1]; leftAliases[0] = alias; @@ -6263,12 +6425,18 @@ private QBJoinTree genJoinTree(QB qb, ASTNode joinParseTree, } if ((right.getToken().getType() == HiveParser.TOK_TABREF) - || (right.getToken().getType() == HiveParser.TOK_SUBQUERY)) { + || (right.getToken().getType() == HiveParser.TOK_SUBQUERY) + || (right.getToken().getType() == HiveParser.TOK_PTBLFUNCTION)) { String tableName = getUnescapedUnqualifiedTableName((ASTNode) right.getChild(0)) .toLowerCase(); String alias = right.getChildCount() == 1 ? tableName : unescapeIdentifier(right.getChild(right.getChildCount() - 1) - .getText().toLowerCase()); + .getText().toLowerCase()); + // ptf node form is: ^(TOK_PTBLFUNCTION $name $alias? partitionTableFunctionSource partitioningSpec? expression*) + // guranteed to have an lias here: check done in processJoin + alias = (right.getToken().getType() == HiveParser.TOK_PTBLFUNCTION) ? + unescapeIdentifier(right.getChild(1).getText().toLowerCase()) : + alias; String[] rightAliases = new String[1]; rightAliases[0] = alias; joinTree.setRightAliases(rightAliases); @@ -7051,6 +7219,12 @@ private Operator genPostGroupByBodyPlan(Operator curr, String dest, QB qb) curr = genHavingPlan(dest, qb, curr); } + + if(queryProperties.hasPTF() && qb.getWindowingPTFSpec(dest) != null) { + //TODO: create a select * operator here + curr = genPTFPlan(qb.getWindowingPTFSpec(dest), curr); + } + curr = genSelectPlan(dest, qb, curr); Integer limit = qbp.getDestLimit(dest); @@ -7090,6 +7264,7 @@ private Operator genPostGroupByBodyPlan(Operator curr, String dest, QB qb) curr = genReduceSinkPlan(dest, qb, curr, numReducers); } + if (qbp.getIsSubQ()) { if (limit != null) { // In case of order by, only 1 reducer is used, so no need of @@ -7735,11 +7910,43 @@ public Operator genPlan(QB qb) throws SemanticException { aliasToOpInfo.put(alias, op); } + Operator srcOpInfo = null; + Operator lastPTFOp = null; + + if(queryProperties.hasPTF()){ + //After processing subqueries and source tables, process + // partitioned table functions + + HashMap ptfNodeToSpec = qb.getPTFNodeToSpec(); + if ( ptfNodeToSpec != null ) { + for(Map.Entry entry : ptfNodeToSpec.entrySet()) { + ASTNode ast = entry.getKey(); + PTFSpec spec = entry.getValue(); + if ( qb.getAllWindowingPTFSpecs().containsValue(spec) ) { + // this ptf invocation will be handled when we get to Windowing Clauses handling + // in the genPostGroupByBodyPlan + continue; + } + String inputAlias = PTFTranslator.getHiveTableSpec(spec).getTableName(); + Operator inOp = aliasToOpInfo.get(inputAlias); + if ( inOp == null ) { + throw new SemanticException(generateErrorMessage(ast, + "Cannot resolve input Operator for PTF invocation")); + } + lastPTFOp = genPTFPlan(spec, inOp); + String ptfAlias = ((TableFuncSpec)spec.getInput()).getAlias(); + if ( ptfAlias != null ) { + aliasToOpInfo.put(ptfAlias, lastPTFOp); + } + } + } + + } + // For all the source tables that have a lateral view, attach the // appropriate operators to the TS genLateralViewPlans(aliasToOpInfo, qb); - Operator srcOpInfo = null; // process join if (qb.getParseInfo().getJoinExpr() != null) { @@ -7762,6 +7969,8 @@ public Operator genPlan(QB qb) throws SemanticException { // Now if there are more than 1 sources then we have a join case // later we can extend this to the union all case as well srcOpInfo = aliasToOpInfo.values().iterator().next(); + // with ptfs, there maybe more (note for PTFChains: 1 ptf invocation may entail multiple PTF operators) + srcOpInfo = lastPTFOp != null ? lastPTFOp : srcOpInfo; } Operator bodyOpInfo = genBodyPlan(qb, srcOpInfo); @@ -8420,10 +8629,16 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { } // continue analyzing from the child ASTNode. - if (!doPhase1(child, qb, initPhase1Ctx())) { + associateInsertNodesToPTFChains(child); + Phase1Ctx ctx_1 = initPhase1Ctx(); + if (!doPhase1(child, qb, ctx_1)) { // if phase1Result false return return; } + ensurePTFSpecOnQB(qb, ctx_1.dest); + ensurePTFChainHasPartitioning(qb.getWindowingPTFSpec(ctx_1.dest), + generateErrorMessage(ast, + "No partition specification associated with start of PTF chain ")); LOG.info("Completed phase 1 of Semantic Analysis"); @@ -8436,6 +8651,8 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { // up with later. Operator sinkOp = genPlan(qb); + dumpOperatorChain(sinkOp, null); + resultSchema = convertRowSchemaToViewSchema(opParseCtx.get(sinkOp).getRowResolver()); @@ -9446,4 +9663,1442 @@ public QB getQB() { public void setQB(QB qb) { this.qb = qb; } + +//--------------------------- PTF handling ----------------------------------- + + /* + * If the from Clause: + * - contains only a PTF invocation + * - there is no GBy or where clause + * - and there is only 1 insert clause + * then: + * - we can process any Window Expressions in the SelectList as part of the same + * PTF chain as the chain in the from clause. + * Any Windowing specifications on the Select list will be handled as an invocation + * on the internal WindowingTableFunction PTF added to the end of the PTF chain. + */ + private final HashMap destinationNodeToPTFChainNode; + + private HashMap getDestinationNodeToPTFChainNodeMap() { + return destinationNodeToPTFChainNode; + } + + private ASTNode getAssociatedPTFNode(ASTNode destinationNode) { + return destinationNodeToPTFChainNode.get(destinationNode); + } + + private void associateDestinationNodeToPTFNode(ASTNode iNode, ASTNode pNode) { + destinationNodeToPTFChainNode.put(iNode, pNode); + } + + /* + * - called before starting Phase1. + * - association maintained by SemanticAnalyzer. + * - @revisit: what about views? + */ + private void associateInsertNodesToPTFChains(ASTNode node) { + TreeWizard tw = new TreeWizard(ParseDriver.adaptor, HiveParser.tokenNames); + tw.visit(node, HiveParser.TOK_QUERY, new AssociateInsertAndPTFNodes()); + } + + /* + * If the from Clause: + * - contains only a PTF invocation + * - there is no GBy or where clause + * - and there is only 1 insert clause + * then: + * - we can process any Window Expressions in the SelectList as part of the same + * PTF chain as the chain in the from clause. + * Any Windowing specifications on the Select list will be handled as an invocation + * on the internal WindowingTableFunction PTF added to the end of the PTF chain. + * + * AST pattern we are looking for is: + * ^(TOK_QUERY ^(TOK_FROM ^(TOK_PTBLFUNCTION...)) ^(TOK_INSERT {a single destination})) + * AST structure for an INSERT tree is: + * ^(TOK_INSERT insertClause + selectClause whereClause? groupByClause? havingClause? orderByClause? clusterByClause? + distributeByClause? sortByClause? window_clause? limitClause?) + */ + public class AssociateInsertAndPTFNodes implements ContextVisitor { + + @SuppressWarnings("rawtypes") + @Override + public void visit(Object t, Object parent, int childIndex, Map labels) { + ASTNode queryNode = (ASTNode) t; + + ASTNode fromClause = (ASTNode) queryNode.getChild(0); + ASTNode insertClause = (ASTNode) queryNode.getChild(1); + int numInserts = queryNode.getChildCount() - 1; + + ASTNode fromSource = (ASTNode) fromClause.getChild(0); + + if ( numInserts > 1 ) { + return; + } + + if ( fromSource.getType() != HiveParser.TOK_PTBLFUNCTION) { + return; + } + + int insertChildCount = insertClause.getChildCount(); + boolean hasGBy = false, hasWhere = false; + for(int i=1; i inputIdx ? (ASTNode) ptf.getChild(pSpecIdx) : null; + if (pSpecNode != null && pSpecNode.getType() == HiveParser.TOK_PARTITIONINGSPEC) + { + WindowSpec wSpec = new WindowSpec(); + processPTFPartitionSpec(pSpecNode, wSpec); + ptfSpec.setPartition(wSpec.getPartition()); + ptfSpec.setOrder(wSpec.getOrder()); + argStartIdx++; + } + + /* + * arguments + */ + for(int i=argStartIdx; i < ptf.getChildCount(); i++) + { + ptfSpec.addArg((ASTNode) ptf.getChild(i)); + } + + return ptfSpec; + } + + /* + * - invoked during FROM AST tree processing, on encountering a PTF invocation. + * - tree form is ^(TOK_PTBLFUNCTION name partitionTableFunctionSource partitioningSpec? arguments*) + * - setup a QuerySpec for this top level PTF invocation. + * - if the Query needs Windowing handling, then when encountering the associated DESTINATION AST node, + * this QuerySpec will be associated with the QB. + */ + private void processPTF(QB qb, ASTNode ptf) throws SemanticException{ + + TableFuncSpec ptfSpec = processPTFChain(qb, ptf); + + if ( ptfSpec.getAlias() != null ) { + qb.addAlias(ptfSpec.getAlias()); + } + + PTFSpec spec = new PTFSpec(); + spec.setInput(ptfSpec); + ensurePTFChainHasPartitioning(spec, generateErrorMessage(ptf, + "No partition specification associated with start of PTF chain ")); + qb.addPTFNodeToSpec(ptf, spec); + } + + /* + * ensure that the PTF chain has a partitioning specification associated. This method + * should be called when: + * - a PTF chain is encountered as a fromSource. (from the processPTF method) + * - at the end of Phase1 (or before constructing the PTF QueryDef) to check that the WindowingTableFunction + * is driven from a partitioning specification (specified in the distribute by or cluster by clauses). + */ + private void ensurePTFChainHasPartitioning(PTFSpec qSpec, String errString) throws SemanticException { + if(qSpec == null){ + return; + } + Iterator it = PTFTranslator.iterateTableFuncSpecs(qSpec, true); + PTFInputSpec iSpec = it.next(); + PartitionSpec pSpec = iSpec.getPartition(); + if ( pSpec == null ) { + throw new SemanticException(errString); + } + } + + /* + * - Select tree form is: ^(TOK_SELECT ^(TOK_SELECTEXPR...) ^(TOK_SELECTEXPR...) ...) + * - A Select Item form is: ^(TOK_SELEXPR selectExpression Identifier* window_specification?) + * + * We need to extract from the SelectList any SelectItems that must be handled during Windowing processing. These are: + * - SelectItems that have a window_specification + * - SelectItems that invoke row navigation functions: Lead/Lag. + + */ + private void findWindowingExpressions(ASTNode selectNode, + ArrayList windowingFunctionExprs, + ArrayList windowingFunctionAliases, + ArrayList windowingFunctionWSpecs, + ArrayList windowingSelectExprs, + ArrayList windowingSelectExprAliases) + throws SemanticException { + TreeWizard tw = new TreeWizard(ParseDriver.adaptor, HiveParser.tokenNames); + CheckLeadLagInSelectExprs checkLLFunctions = new CheckLeadLagInSelectExprs(); + + + for(int i=0; i < selectNode.getChildCount(); i++) + { + ASTNode selectExpr = (ASTNode) selectNode.getChild(i); + if ( selectExpr.getType() != HiveParser.TOK_SELEXPR ) + { + continue; + } + boolean hasWindowingExprs = checkAndExtractIsWindowingFunctionInvocation(selectExpr, + windowingFunctionExprs, + windowingFunctionAliases, + windowingFunctionWSpecs); + + if ( !hasWindowingExprs ) { + tw.visit(selectExpr, HiveParser.TOK_FUNCTION, checkLLFunctions); + + if ( checkLLFunctions.isError() ) { + throw new SemanticException(generateErrorMessage(selectExpr, checkLLFunctions.getErrString())); + } + + hasWindowingExprs = checkLLFunctions.isHasWindowingExprs(); + + if ( hasWindowingExprs ) + { + ASTNode expr = (ASTNode)selectExpr.getChild(0); + windowingSelectExprs.add(expr); + String alias; + int childCount = selectExpr.getChildCount(); + /* + * @revisit what if there are multiple Identifiers(lateral view) + */ + if ( childCount < 2 ) { + /* + * generate an alias using the function AST tree. + * - calling getColAlias with a null InputResolver and null defaultName. + * - so includeFunctionName should be true. + */ + String[] colAlias = getColAlias(expr, null, null, true, -1); + alias = colAlias[1]; + } + else { + alias = selectExpr.getChild(1).getText(); + } + windowingSelectExprAliases.add(alias); + } + } + } + } + + private static class CheckLeadLagInSelectExprs implements ContextVisitor + { + boolean hasWindowingExprs = false; + boolean error = false; + String errString; + + public void visit(Object t, Object parent, int childIndex, Map labels) + { + error = false; errString = null; + + ASTNode function = (ASTNode) t; + String fnName = function.getChild(0).getText().toLowerCase(); + if (fnName.equals(FunctionRegistry.LEAD_FUNC_NAME) + || fnName.equals(FunctionRegistry.LAG_FUNC_NAME)) + { + hasWindowingExprs = true; + } + else if ( FunctionRegistry.NAVIGATION_FUNCTIONS.contains(fnName)) { + error = true; + errString = "Currently you cannot use a Navigation Functions: first_value, last_value in expressions"; + } + else if ( FunctionRegistry.RANKING_FUNCTIONS.contains(fnName)) { + error = true; + errString = "Currently you cannot use a Ranking Functions: in expressions"; + } + } + + public boolean isHasWindowingExprs() { + return hasWindowingExprs; + } + + protected boolean isError() { + return error; + } + + protected String getErrString() { + return errString; + } + } + + /* + * - A Select Item form is: ^(TOK_SELEXPR selectExpression Identifier* window_specification?) + * What constitutes a Windowing Select Expression: + * 1. + * - It must be a UDAF function invocation + * - It must have a Windowing Spec + * 2. + * - It must be a Ranking or Navigation Function invocation which is also a UDAF function (so no lead/lag) + * 3. + * - It must be a UDAF function invocation + * - must have an argument which invokes Lead/Lag + * Error Expressions are: + * - a Window Spec without a UDAF invocation: we don't support expressions i Windowing. + */ + private boolean checkAndExtractIsWindowingFunctionInvocation(ASTNode selectExpr, + ArrayList windowingFunctionExprs, + ArrayList windowingFunctionAliases, + ArrayList windowingFunctionWSpecs) throws SemanticException { + + int childCount = selectExpr.getChildCount(); + ASTNode windowSpec = (ASTNode) selectExpr.getChild(childCount - 1); + + boolean hasWindowSpec = windowSpec.getType() == HiveParser.TOK_WINDOWSPEC; + ASTNode function = (ASTNode) selectExpr.getChild(0); + boolean isFunction = ( function.getType() == HiveParser.TOK_FUNCTION || + function.getType() == HiveParser.TOK_FUNCTIONDI || + function.getType() == HiveParser.TOK_FUNCTIONSTAR ); + String fnName = isFunction ? function.getChild(0).getText().toLowerCase() : null; + boolean isWindowFunction = isFunction ? + FunctionRegistry.isWindowFunction(fnName) : false; + boolean isRankingOrNavFunction = !isFunction ? false : + ( + FunctionRegistry.RANKING_FUNCTIONS.contains(fnName) || + FunctionRegistry.NAVIGATION_FUNCTIONS.contains(fnName) + ); + + boolean hasLLArgs = false; + + if ( !isFunction ) { + return false; + } + + /* + * If this is a Windowing Function and it has LeadLag expression in its args, then it will be handled + * by WindowingTabFunc. + */ + if ( isWindowFunction && !isRankingOrNavFunction ) { + + TreeWizard tw = new TreeWizard(ParseDriver.adaptor, HiveParser.tokenNames); + CheckLeadLagInSelectExprs checkLLFunctions = new CheckLeadLagInSelectExprs(); + for(int i=1; !hasLLArgs && i < function.getChildCount(); i++) { + ASTNode child = (ASTNode) function.getChild(i); + tw.visit(child, HiveParser.TOK_FUNCTION, checkLLFunctions); + if ( checkLLFunctions.isError() ) { + throw new SemanticException(generateErrorMessage(selectExpr, checkLLFunctions.getErrString())); + } + hasLLArgs = checkLLFunctions.isHasWindowingExprs(); + } + } + + if ( hasWindowSpec && !isWindowFunction ) { + throw new SemanticException(generateErrorMessage(selectExpr, + "Currently windowing Specification can only be associated with a UDAF invocation or navigation functions")); + } + + if ( + (isWindowFunction && hasWindowSpec) || + (isRankingOrNavFunction && isWindowFunction) || + (isWindowFunction && hasLLArgs) + ) { + /* + * @revisit: what should I do if there are more than 3 children; i.e. more than one Identifier. + */ + + String alias = null; + ASTNode secondChildNode = (ASTNode) selectExpr.getChild(1); + if ( childCount > 1 && secondChildNode.getType() == HiveParser.Identifier) { + alias = secondChildNode.getText(); + } + else { + /* + * generate an alias using the function AST tree. + * - calling getColAlias with a null InputResolver and null defaultName. + * - so includeFunctionName should be true. + */ + String[] colAlias = getColAlias(function, null, null, true, -1); + alias = colAlias[1]; + } + + + windowingFunctionExprs.add(function); + windowingFunctionAliases.add(alias); + windowingFunctionWSpecs.add(hasWindowSpec ? windowSpec : null); + return true; + + } + + return false; + + } + + /* + * - Invoked during Phase1 when a TOK_SELECT is encountered. + * - Select tree form is: ^(TOK_SELECT ^(TOK_SELECTEXPR...) ^(TOK_SELECTEXPR...) ...) + * - A Select Item form is: ^(TOK_SELEXPR selectExpression Identifier* window_specification?) + * + * We need to extract the SelectList any SelectItems that must be handled during Windowing processing. These are: + * - SelectItems that have a window_specification + * - SelectItems that invoke row navigation functions: Lead/Lag. + * + * Do we need to change the SelectList in any way? + * - initially we thought of replacing the selectExpressions handled by Windowing with a ASTNode that is of type Identfier and + * references the alias to the orginal expression. Why? + * - the output of processing the PTF Operator that handles windowing will contain the values of the Windowing expressions. + * - the final Select Op that is a child of the above PTF Op can get these values from its input by referring to the computed + * windowing expression via its alias. + * - but this is not needed. Why? + * - When transforming a AST tree to an ExprNodeDesc the TypeCheckFactory checks if there is a mapping from an AST tree to an output + * column in the InputResolver; if there is it uses the alias for the Output column + * - This is how values get handed from a GBy Op to the next Select Op; + * - we need the same thing to happen. + */ + private void handleWindowingExprsInSelectList(QB qb, Phase1Ctx ctx_1, ASTNode ast) throws SemanticException { + ArrayList windowingSelectExprs = new ArrayList(); + ArrayList windowingSelectExprAliases = new ArrayList(); + ArrayList windowingFunctionExprs = new ArrayList(); + ArrayList windowingFunctionAliases = new ArrayList(); + ArrayList windowingFunctionWSpecs = new ArrayList(); + + findWindowingExpressions(ast, + windowingFunctionExprs, + windowingFunctionAliases, + windowingFunctionWSpecs, + windowingSelectExprs, + windowingSelectExprAliases); + + addWindowingFuncsToPTFSpec(qb, ctx_1.dest, + windowingFunctionExprs, + windowingFunctionAliases, + windowingFunctionWSpecs, + windowingSelectExprs, + windowingSelectExprAliases); + } + + private void addWindowingFuncsToPTFSpec(QB currQB, String dest, + ArrayList windowingFunctionExprs, + ArrayList windowingFunctionAliases, + ArrayList windowingFunctionWSpecs, + ArrayList windowingSelectExprs, + ArrayList windowingSelectExprAliases) throws SemanticException { + + if ( (windowingSelectExprs != null && windowingSelectExprs.size() > 0) || + (windowingFunctionExprs != null && windowingFunctionExprs.size() > 0)){ + + queryProperties.setHasPTF(true); + /* + * Either use the PTF QuerySpec already associated with this QB or create a new one. + */ + PTFSpec spec = currQB.getWindowingPTFSpec(dest); + SelectSpec ss; + PTFInputSpec qInSpec; + if ( spec == null ) { + spec = new PTFSpec(); + currQB.addDestToWindowingPTFSpec(dest, spec); + /* + * set the Input TableSpec to the destination. + * Use this to find the input Operator during plan generation. + */ + qInSpec = new PTFTableOrSubQueryInputSpec(); + ((PTFTableOrSubQueryInputSpec)qInSpec).setTableName(dest); + } + else { + /* + * since there is already a PTF QSpec assocaited with this QB just add the WindowTblFunc to the chain. + */ + qInSpec = spec.getInput(); + } + ss = spec.getSelectList() == null? new SelectSpec() : spec.getSelectList(); + spec.setSelectList(ss); + + /* + * If query has no table function, then add one: + * - if query has window functions add the WindowingTableFunc; else add the Noop table function. + */ + boolean addTabFunction = false; + if(!windowingFunctionExprs.isEmpty() || !(qInSpec instanceof TableFuncSpec) ){ + addTabFunction = true; + } + + String tabFuncName = null; + TableFuncSpec tabSpec = null; + if(qInSpec instanceof TableFuncSpec){ + tabSpec = (TableFuncSpec) qInSpec; + tabFuncName = tabSpec.getName(); + } + + if ( addTabFunction ) { + boolean addWindowFunction = !windowingFunctionExprs.isEmpty(); + String fnName = addWindowFunction ? + FunctionRegistry.WINDOWING_TABLE_FUNCTION : + FunctionRegistry.NOOP_TABLE_FUNCTION; + //Need this check to avoid adding WINDOWING_TABLE_FUNCTION again when we + // invoke moveAggregationExprsToPTFSpec + if(!(tabFuncName != null && tabFuncName.equals(fnName))){ + tabSpec = new TableFuncSpec(); + tabSpec.setName(fnName); + tabSpec.setInput(qInSpec); + spec.setInput(tabSpec); + } + } + + LinkedHashMap windowingExprsMap = new LinkedHashMap(); + + for(int i=0; i < windowingFunctionExprs.size(); i++) { + String alias = windowingFunctionAliases.get(i); + WindowFunctionSpec wFn = processWindowFunction(windowingFunctionExprs.get(i), + windowingFunctionWSpecs == null? null: windowingFunctionWSpecs.get(i)); + tabSpec.inferDefaultWindowingSpec(wFn); + ss.addWindowFunc(wFn, alias); + windowingExprsMap.put(wFn.getExpression().toStringTree(), wFn.getExpression()); + } + + for(int i=0; windowingSelectExprs!= null && i < windowingSelectExprs.size(); i++) { + String alias = windowingSelectExprAliases.get(i); + ss.addExpression(windowingSelectExprs.get(i), alias); + windowingExprsMap.put(windowingSelectExprs.get(i).toStringTree(), windowingSelectExprs.get(i)); + } + + qb.getParseInfo().addWindowingExprsForClause(dest, windowingExprsMap); + + }else { + /* + * if we asscoiated the QB with a PTF QSpec, then we can remove this association. + * Since there are no Windowing Expressions to be handled in this Query; we don't + * need to defer processing of the PTF QSpec until after the GBy Op. + */ + currQB.addDestToWindowingPTFSpec(dest, null); + } + } + + /* + * this method is called in the case when the Query has Windowing Clauses and a Distribute/Cluster By. + * - the Distribute/Cluster By is associated with the WindowingTableFunc + * - If the Query has no Group by, but there are qb.destToAggExprs then move these to the Windowing QSpec. + */ + private void handleClusterOrDistributeByForWindowing(QB qb, Phase1Ctx ctx_1, ASTNode ast) throws SemanticException { + QBParseInfo qbp = qb.getParseInfo(); + PartitionSpec pSpec = processPartitionSpec(ast); + PTFSpec spec = qb.getWindowingPTFSpec(ctx_1.dest); + SelectSpec ss = spec.getSelectList(); + if ( ss == null ) { + ss = new SelectSpec(); + spec.setSelectList(ss); + } + TableFuncSpec tFn = (TableFuncSpec) spec.getInput(); + tFn.setPartition(pSpec); + tFn.setDefaultWindowSpec(null); + } + + private void handleSortByForWindowing(QB qb, Phase1Ctx ctx_1, ASTNode ast) throws SemanticException { + OrderSpec oSpec = processOrderSpec(ast); + PTFSpec spec = qb.getWindowingPTFSpec(ctx_1.dest); + TableFuncSpec tFn = (TableFuncSpec) spec.getInput(); + tFn.setOrder(oSpec); + tFn.inferDefaultOrderSpec(oSpec); + } + + private void handleQueryWindowClauses(QB qb, Phase1Ctx ctx_1, ASTNode node) throws SemanticException { + PTFSpec spec = qb.getWindowingPTFSpec(ctx_1.dest); + for(int i=0; i < node.getChildCount(); i++) { + processQueryWindowClause(spec, (ASTNode) node.getChild(i)); + } + } + + private void handleWindowingHavingClause(QB qb, Phase1Ctx ctx_1, ASTNode node) { + PTFSpec spec = qb.getWindowingPTFSpec(ctx_1.dest); + spec.setWhereExpr(node); + } + + PartitionSpec processPartitionSpec(ASTNode node) { + PartitionSpec pSpec = new PartitionSpec(); + int exprCnt = node.getChildCount(); + for(int i=0; i < exprCnt; i++) { + ColumnSpec exprSpec = new ColumnSpec(); + exprSpec.setExpression((ASTNode) node.getChild(i)); + //exprSpec.setColumnName(node.getChild(i).getChild(0).getText()); + pSpec.addColumn(exprSpec); + } + return pSpec; + } + + OrderSpec processOrderSpec(ASTNode sortNode) { + OrderSpec oSpec = new OrderSpec(); + int exprCnt = sortNode.getChildCount(); + for(int i=0; i < exprCnt; i++) { + OrderColumnSpec exprSpec = new OrderColumnSpec(); + exprSpec.setExpression((ASTNode) sortNode.getChild(i).getChild(0)); + //exprSpec.setColumnName(sortNode.getChild(i).getChild(0).getChild(0).getText()); + if ( sortNode.getChild(i).getType() == HiveParser.TOK_TABSORTCOLNAMEASC ) { + exprSpec.setOrder(org.apache.hadoop.hive.ql.parse.PTFSpec.Order.ASC); + } + else { + exprSpec.setOrder(org.apache.hadoop.hive.ql.parse.PTFSpec.Order.DESC); + } + oSpec.addColumn(exprSpec); + } + return oSpec; + } + + private void processPTFPartitionSpec(ASTNode pSpecNode, WindowSpec wSpec) + { + ASTNode firstChild = (ASTNode) pSpecNode.getChild(0); + int type = firstChild.getType(); + int exprCnt; + + PartitionSpec pSpec = processPartitionSpec(firstChild); + wSpec.setPartition(pSpec); + + if ( type == HiveParser.TOK_DISTRIBUTEBY ) + { + ASTNode sortNode = pSpecNode.getChildCount() > 1 ? (ASTNode) pSpecNode.getChild(1) : null; + if ( sortNode != null ) + { + OrderSpec oSpec = processOrderSpec(sortNode); + wSpec.setOrder(oSpec); + } + } + } + + private WindowFunctionSpec processWindowFunction(ASTNode node, ASTNode wsNode) throws SemanticException { + WindowFunctionSpec wfSpec = new WindowFunctionSpec(); + + switch(node.getType()) { + case HiveParser.TOK_FUNCTIONSTAR: + wfSpec.setStar(true); + break; + case HiveParser.TOK_FUNCTIONDI: + wfSpec.setDistinct(true); + break; + } + + if ( wfSpec.isDistinct() ) { + throw new SemanticException(generateErrorMessage(node, + "Count/Sum distinct not supported with Windowing")); + } + + wfSpec.setExpression(node); + + ASTNode nameNode = (ASTNode) node.getChild(0); + wfSpec.setName(nameNode.getText()); + + for(int i=1; i < node.getChildCount(); i++) { + ASTNode child = (ASTNode) node.getChild(i); + wfSpec.addArg(child); + } + + if ( wsNode != null ) { + WindowSpec ws = processWindowSpec(wsNode); + wfSpec.setWindowSpec(ws); + } + + /* + * In order to distinguish between different UDAF invocations on the same UDAF but different Windows + * add the WdwSpec node as a child of the Function Node. + * It is safe to do this after the function node has been converetd to a WdwFuncSpec. + */ + if ( wsNode != null ) { + node.addChild(wsNode); + } + + return wfSpec; + } + + private void processQueryWindowClause(PTFSpec qSpec, ASTNode node) throws SemanticException { + ASTNode nameNode = (ASTNode) node.getChild(0); + ASTNode wsNode = (ASTNode) node.getChild(1); + if(qSpec.getWindowSpecs().containsKey(nameNode.getText())){ + throw new SemanticException(generateErrorMessage(nameNode, + "Duplicate definition of window " + nameNode.getText() + + " is not allowed")); + } + WindowSpec ws = processWindowSpec(wsNode); + PTFInputSpec inpSpec = qSpec.getInput(); + /* + * If no Partitioning spec has been encountered so far, set this as the default Partitioning. + */ + if ( inpSpec instanceof TableFuncSpec ) { + TableFuncSpec tblFnSpec = (TableFuncSpec) inpSpec; + tblFnSpec.inferDefaultWindowingSpec(ws); + } + qSpec.addWindowSpec(nameNode.getText(), ws); + } + + private WindowSpec processWindowSpec(ASTNode node) { + String sourceId = null; + PartitionSpec partition = null; + OrderSpec order = null; + WindowFrameSpec windowFrame = null; + + boolean hasSrcId = false, hasPartSpec = false, hasWF = false; + int srcIdIdx = -1, partIdx = -1, wfIdx = -1; + + for(int i=0; i < node.getChildCount(); i++) + { + int type = node.getChild(i).getType(); + switch(type) + { + case HiveParser.Identifier: + hasSrcId = true; srcIdIdx = i; + break; + case HiveParser.TOK_PARTITIONINGSPEC: + hasPartSpec = true; partIdx = i; + break; + case HiveParser.TOK_WINDOWRANGE: + case HiveParser.TOK_WINDOWVALUES: + hasWF = true; wfIdx = i; + break; + } + } + + WindowSpec ws = new WindowSpec(); + + if (hasSrcId) { + ASTNode nameNode = (ASTNode) node.getChild(srcIdIdx); + ws.setSourceId(nameNode.getText()); + } + + if (hasPartSpec) { + ASTNode partNode = (ASTNode) node.getChild(partIdx); + processPTFPartitionSpec(partNode, ws); + } + + if ( hasWF) + { + ASTNode wfNode = (ASTNode) node.getChild(wfIdx); + WindowFrameSpec wfSpec = processWindowFrame(wfNode); + ws.setWindow(wfSpec); + } + + return ws; + } + + private WindowFrameSpec processWindowFrame(ASTNode node) { + int type = node.getType(); + BoundarySpec start = null, end = null; + + switch(type) + { + case HiveParser.TOK_WINDOWRANGE: + start = processRangeBoundary((ASTNode) node.getChild(0)); + end = processRangeBoundary((ASTNode) node.getChild(1)); + break; + case HiveParser.TOK_WINDOWVALUES: + start = processValueBoundary((ASTNode) node.getChild(0)); + end = processValueBoundary((ASTNode) node.getChild(1)); + break; + } + + return new WindowFrameSpec(start, end); + } + + private BoundarySpec processRangeBoundary(ASTNode node) { + RangeBoundarySpec rbs = new RangeBoundarySpec(); + BoundarySpec bs = rbs; + int type = node.getType(); + boolean hasAmt = true; + + switch(type) + { + case HiveParser.KW_PRECEDING: + rbs.setDirection(Direction.PRECEDING); + break; + case HiveParser.KW_FOLLOWING: + rbs.setDirection(Direction.FOLLOWING); + break; + case HiveParser.KW_CURRENT: + bs = new WindowFrameSpec.CurrentRowSpec(); + hasAmt = false; + break; + } + + if ( hasAmt ) + { + ASTNode amtNode = (ASTNode) node.getChild(0); + if ( amtNode.getType() == HiveParser.KW_UNBOUNDED) + { + rbs.setAmt(BoundarySpec.UNBOUNDED_AMOUNT); + } + else + { + rbs.setAmt(Integer.parseInt(amtNode.getText())); + } + } + + return bs; + } + + private BoundarySpec processValueBoundary(ASTNode node) { + BoundarySpec vbs = null; + int type = node.getType(); + + switch(type) + { + case HiveParser.KW_PRECEDING: + vbs = new ValueBoundarySpec(Direction.PRECEDING, null, BoundarySpec.UNBOUNDED_AMOUNT); + break; + case HiveParser.KW_FOLLOWING: + vbs = new ValueBoundarySpec(Direction.FOLLOWING, null, BoundarySpec.UNBOUNDED_AMOUNT); + break; + case HiveParser.KW_CURRENT: + vbs = new WindowFrameSpec.CurrentRowSpec(); + break; + case HiveParser.KW_LESS: + vbs = new ValueBoundarySpec(Direction.PRECEDING, + (ASTNode) node.getChild(0), + Integer.parseInt(node.getChild(1).getText())); + break; + case HiveParser.KW_MORE: + vbs = new ValueBoundarySpec(Direction.FOLLOWING, + (ASTNode) node.getChild(0), + Integer.parseInt(node.getChild(1).getText())); + break; + } + return vbs; + } + +//--------------------------- PTF handling: Qspec to QDef ----------------------------------- + + private PTFDef translateSpecToDef(PTFSpec qSpec, RowResolver inputRR) throws SemanticException{ + PTFDef qDef = null; + PTFTranslator translator = new PTFTranslator(); + qDef = translator.translate(qSpec, conf, inputRR, unparseTranslator); + return qDef; + } + + + Operator genPTFPlan(PTFSpec ptfQSpec, Operator input) throws SemanticException { + ArrayList componentQueries = PTFTranslator.componentize(ptfQSpec); + for (PTFSpec ptfSpec : componentQueries) { + input = genPTFPlanForComponentQuery(ptfSpec, input); + } + if (LOG.isDebugEnabled()) { + LOG.debug("Created PTF Plan "); + } + return input; + } + + + /** + * Construct the data structures containing ExprNodeDesc for partition + * columns and order columns. Use the input definition to construct the list + * of output columns for the ReduceSinkOperator + * + * @throws SemanticException + */ + void buildPTFReduceSinkDetails(TableFuncDef tabDef, + RowResolver inputRR, + ArrayList partCols, + ArrayList valueCols, + ArrayList orderCols, + Map colExprMap, + List outputColumnNames, + StringBuilder orderString, + RowResolver extractRR) throws SemanticException { + + ArrayList partColList = tabDef.getWindow().getPartDef().getColumns(); + + for (ColumnDef colDef : partColList) { + partCols.add(colDef.getExprNode()); + } + + /* + * Order columns are used as key columns for constructing + * the ReduceSinkOperator + * Since we do not explicitly add these to outputColumnNames, + * we need to set includeKeyCols = false while creating the + * ReduceSinkDesc + */ + ArrayList orderColList = tabDef.getWindow().getOrderDef().getColumns(); + for (int i = 0; i < orderColList.size(); i++) { + OrderColumnDef colDef = orderColList.get(i); + org.apache.hadoop.hive.ql.parse.PTFSpec.Order order = colDef.getOrder(); + if (order.name().equals("ASC")) { + orderString.append('+'); + } else { + orderString.append('-'); + } + orderCols.add(colDef.getExprNode()); + } + + /* + * We add the column to value columns or output column names + * only if it is not a virtual column + */ + ArrayList colInfoList = inputRR.getColumnInfos(); + LinkedHashMap colsAddedByHaving = new LinkedHashMap(); + int pos = 0; + for (ColumnInfo colInfo : colInfoList) { + if (!colInfo.isHiddenVirtualCol()) { + ExprNodeDesc valueColExpr = new ExprNodeColumnDesc(colInfo.getType(), colInfo + .getInternalName(), colInfo.getTabAlias(), colInfo + .getIsVirtualCol()); + valueCols.add(valueColExpr); + colExprMap.put(colInfo.getInternalName(), valueColExpr); + String outColName = SemanticAnalyzer.getColumnInternalName(pos++); + outputColumnNames.add(outColName); + + String[] alias = inputRR.reverseLookup(colInfo.getInternalName()); + /* + * if we have already encountered this colInfo internalName. + * We encounter it again because it must be put for the Having clause. + * We will add these entries in the end; in a loop on colsAddedByHaving. See below. + */ + if ( colsAddedByHaving.containsKey(alias)) { + continue; + } + ASTNode astNode = PTFTranslator.getASTNode(colInfo, inputRR); + ColumnInfo eColInfo = new ColumnInfo( + outColName, colInfo.getType(), alias[0], + colInfo.getIsVirtualCol(), colInfo.isHiddenVirtualCol()); + + if ( astNode == null ) { + extractRR.put(alias[0], alias[1], eColInfo); + } + else { + /* + * in case having clause refers to this column may have been added twice; once with the ASTNode.toStringTree as the alias + * and then with the real alias. + */ + extractRR.putExpression(astNode, eColInfo); + if ( !astNode.toStringTree().toLowerCase().equals(alias[1]) ) { + colsAddedByHaving.put(alias, eColInfo); + } + } + } + } + + for(Map.Entry columnAddedByHaving : colsAddedByHaving.entrySet() ) { + String[] alias = columnAddedByHaving.getKey(); + ColumnInfo eColInfo = columnAddedByHaving.getValue(); + extractRR.put(alias[0], alias[1], eColInfo); + } + } + + private Operator genPTFPlanForComponentQuery(PTFSpec ptfQSpec, Operator input) throws SemanticException { + /* + * 1. Create the QDef from the Qspec attached to this QB. + */ + RowResolver rr = opParseCtx.get(input).getRowResolver(); + PTFDef qdef = translateSpecToDef(ptfQSpec, rr); + /* + * Build an RR for the Extract Op from the ResuceSink Op's RR. + * Why? + * We need to remove the Virtual Columns present in the RS's RR. The OI + * that gets passed to Extract at runtime doesn't contain the Virtual Columns. + * So internal names get changed. Consider testCase testJoinWithLeadLag, which is a self join on part + * and also has a Windowing expression. The RR of the RS op at transaltion time looks something like this: + * (_co1,_col2,..,_col7, _col8(vc=true),_col9(vc=true),_col10,_col11,.._col15(vc=true),_col16(vc=true),..) + * At runtime the Virtual columns are removed and all the columns after _col7 are shifted 1 or 2 positions. + * So in child Operators ColumnExprNodeDesc's are no longer referring to the right columns. + * + * So we build a new RR for the Extract Op, with the Virtual Columns removed. We hand this to the PTFTranslator as the + * starting RR to use to translate a PTF Chain. + */ + RowResolver extractOpRR = new RowResolver(); + + /* + * 2. build Map-side Op Graph. Graph template is either: + * Input -> PTF_map -> ReduceSink + * or + * Input -> ReduceSink + * + * Here the ExprNodeDescriptors in the QueryDef are based on the Input Operator's RR. + */ + { + rr = opParseCtx.get(input).getRowResolver(); + TableFuncDef tabDef = PTFTranslator.getFirstTableFunction(qdef); + + /* + * a. add Map-side PTF Operator if needed + */ + if (PTFTranslator.addPTFMapOperator(qdef)) + { + RowResolver ptfMapRR = qdef.getTranslationInfo().getInputInfo(tabDef).getRowResolver(); + + input = putOpInsertMap(OperatorFactory.getAndMakeChild(new PTFDesc( + PTFUtils.serializeQueryDef(qdef)), + new RowSchema(ptfMapRR.getColumnInfos()), + input), ptfMapRR); + rr = opParseCtx.get(input).getRowResolver(); + } + + /* + * b. Build Reduce Sink Details (keyCols, valueCols, outColNames etc.) for this QDef. + */ + + ArrayList partCols = new ArrayList(); + ArrayList valueCols = new ArrayList(); + ArrayList orderCols = new ArrayList(); + Map colExprMap = new HashMap(); + List outputColumnNames = new ArrayList(); + StringBuilder orderString = new StringBuilder(); + + /* + * Use the input RR of TableScanOperator in case there is no map-side + * reshape of input. + * If the parent of ReduceSinkOperator is PTFOperator, use it's + * output RR. + */ + buildPTFReduceSinkDetails(tabDef, + rr, + partCols, + valueCols, + orderCols, + colExprMap, + outputColumnNames, + orderString, + extractOpRR); + + input = putOpInsertMap(OperatorFactory.getAndMakeChild(PlanUtils + .getReduceSinkDesc(orderCols, + valueCols, outputColumnNames, false, + -1, partCols, orderString.toString(), -1), + new RowSchema(rr.getColumnInfos()), input), rr); + input.setColumnExprMap(colExprMap); + } + + /* + * 3. build Reduce-side Op Graph + */ + { + /* + * b. Construct Extract Operator. + */ + input = putOpInsertMap(OperatorFactory.getAndMakeChild( + new ExtractDesc( + PTFTranslator.getStringColumn(Utilities.ReduceField.VALUE + .toString())), + new RowSchema(extractOpRR.getColumnInfos()), + input), extractOpRR); + + /* + * c. Rebuilt the QueryDef. + * Why? + * - so that the ExprNodeDescriptors in the QueryDef are based on the Extract Operator's RowResolver + */ + rr = opParseCtx.get(input).getRowResolver(); + qdef = translateSpecToDef(ptfQSpec, rr); + + /* + * d. Construct PTF Operator. + */ + RowResolver ptfOpRR = qdef.getSelectList().getRowResolver(); + input = putOpInsertMap(OperatorFactory.getAndMakeChild(new PTFDesc( + PTFUtils.serializeQueryDef(qdef)), + new RowSchema(ptfOpRR.getColumnInfos()), + input), ptfOpRR); + + + dumpOperatorChain(input, qdef); + } + + return input; + + } + + + /* + * check if a Select Expr is a constant. + * - current logic used is to look for HiveParser.TOK_TABLE_OR_COL + * - if there is none then the expression is a constant. + */ + private static class ConstantExprCheck implements ContextVisitor { + boolean isConstant = true; + + public void visit(Object t, Object parent, int childIndex, Map labels) { + if ( !isConstant ) { + return; + } + ASTNode node = (ASTNode) t; + if (ParseDriver.adaptor.getType(t) == HiveParser.TOK_TABLE_OR_COL ) { + isConstant = false; + } + } + + public void reset() { + isConstant = true; + } + + protected boolean isConstant() { + return isConstant; + } + } + + private static class AggregationExprCheck implements ContextVisitor { + HashMap destAggrExprs; + boolean isAggr = false; + + public AggregationExprCheck(HashMap destAggrExprs) { + super(); + this.destAggrExprs = destAggrExprs; + } + + public void visit(Object t, Object parent, int childIndex, Map labels) { + if ( isAggr ) { + return; + } + if ( destAggrExprs.values().contains(t)) { + isAggr = true; + } + } + + public void reset() { + isAggr = false; + } + + protected boolean isAggr() { + return isAggr; + } + } + + /* + * If we decide to create a PTFSpec for the QB (see ensurePTFSpecOnQB(...)) + * then this method is invoked. + * Constructs the function expr and alias list and invokes addWindowingFuncsToPTFSpec + */ + private void moveaggregationExprsToPTFSpec(QB currQB, String clause) throws SemanticException { + HashMap aggregationTree = currQB.getParseInfo().getAggregationExprsForClause(clause); + if((aggregationTree != null) && !(aggregationTree.isEmpty()) ){ + Collection aggrExprs = aggregationTree.values(); + ArrayList windowingFunctionExprs = new ArrayList(); + ArrayList windowingFunctionAliases = new ArrayList(); + for (ASTNode expr : aggrExprs) { + windowingFunctionExprs.add(expr); + String alias = currQB.getParseInfo().getExprToColumnAlias(expr); + if ( alias == null ) { + /* + * generate an alias using the function AST tree. + * - calling getColAlias with a null InputResolver and null defaultName. + * - so includeFunctionName should be true. + */ + String[] colAlias = getColAlias(expr, null, null, true, -1); + alias = colAlias[1]; + } + windowingFunctionAliases.add(alias); + currQB.getParseInfo().getAllExprToColumnAlias().remove(expr); + } + currQB.getParseInfo().clearAggregationExprsForClause(clause); + addWindowingFuncsToPTFSpec(currQB, clause, windowingFunctionExprs, + windowingFunctionAliases, null, null, null); + } + } + + /* + * Returns false if there is a SelectExpr that is not a constant or an aggr. + * + */ + private boolean isValidGroupBySelectList(QB currQB, String clause){ + ConstantExprCheck constantExprCheck = new ConstantExprCheck(); + AggregationExprCheck aggrExprCheck = new AggregationExprCheck( + currQB.getParseInfo().getAggregationExprsForClause(clause)); + + TreeWizard tw = new TreeWizard(ParseDriver.adaptor, HiveParser.tokenNames); + ASTNode selectNode = currQB.getParseInfo().getSelForClause(clause); + + /* + * for Select Distinct Queries we don't move any aggregations. + */ + if ( selectNode != null && selectNode.getType() == HiveParser.TOK_SELECTDI ) { + return true; + } + + for (int i = 0; selectNode != null && i < selectNode.getChildCount(); i++) { + ASTNode selectExpr = (ASTNode) selectNode.getChild(i); + //check for TOK_HINTLIST expressions on ast + if(selectExpr.getType() != HiveParser.TOK_SELEXPR){ + continue; + } + + constantExprCheck.reset(); + PTFTranslator.visit(selectExpr.getChild(0), constantExprCheck); + + if ( !constantExprCheck.isConstant() ) { + aggrExprCheck.reset(); + PTFTranslator.visit(selectExpr.getChild(0), aggrExprCheck); + if (!aggrExprCheck.isAggr() ) { + return false; + } + } + + } + return true; + } + + /** + * If the query has no group by and has aggregations, then we treat this as a + * windowing query. + * There is one caveat: if the select list has only aggregations, then + * this is not handled by windowing. + */ + private void ensurePTFSpecOnQB(QB currQB, String clause) throws SemanticException{ + if( currQB.getParseInfo().getAggregationExprsForClause(clause) != null + && !queryProperties.hasGroupBy()){ + boolean isValid = isValidGroupBySelectList(currQB, clause); + if(!isValid) { + moveaggregationExprsToPTFSpec(currQB, clause); + } + } + } + + public static SelectSpec parseSelect(String selectExprStr) throws SemanticException + { + ASTNode selNode = null; + try { + ParseDriver pd = new ParseDriver(); + selNode = pd.parseSelect(selectExprStr, null); + } catch (ParseException pe) { + throw new SemanticException(pe); + } + + SelectSpec selSpec = new SelectSpec(); + int childCount = selNode.getChildCount(); + for (int i = 0; i < childCount; i++) { + ASTNode selExpr = (ASTNode) selNode.getChild(i); + if (selExpr.getType() != HiveParser.TOK_SELEXPR) { + throw new SemanticException(PTFUtils.sprintf( + "Only Select expressions supported in dynamic select list", selectExprStr)); + } + ASTNode expr = (ASTNode) selExpr.getChild(0); + if (expr.getType() == HiveParser.TOK_ALLCOLREF) { + throw new SemanticException(PTFUtils.sprintf("'*' column not allowed in dynamic select list", + selectExprStr)); + } + ASTNode aliasNode = selExpr.getChildCount() > 1 + && selExpr.getChild(1).getType() == HiveParser.Identifier ? + (ASTNode) selExpr.getChild(1) : null; + String alias = null; + if ( aliasNode != null ) { + alias = aliasNode.getText(); + } + else { + String[] tabColAlias = getColAlias(selExpr, null, null, true, -1); + alias = tabColAlias[1]; + } + selSpec.addExpression(expr, alias); + } + + return selSpec; + } + + // debug methods + void dumpOperatorChain(Operator sinkOp, PTFDef qdef) { + Stack stack = new Stack(); + Operator op = sinkOp; + while(op != null ) { + stack.push(op); + List parentOps =op.getParentOperators(); + if (parentOps != null ) { + op = parentOps.get(0); + } + else { + op = null; + } + } + + int opNum = 1; + StringBuilder buf = new StringBuilder(); + while(!stack.isEmpty()) { + op = stack.pop(); + buf.append("\n").append(opNum).append("."); + buf.append(op.getName()); + buf.append(" :\n"); + RowResolver rr = opParseCtx.get(op).getRowResolver(); + dumpRowResolver(buf, rr); + if ( op instanceof PTFOperator && qdef != null ) { + /* + * 1/21 hb: this is no longer correct; in a chain containing multiple PTFOps, every PTFOp dump prints the info from the + * last PTFDef + */ + dump(buf, qdef); + } + opNum++; + } + System.out.println(buf); + } + + static void dumpRowResolver(StringBuilder buf, RowResolver rr) { + buf.append("RowResolver::\n"); + buf.append("\tcolumns:["); + boolean first = true; + for(ColumnInfo cInfo : rr.getRowSchema().getSignature()) { + String tabalias = cInfo.getTabAlias(); + String cname = cInfo.getInternalName(); + if (!first) { + buf.append(", "); + } else { + first = false; + } + buf.append(tabalias != null ? tabalias : ""); + buf.append("."); + buf.append(cname); + } + buf.append("]\n"); + buf.append("\tAliases:["); + for(Map.Entry> entry : rr.getRslvMap().entrySet() ) { + String tabalias = entry.getKey(); + buf.append("\n\t\t"); + buf.append(tabalias != null ? tabalias : ""); + buf.append(":["); + LinkedHashMap colAliases = entry.getValue(); + first = true; + for(Map.Entry column: colAliases.entrySet()) { + if (!first) { + buf.append(", "); + } else { + first = false; + } + buf.append(column.getKey()).append(" -> ").append(column.getValue().getInternalName()); + } + } + buf.append("\n\t]\n"); + buf.append("\tcolumns mapped to expressions:["); + first = true; + for(Map.Entry exprs : rr.getExpressionMap().entrySet()) { + if (!first) { + buf.append(", "); + } else { + first = false; + } + buf.append("\n\t\t"); + buf.append(exprs.getKey()); + buf.append(" -> "); + buf.append(exprs.getValue().toStringTree()); + } + buf.append("\n\t]\n"); + } + + private static void dump(StringBuilder buf, PTFDef qDef) { + Iterator iDefs = PTFTranslator.iterateInputDefs(qDef, true); + while(iDefs.hasNext() ) { + PTFInputDef iDef = iDefs.next(); + if ( iDef instanceof PTFTableOrSubQueryInputDef ) { + dump(buf, (PTFTableOrSubQueryInputDef) iDef); + }else if ( iDef instanceof PTFComponentQueryDef ) { + dump(buf, (PTFComponentQueryDef) iDef); + }else { + dump(buf, (TableFuncDef) iDef); + } + } + + StructObjectInspector OI = (StructObjectInspector) qDef.getSelectList().getOI(); + buf.append("\nSelectList:"); + dump(buf, OI); + } + + private static void dump(StringBuilder buf, TableFuncDef tFnDef) { + buf.append("\n").append(tFnDef.getName()).append(":"); + dump(buf, (PTFInputDef)tFnDef); + TableFunctionEvaluator tFn = tFnDef.getFunction(); + + if ( tFn.isTransformsRawInput() ) { + buf.append("\nEvaluator RawInput ObjectInspector:["); + dump(buf, tFn.getRawInputOI()); + buf.append("]"); + } + + buf.append("\nEvaluator Output ObjectInspector:["); + dump(buf, tFn.getOutputOI()); + buf.append("]"); + + } + private static void dump(StringBuilder buf, PTFTableOrSubQueryInputDef htblDef) { + buf.append("\n").append(htblDef.getHiveInputSpec().getTableName()).append(":"); + dump(buf, (PTFInputDef)htblDef); + } + + private static void dump(StringBuilder buf, PTFInputDef qInDef) { + StructObjectInspector OI = (StructObjectInspector) qInDef.getOI(); + buf.append("\nDef ObjectInspector:["); + dump(buf, OI); + buf.append("]\nSerDe:").append(qInDef.getSerde().getClass().getName()); + } + + private static void dump(StringBuilder buf, StructObjectInspector OI) { + boolean first = true; + for(StructField field : OI.getAllStructFieldRefs() ) { + if (!first) { + buf.append(", "); + } else { + first = false; + } + buf.append(field.getFieldName()); + } + } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingExprNodeEvaluatorFactory.java ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingExprNodeEvaluatorFactory.java new file mode 100644 index 0000000..cd0106f --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingExprNodeEvaluatorFactory.java @@ -0,0 +1,156 @@ +package org.apache.hadoop.hive.ql.parse; + +import java.util.List; + +import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; +import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.parse.PTFTranslator.PTFTranslationInfo; +import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFLeadLag.GenericUDFLag; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFLeadLag.GenericUDFLead; + +/* + * When constructing the Evaluator Tree from an ExprNode Tree + * - look for any descendant LeadLag Function Expressions + * - if they are found: + * - add them to the LLInfo.leadLagExprs and + * - add a mapping from the Expr Tree root to the LLFunc Expr in LLInfo.mapTopExprToLLFunExprs + */ + +public class WindowingExprNodeEvaluatorFactory +{ + public static ExprNodeEvaluator get(PTFTranslationInfo tInfo, ExprNodeDesc desc) throws HiveException + { + FindLeadLagFuncExprs visitor = new FindLeadLagFuncExprs(tInfo, desc); + new ExprNodeWalker(visitor).walk(desc); + return ExprNodeEvaluatorFactory.get(desc); + } + + public static class FindLeadLagFuncExprs extends ExprNodeVisitor + { + ExprNodeDesc topExpr; + PTFTranslationInfo tInfo; + + FindLeadLagFuncExprs(PTFTranslationInfo tInfo, ExprNodeDesc topExpr) + { + this.tInfo = tInfo; + this.topExpr = topExpr; + } + + @Override + public void visit(ExprNodeGenericFuncDesc fnExpr) throws HiveException + { + GenericUDF fn = fnExpr.getGenericUDF(); + if (fn instanceof GenericUDFLead || fn instanceof GenericUDFLag ) + { + tInfo.getLLInfo().addLLFuncExprForTopExpr(topExpr, fnExpr); + } + } + } + + static class ExprNodeVisitor + { + public void visit(ExprNodeColumnDesc e) throws HiveException + { + } + + public void visit(ExprNodeConstantDesc e) throws HiveException + { + } + + public void visit(ExprNodeFieldDesc e) throws HiveException + { + } + + public void visit(ExprNodeGenericFuncDesc e) throws HiveException + { + } + + public void visit(ExprNodeNullDesc e) throws HiveException + { + } + } + + static class ExprNodeWalker + { + ExprNodeVisitor visitor; + + public ExprNodeWalker(ExprNodeVisitor visitor) + { + super(); + this.visitor = visitor; + } + + public void walk(ExprNodeDesc e) throws HiveException + { + if ( e == null ) { + return; + } + List children = e.getChildren(); + if ( children != null ) + { + for(ExprNodeDesc child : children) + { + walk(child); + } + } + + if ( e instanceof ExprNodeColumnDesc) + { + walk((ExprNodeColumnDesc) e); + } + else if ( e instanceof ExprNodeConstantDesc) + { + walk((ExprNodeConstantDesc) e); + } + else if ( e instanceof ExprNodeFieldDesc) + { + walk((ExprNodeFieldDesc) e); + } + else if ( e instanceof ExprNodeGenericFuncDesc) + { + walk((ExprNodeGenericFuncDesc) e); + } + else if ( e instanceof ExprNodeNullDesc) + { + walk((ExprNodeNullDesc) e); + } + else + { + throw new HiveException("Unknown Expr Type " + e.getClass().getName()); + } + } + + private void walk(ExprNodeColumnDesc e) throws HiveException + { + visitor.visit(e); + } + + private void walk(ExprNodeConstantDesc e) throws HiveException + { + visitor.visit(e); + } + + private void walk(ExprNodeFieldDesc e) throws HiveException + { + visitor.visit(e); + } + + private void walk(ExprNodeGenericFuncDesc e) throws HiveException + { + visitor.visit(e); + } + + private void walk(ExprNodeNullDesc e) throws HiveException + { + visitor.visit(e); + } + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDef.java ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDef.java new file mode 100644 index 0000000..76a361d --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDef.java @@ -0,0 +1,1192 @@ +package org.apache.hadoop.hive.ql.plan; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; +import org.apache.hadoop.hive.ql.exec.PTFUtils; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.PTFSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.ColumnSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.Order; +import org.apache.hadoop.hive.ql.parse.PTFSpec.OrderColumnSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.OrderSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.PTFInputSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.PTFTableOrSubQueryInputSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.PartitionSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.SelectSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.TableFuncSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowFrameSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowFrameSpec.BoundarySpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowFrameSpec.CurrentRowSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowFrameSpec.Direction; +import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowFrameSpec.RangeBoundarySpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowFrameSpec.ValueBoundarySpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowFunctionSpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowSpec; +import org.apache.hadoop.hive.ql.parse.PTFTranslator.PTFInputInfo; +import org.apache.hadoop.hive.ql.parse.PTFTranslator.PTFTranslationInfo; +import org.apache.hadoop.hive.ql.parse.RowResolver; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; +import org.apache.hadoop.hive.ql.udf.ptf.TableFunctionEvaluator; +import org.apache.hadoop.hive.serde2.SerDe; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; + +public class PTFDef +{ + @SuppressWarnings("unused") + private static final Log LOG = LogFactory.getLog(PTFDef.class.getName()); + PTFSpec spec; + transient PTFTranslationInfo translationInfo; + Map windowDefs; + SelectDef selectList; + PTFInputDef input; + WhereDef whereDef; + static{ + PTFUtils.makeTransient(PTFDef.class, "translationInfo"); + } + + public PTFDef(){ + + } + + public PTFSpec getSpec() + { + return spec; + } + + public void setSpec(PTFSpec spec) + { + this.spec = spec; + } + + public PTFTranslationInfo getTranslationInfo() + { + return translationInfo; + } + + public void setTranslationInfo(PTFTranslationInfo translationInfo) + { + this.translationInfo = translationInfo; + } + + public Map getWindowDefs() + { + return windowDefs; + } + + public void setWindowDefs(Map windowDefs) + { + this.windowDefs = windowDefs; + } + + public SelectDef getSelectList() + { + selectList = selectList == null ? new SelectDef() : selectList; + return selectList; + } + + public void setSelectList(SelectDef selectList) + { + this.selectList = selectList; + } + + public PTFInputDef getInput() + { + return input; + } + public void setInput(PTFInputDef input) + { + this.input = input; + } + + public WhereDef getWhere() + { + return whereDef; + } + + public void setWhere(WhereDef whereDef) + { + this.whereDef = whereDef; + } + + /* + * get the Hive Table associated with this input chain. + */ + public PTFInputSpec getHiveTableSpec() + { + return getInput().getHiveInputSpec(); + } + + public PTFInputDef getHiveTableDef() + { + return getInput().getHiveInputDef(); + } + + /* + * QueryInputDef class + */ + public static abstract class PTFInputDef + { + PTFInputSpec inputSpec; + WindowDef window; + transient StructObjectInspector OI; + transient SerDe serde; + /* + * The inputInfo is provided so that the setupOI call use the input RowResolver & TypeCheckCtx. + * This is not needed normally. The Args to a TableFunction are already translated when the setupOI call is made. + * But in the case of dynamic SQL: such as the ResultExpression in NPath, where an argument is parsed as a Select List + * the external names(aliases) of columns are needed to translate user specified columns to internal names. + */ + transient PTFInputInfo inputInfo; + String alias; + + static{ + PTFUtils.makeTransient(PTFInputDef.class, "serde"); + PTFUtils.makeTransient(PTFInputDef.class, "OI"); + PTFUtils.makeTransient(PTFInputDef.class, "inputInfo"); + } + + + public PTFInputDef(){ + + } + + public PTFInputSpec getSpec() + { + return inputSpec; + } + + public void setSpec(PTFInputSpec inputSpec) + { + this.inputSpec = inputSpec; + } + + public WindowDef getWindow() + { + return window; + } + + public void setWindow(WindowDef window) + { + this.window = window; + } + + public StructObjectInspector getOI() + { + return OI; + } + + public void setOI(StructObjectInspector oI) + { + OI = oI; + } + + public PTFInputInfo getInputInfo() { + return inputInfo; + } + + public void setInputInfo(PTFInputInfo inputInfo) { + this.inputInfo = inputInfo; + } + + public SerDe getSerde() + { + return serde; + } + + public void setSerde(SerDe serde) + { + this.serde = serde; + } + + public String getAlias() + { + return alias; + } + + public void setAlias(String alias) + { + this.alias = alias; + } + + /* + * get the Hive Table associated with this input chain. + */ + public abstract PTFInputSpec getHiveInputSpec(); + + public abstract PTFInputDef getHiveInputDef(); + + public PTFInputSpec getInputSpec() { + return inputSpec; + } + + public void setInputSpec(PTFInputSpec inputSpec) { + this.inputSpec = inputSpec; + } + } + + /* + * PTF Component Query Def + */ + public static class PTFComponentQueryDef extends PTFInputDef + { + + String compSerdeClassName; + + public String getCompSerdeClassName() { + return compSerdeClassName; + } + + public void setCompSerdeClassName(String compSerdeClassName) { + this.compSerdeClassName = compSerdeClassName; + } + + @Override + public PTFInputSpec getHiveInputSpec() { + return this.getSpec(); + } + + @Override + public PTFInputDef getHiveInputDef() { + return this; + } + } + + /* + * HiveTableDef + */ + public static class PTFTableOrSubQueryInputDef extends PTFInputDef + { + String tableSerdeClassName; + Map tableSerdeProps; + + String location; + String inputFormatClassName; + + @Override + public PTFTableOrSubQueryInputSpec getHiveInputSpec() + { + return (PTFTableOrSubQueryInputSpec) inputSpec; + } + + public String getTableSerdeClassName() + { + return tableSerdeClassName; + } + + public void setTableSerdeClassName(String tableSerdeClassName) + { + this.tableSerdeClassName = tableSerdeClassName; + } + + public Map getTableSerdeProps() + { + return tableSerdeProps; + } + + public void setTableSerdeProps(Map tableSerdeProps) + { + this.tableSerdeProps = tableSerdeProps; + } + + public String getLocation() + { + return location; + } + + public void setLocation(String location) + { + this.location = location; + } + + public String getInputFormatClassName() + { + return inputFormatClassName; + } + + public void setInputFormatClassName(String inputFormatClassName) + { + this.inputFormatClassName = inputFormatClassName; + } + + @Override + public PTFTableOrSubQueryInputDef getHiveInputDef() { + return this; + } + + } + + + /* + * TableFuncDef + */ + public static class TableFuncDef extends PTFInputDef + { + ArrayList args; + PTFInputDef input; + ArrayList outputColumnNames; + /* + * set during translation, based on value provided by PTFResolver. + * This is used during translation to decide if the internalName -> alias mapping from the Input to the PTF is carried + * forward when building the Output RR for this PTF. + * See {@link PTFResolver#carryForwardNames()} for details. + */ + boolean carryForwardNames; + TableFunctionEvaluator tFunction; + ArrayList rawInputColumnNames; + transient ObjectInspector rawInputOI; + transient PTFInputInfo rawInputInfo; + + /* + * @remove + * used for validating that OI columnNames are the same during runtime. + */ + HashMap rawInputSerdeProps; + /* + * @remove + * used for validating that OI columnNames are the same during runtime. + */ + HashMap outputSerdeProps; + + static{ + PTFUtils.makeTransient(TableFuncDef.class, "rawInputOI"); + PTFUtils.makeTransient(TableFuncDef.class, "rawInputInfo"); + } + + + public TableFuncSpec getTableFuncSpec() + { + return (TableFuncSpec) inputSpec; + } + + public ArrayList getArgs() + { + return args; + } + + public void setArgs(ArrayList args) + { + this.args = args; + } + + public void addArg(ArgDef arg) + { + args = args == null ? new ArrayList() : args; + args.add(arg); + } + + public PTFInputDef getInput() + { + return input; + } + + public void setInput(PTFInputDef input) + { + this.input = input; + } + + public String getName() + { + return getTableFuncSpec().getName(); + } + + public TableFunctionEvaluator getFunction() + { + return tFunction; + } + + public void setFunction(TableFunctionEvaluator tFunction) + { + this.tFunction = tFunction; + } + + @Override + public PTFInputSpec getHiveInputSpec() + { + return input.getHiveInputSpec(); + } + public ObjectInspector getRawInputOI() { + return rawInputOI; + } + + public void setRawInputOI(ObjectInspector rawInputOI) { + this.rawInputOI = rawInputOI; + } + + public PTFInputInfo getRawInputInfo() { + return rawInputInfo; + } + + public void setRawInputInfo(PTFInputInfo rawInputInfo) { + this.rawInputInfo = rawInputInfo; + } + + @Override + public PTFInputDef getHiveInputDef() { + return input.getHiveInputDef(); + } + + public ArrayList getOutputColumnNames() { + return outputColumnNames; + } + + public void setOutputColumnNames(ArrayList outputColumnNames) { + this.outputColumnNames = outputColumnNames; + } + + public ArrayList getRawInputColumnNames() { + return rawInputColumnNames; + } + + public void setRawInputColumnNames(ArrayList rawInputColumnNames) { + this.rawInputColumnNames = rawInputColumnNames; + } + + public boolean isCarryForwardNames() { + return carryForwardNames; + } + + public void setCarryForwardNames(boolean carryForwardNames) { + this.carryForwardNames = carryForwardNames; + } + + public HashMap getRawInputSerdeProps() { + return rawInputSerdeProps; + } + + public void setRawInputSerdeProps(HashMap rawInputSerdeProps) { + this.rawInputSerdeProps = rawInputSerdeProps; + } + + public HashMap getOutputSerdeProps() { + return outputSerdeProps; + } + + public void setOutputSerdeProps(HashMap outputSerdeProps) { + this.outputSerdeProps = outputSerdeProps; + } + } + + /* + * WindowFuncDef class + */ + public static class WindowFunctionDef + { + WindowFunctionSpec wSpec; + ArrayList args; + WindowDef window; + transient GenericUDAFEvaluator wFnEval; + transient ObjectInspector OI; + + static{ + PTFUtils.makeTransient(WindowFunctionDef.class, "wFnEval"); + PTFUtils.makeTransient(WindowFunctionDef.class, "OI"); + } + + + public WindowFunctionDef(){ + + } + + public WindowFunctionSpec getSpec() + { + return wSpec; + } + + public void setSpec(WindowFunctionSpec wSpec) + { + this.wSpec = wSpec; + } + + public ArrayList getArgs() + { + return args; + } + + public void setArgs(ArrayList args) + { + this.args = args; + } + + public void addArg(ArgDef arg) + { + args = args == null ? new ArrayList() : args; + args.add(arg); + } + + public WindowDef getWindow() + { + return window; + } + + public void setWindow(WindowDef window) + { + this.window = window; + } + + public GenericUDAFEvaluator getEvaluator() + { + return wFnEval; + } + + public void setEvaluator(GenericUDAFEvaluator wFnEval) + { + this.wFnEval = wFnEval; + } + + public ObjectInspector getOI() + { + return OI; + } + + public void setOI(ObjectInspector oI) + { + OI = oI; + } + + } + + /** + * represents and argument of a Window or table Function. + * + */ + public static class ArgDef + { + ASTNode expression; + ExprNodeDesc exprNode; + transient ExprNodeEvaluator exprEvaluator; + transient ObjectInspector OI; + + static{ + PTFUtils.makeTransient(ArgDef.class, "exprEvaluator"); + PTFUtils.makeTransient(ArgDef.class, "OI"); + } + + + public ArgDef(){ + + } + + public ASTNode getExpression() + { + return expression; + } + + public void setExpression(ASTNode expression) + { + this.expression = expression; + } + + public ExprNodeDesc getExprNode() + { + return exprNode; + } + + public void setExprNode(ExprNodeDesc exprNode) + { + this.exprNode = exprNode; + } + + public ExprNodeEvaluator getExprEvaluator() + { + return exprEvaluator; + } + + public void setExprEvaluator(ExprNodeEvaluator exprEvaluator) + { + this.exprEvaluator = exprEvaluator; + } + + public ObjectInspector getOI() + { + return OI; + } + + public void setOI(ObjectInspector oI) + { + OI = oI; + } + + } + + /* + * Represents either a (Partition, Order) definition or a QueryInput or represents a WindowSpec for a WindowFunc. + *
    + *
  • sourceWSpec will be null when this represents a (Partition, Order) definition or a QueryInput + *
  • If the sourceWSpec has a sourceId then the unspecified elements in the WindowSpec are applied from the referenced WIndowSpec. + *
+ */ + public static class WindowDef + { + WindowSpec spec; + PartitionDef partDef; + OrderDef orderDef; + WindowFrameDef window; + + public WindowDef(){ + + } + + public WindowDef(WindowSpec spec) + { + this.spec = spec; + } + + public WindowDef(WindowSpec spec, WindowDef srcWdwDef) + { + this(spec); + partDef = srcWdwDef.partDef; + orderDef = srcWdwDef.orderDef; + window = srcWdwDef.window; + } + + public WindowSpec getsSpec() + { + return spec; + } + + public PartitionDef getPartDef() + { + return partDef; + } + + public void setPartDef(PartitionDef partDef) + { + this.partDef = partDef; + } + + public OrderDef getOrderDef() + { + return orderDef; + } + + public void setOrderDef(OrderDef orderDef) + { + this.orderDef = orderDef; + } + + public WindowFrameDef getWindow() + { + return window; + } + + public void setWindow(WindowFrameDef window) + { + this.window = window; + } + + } + + /* + * PartitionDef class + */ + public static class PartitionDef + { + PartitionSpec spec; + ArrayList columns; + + public PartitionDef(){ + + } + + public PartitionDef(PartitionSpec spec) + { + this.spec = spec; + } + + public PartitionSpec getSpec() + { + return spec; + } + + public ArrayList getColumns() + { + return columns; + } + + public void setColumns(ArrayList columns) + { + this.columns = columns; + } + + public void addColumn(ColumnDef c) + { + columns = columns == null ? new ArrayList() : columns; + columns.add(c); + } + + } + + /* + * OrderDef class + */ + public static class OrderDef + { + OrderSpec spec; + ArrayList columns; + + public OrderDef(){ + + } + + public OrderDef(OrderSpec spec) + { + this.spec = spec; + } + + public OrderDef(PartitionDef pDef) + { + this.spec = new OrderSpec(pDef.getSpec()); + for(ColumnDef cDef : pDef.getColumns()) + { + addColumn(new OrderColumnDef(cDef)); + } + } + + public OrderSpec getSpec() + { + return spec; + } + + public ArrayList getColumns() + { + return columns; + } + + public void setColumns(ArrayList columns) + { + this.columns = columns; + } + + public void addColumn(OrderColumnDef c) + { + columns = columns == null ? new ArrayList() : columns; + columns.add(c); + } + } + + /* + * represents a Column reference in a Partition or Order Spec or an Expr in the SelectSpec. + */ + public static class ColumnDef + { + /* + * non-null if this is a Column reference in a Partition or Order Spec. + */ + ColumnSpec spec; + + String alias; + + ASTNode expression; + ExprNodeDesc exprNode; + transient ExprNodeEvaluator exprEvaluator; + transient ObjectInspector OI; + + static{ + PTFUtils.makeTransient(ColumnDef.class, "exprEvaluator"); + PTFUtils.makeTransient(ColumnDef.class, "OI"); + } + + public ColumnDef(){ + + } + + public ColumnDef(ColumnSpec spec) + { + this.spec = spec; + } + + public ColumnDef(ColumnDef cDef) + { + spec = cDef.getSpec(); + alias = cDef.getAlias(); + expression = cDef.getExpression(); + exprNode = cDef.getExprNode(); + exprEvaluator = cDef.getExprEvaluator(); + OI = cDef.getOI(); + } + + public void setSpec(ColumnSpec spec) { + this.spec = spec; + } + + public ColumnSpec getSpec() + { + return spec; + } + + public String getAlias() + { + return alias; + } + + public void setAlias(String alias) + { + this.alias = alias; + } + + public ASTNode getExpression() + { + return expression; + } + + public void setExpression(ASTNode expression) + { + this.expression = expression; + } + + public ExprNodeDesc getExprNode() + { + return exprNode; + } + + public void setExprNode(ExprNodeDesc exprNode) + { + this.exprNode = exprNode; + } + + public ObjectInspector getOI() + { + return OI; + } + + public void setOI(ObjectInspector oI) + { + OI = oI; + } + + public ExprNodeEvaluator getExprEvaluator() + { + return exprEvaluator; + } + + public void setExprEvaluator(ExprNodeEvaluator exprEvaluator) + { + this.exprEvaluator = exprEvaluator; + } + + } + + /* + * OrderColumnDef class + */ + public static class OrderColumnDef extends ColumnDef + { + + public OrderColumnDef(){ + + } + + public OrderColumnDef(OrderColumnSpec spec) + { + super(spec); + } + + public OrderColumnDef(ColumnDef cDef) + { + super(cDef); + this.spec = new OrderColumnSpec(spec); + } + + public Order getOrder() + { + return ((OrderColumnSpec)getSpec()).getOrder(); + } + + } + + /* + * Window Frame classes + */ + public static class WindowFrameDef + { + WindowFrameSpec spec; + BoundaryDef start; + BoundaryDef end; + + public WindowFrameDef(){ + + } + + public WindowFrameDef(WindowFrameSpec spec) { this.spec = spec; } + + public BoundaryDef getStart() + { + return start; + } + + public void setStart(BoundaryDef start) + { + this.start = start; + } + + public BoundaryDef getEnd() + { + return end; + } + + public void setEnd(BoundaryDef end) + { + this.end = end; + } + + public abstract static class BoundaryDef implements Comparable + { + BoundarySpec spec; + + public BoundaryDef(){} + + public BoundarySpec getSpec() { + return spec; + } + + public void setSpec(BoundarySpec spec) { + this.spec = spec; + } + + public BoundaryDef(BoundarySpec spec) + { + this.spec = spec; + } + + public Direction getDirection() { return spec.getDirection(); } + } + + public static class RangeBoundaryDef extends BoundaryDef + { + public RangeBoundaryDef(){} + + public RangeBoundaryDef(RangeBoundarySpec spec) { super(spec);} + + public int getAmt() + { + return ((RangeBoundarySpec)spec).getAmt(); + } + + public int compareTo(BoundaryDef other) + { + int c = getDirection().compareTo(other.getDirection()); + if ( c != 0) { + return c; + } + RangeBoundaryDef rb = (RangeBoundaryDef) other; + return getAmt() - rb.getAmt(); + } + } + + public static class CurrentRowDef extends BoundaryDef + { + + public CurrentRowDef(){} + + public CurrentRowDef(CurrentRowSpec spec) { super(spec);} + + public int compareTo(BoundaryDef other) + { + return getDirection().compareTo(other.getDirection()); + } + } + + public static class ValueBoundaryDef extends BoundaryDef + { + ExprNodeDesc exprNode; + transient ExprNodeEvaluator exprEvaluator; + transient ObjectInspector OI; + + static{ + PTFUtils.makeTransient(ValueBoundaryDef.class, "exprEvaluator"); + PTFUtils.makeTransient(ValueBoundaryDef.class, "OI"); + } + + public ValueBoundaryDef(){ } + + public ValueBoundaryDef(ValueBoundarySpec spec) { super(spec);} + + + public int getAmt() + { + return ((ValueBoundarySpec)spec).getAmt(); + } + + public ExprNodeDesc getExprNode() + { + return exprNode; + } + + public void setExprNode(ExprNodeDesc exprNode) + { + this.exprNode = exprNode; + } + + public ExprNodeEvaluator getExprEvaluator() + { + return exprEvaluator; + } + + + public void setExprEvaluator(ExprNodeEvaluator exprEvaluator) + { + this.exprEvaluator = exprEvaluator; + } + + public ObjectInspector getOI() + { + return OI; + } + + + public void setOI(ObjectInspector oI) + { + OI = oI; + } + + public int compareTo(BoundaryDef other) + { + int c = getDirection().compareTo(other.getDirection()); + if ( c != 0) { + return c; + } + ValueBoundaryDef vb = (ValueBoundaryDef) other; + return getAmt() - vb.getAmt(); + } + } + + } + + /* + * SelectDef class + */ + public static class SelectDef + { + SelectSpec selectSpec; + ArrayList windowFuncs; + ArrayList columns; + transient StructObjectInspector OI; + transient RowResolver rowResolver; + + static{ + PTFUtils.makeTransient(SelectDef.class, "OI"); + PTFUtils.makeTransient(SelectDef.class, "rowResolver"); + } + + + public SelectDef(){ + + } + + public SelectSpec getSelectSpec() + { + return selectSpec; + } + + public void setSelectSpec(SelectSpec selectSpec) + { + this.selectSpec = selectSpec; + } + + public ArrayList getWindowFuncs() + { + return windowFuncs; + } + + public void setWindowFuncs(ArrayList windowFuncs) + { + this.windowFuncs = windowFuncs; + } + + public ArrayList getColumns() + { + return columns; + } + + public void setColumns(ArrayList columns) + { + this.columns = columns; + } + + public void addColumn(ColumnDef cDef) + { + columns = columns == null ? new ArrayList() : columns; + columns.add(cDef); + } + + public StructObjectInspector getOI() + { + return OI; + } + + public void setOI(StructObjectInspector oI) + { + OI = oI; + } + + public RowResolver getRowResolver() { + return rowResolver; + } + + public void setRowResolver(RowResolver rowResolver) { + this.rowResolver = rowResolver; + } + } + + public static class WhereDef + { + ASTNode expression; + ExprNodeDesc exprNode; + transient ExprNodeEvaluator exprEvaluator; + transient ObjectInspector OI; + + static{ + PTFUtils.makeTransient(WhereDef.class, "exprEvaluator"); + PTFUtils.makeTransient(WhereDef.class, "OI"); + } + + public WhereDef(){} + + public ASTNode getExpression() + { + return expression; + } + + public void setExpression(ASTNode expression) + { + this.expression = expression; + } + + public ExprNodeDesc getExprNode() + { + return exprNode; + } + + public void setExprNode(ExprNodeDesc exprNode) + { + this.exprNode = exprNode; + } + + public ExprNodeEvaluator getExprEvaluator() + { + return exprEvaluator; + } + + public void setExprEvaluator(ExprNodeEvaluator exprEvaluator) + { + this.exprEvaluator = exprEvaluator; + } + + public ObjectInspector getOI() + { + return OI; + } + + public void setOI(ObjectInspector oI) + { + OI = oI; + } + } + +} \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java new file mode 100644 index 0000000..839e5ea --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java @@ -0,0 +1,30 @@ +package org.apache.hadoop.hive.ql.plan; + +@Explain(displayName = "PTF Operator") +public class PTFDesc extends AbstractOperatorDesc +{ + + private static final long serialVersionUID = 1L; + String queryDefStr; + + public PTFDesc() + { + + } + + public PTFDesc(String queryDefStr) + { + this.queryDefStr = queryDefStr; + } + + public String getQueryDefStr() + { + return queryDefStr; + } + + public void setQueryDefStr(String queryDefStr) + { + this.queryDefStr = queryDefStr; + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCumeDist.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCumeDist.java new file mode 100644 index 0000000..98b86fd --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCumeDist.java @@ -0,0 +1,65 @@ +package org.apache.hadoop.hive.ql.udf.generic; + +import java.util.ArrayList; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.WindowFunctionDescription; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.IntWritable; + +@WindowFunctionDescription +( + description = @Description( + name = "cumedist", + value = "_FUNC_(x) - The CUME_DIST function (defined as the inverse of percentile in some " + + "statistical books) computes the position of a specified value relative to a set of values. " + + "To compute the CUME_DIST of a value x in a set S of size N, you use the formula: " + + "CUME_DIST(x) = number of values in S coming before " + + " and including x in the specified order/ N" + ), + supportsWindow = false, + pivotResult = true +) +public class GenericUDAFCumeDist extends GenericUDAFRank +{ + + static final Log LOG = LogFactory.getLog(GenericUDAFCumeDist.class.getName()); + + @Override + protected GenericUDAFRankEvaluator createEvaluator() + { + return new GenericUDAFCumeDistEvaluator(); + } + + public static class GenericUDAFCumeDistEvaluator extends GenericUDAFRankEvaluator + { + @Override + public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException + { + super.init(m, parameters); + return ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector); + } + + @Override + public Object terminate(AggregationBuffer agg) throws HiveException + { + ArrayList ranks = ((RankBuffer) agg).rowNums; + double sz = ranks.size(); + ArrayList pranks = new ArrayList(ranks.size()); + + for(IntWritable i : ranks) + { + double pr = ((double)i.get())/sz; + pranks.add(new DoubleWritable(pr)); + } + + return pranks; + } + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFDenseRank.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFDenseRank.java new file mode 100644 index 0000000..9096368 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFDenseRank.java @@ -0,0 +1,42 @@ +package org.apache.hadoop.hive.ql.udf.generic; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.WindowFunctionDescription; + +@WindowFunctionDescription +( + description = @Description( + name = "denserank", + value = "_FUNC_(x) The difference between RANK and DENSE_RANK is that DENSE_RANK leaves no " + + "gaps in ranking sequence when there are ties. That is, if you were " + + "ranking a competition using DENSE_RANK and had three people tie for " + + "second place, you would say that all three were in second place and " + + "that the next person came in third." + ), + supportsWindow = false, + pivotResult = true +) +public class GenericUDAFDenseRank extends GenericUDAFRank +{ + static final Log LOG = LogFactory.getLog(GenericUDAFDenseRank.class.getName()); + + @Override + protected GenericUDAFRankEvaluator createEvaluator() + { + return new GenericUDAFDenseRankEvaluator(); + } + + public static class GenericUDAFDenseRankEvaluator extends GenericUDAFRankEvaluator + { + /* + * Called when the value in the partition has changed. Update the currentRank + */ + @Override + protected void nextRank(RankBuffer rb) + { + rb.currentRank++; + } + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFFirstValue.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFFirstValue.java new file mode 100644 index 0000000..f11a69f --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFFirstValue.java @@ -0,0 +1,149 @@ +package org.apache.hadoop.hive.ql.udf.generic; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.exec.WindowFunctionDescription; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AggregationBuffer; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; + +@WindowFunctionDescription +( + description = @Description( + name = "first_value", + value = "_FUNC_(x)" + ), + supportsWindow = true, + pivotResult = false +) +public class GenericUDAFFirstValue extends AbstractGenericUDAFResolver +{ + static final Log LOG = LogFactory.getLog(GenericUDAFFirstValue.class.getName()); + + @Override + public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticException + { + if (parameters.length > 2) + { + throw new UDFArgumentTypeException(2, "At most 2 arguments expected"); + } + if ( parameters.length > 1 && !parameters[1].equals(TypeInfoFactory.booleanTypeInfo) ) + { + throw new UDFArgumentTypeException(1, "second argument must be a boolean expression"); + } + return createEvaluator(); + } + + protected GenericUDAFFirstValueEvaluator createEvaluator() + { + return new GenericUDAFFirstValueEvaluator(); + } + + static class FirstValueBuffer implements AggregationBuffer + { + Object val; + boolean valSet; + boolean firstRow; + boolean skipNulls; + + FirstValueBuffer() + { + init(); + } + + void init() + { + val = null; + valSet = false; + firstRow = true; + skipNulls = false; + } + + } + + public static class GenericUDAFFirstValueEvaluator extends GenericUDAFEvaluator + { + ObjectInspector inputOI; + ObjectInspector outputOI; + + @Override + public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException + { + super.init(m, parameters); + if (m != Mode.COMPLETE) + { + throw new HiveException( + "Only COMPLETE mode supported for Rank function"); + } + inputOI = parameters[0]; + outputOI = ObjectInspectorUtils.getStandardObjectInspector(inputOI, ObjectInspectorCopyOption.WRITABLE); + return outputOI; + } + + @Override + public AggregationBuffer getNewAggregationBuffer() throws HiveException + { + return new FirstValueBuffer(); + } + + @Override + public void reset(AggregationBuffer agg) throws HiveException + { + ((FirstValueBuffer) agg).init(); + } + + @Override + public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveException + { + FirstValueBuffer fb = (FirstValueBuffer) agg; + + if (fb.firstRow ) + { + fb.firstRow = false; + if ( parameters.length == 2 ) + { + fb.skipNulls = PrimitiveObjectInspectorUtils.getBoolean( + parameters[1], + PrimitiveObjectInspectorFactory.writableBooleanObjectInspector); + } + } + + if ( !fb.valSet ) + { + fb.val = ObjectInspectorUtils.copyToStandardObject(parameters[0], inputOI, ObjectInspectorCopyOption.WRITABLE); + if ( !fb.skipNulls || fb.val != null ) + { + fb.valSet = true; + } + } + } + + @Override + public Object terminatePartial(AggregationBuffer agg) throws HiveException + { + throw new HiveException("terminatePartial not supported"); + } + + @Override + public void merge(AggregationBuffer agg, Object partial) throws HiveException + { + throw new HiveException("merge not supported"); + } + + @Override + public Object terminate(AggregationBuffer agg) throws HiveException + { + return ((FirstValueBuffer) agg).val; + } + + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLastValue.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLastValue.java new file mode 100644 index 0000000..87e5d33 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLastValue.java @@ -0,0 +1,145 @@ +package org.apache.hadoop.hive.ql.udf.generic; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.exec.WindowFunctionDescription; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AggregationBuffer; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; + +@WindowFunctionDescription(description = @Description(name = "last_value", value = "_FUNC_(x)"), supportsWindow = true, pivotResult = false) +public class GenericUDAFLastValue extends AbstractGenericUDAFResolver +{ + static final Log LOG = LogFactory.getLog(GenericUDAFLastValue.class + .getName()); + + @Override + public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) + throws SemanticException + { + if (parameters.length > 2) + { + throw new UDFArgumentTypeException(2, "At most 2 arguments expected"); + } + if ( parameters.length > 1 && !parameters[1].equals(TypeInfoFactory.booleanTypeInfo) ) + { + throw new UDFArgumentTypeException(1, "second argument must be a boolean expression"); + } + return createEvaluator(); + } + + protected GenericUDAFLastValueEvaluator createEvaluator() + { + return new GenericUDAFLastValueEvaluator(); + } + + static class LastValueBuffer implements AggregationBuffer + { + Object val; + boolean firstRow; + boolean skipNulls; + + LastValueBuffer() + { + init(); + } + + void init() + { + val = null; + firstRow = true; + skipNulls = false; + } + + } + + public static class GenericUDAFLastValueEvaluator extends + GenericUDAFEvaluator + { + ObjectInspector inputOI; + ObjectInspector outputOI; + + @Override + public ObjectInspector init(Mode m, ObjectInspector[] parameters) + throws HiveException + { + super.init(m, parameters); + if (m != Mode.COMPLETE) + { + throw new HiveException( + "Only COMPLETE mode supported for Rank function"); + } + inputOI = parameters[0]; + outputOI = ObjectInspectorUtils.getStandardObjectInspector(inputOI, + ObjectInspectorCopyOption.WRITABLE); + return outputOI; + } + + @Override + public AggregationBuffer getNewAggregationBuffer() throws HiveException + { + return new LastValueBuffer(); + } + + @Override + public void reset(AggregationBuffer agg) throws HiveException + { + ((LastValueBuffer) agg).init(); + } + + @Override + public void iterate(AggregationBuffer agg, Object[] parameters) + throws HiveException + { + LastValueBuffer lb = (LastValueBuffer) agg; + if (lb.firstRow ) + { + lb.firstRow = false; + if ( parameters.length == 2 ) + { + lb.skipNulls = PrimitiveObjectInspectorUtils.getBoolean( + parameters[1], + PrimitiveObjectInspectorFactory.writableBooleanObjectInspector); + } + } + + if ( !lb.skipNulls || lb.val != null ) + { + lb.val = parameters[0]; + } + } + + @Override + public Object terminatePartial(AggregationBuffer agg) + throws HiveException + { + throw new HiveException("terminatePartial not supported"); + } + + @Override + public void merge(AggregationBuffer agg, Object partial) + throws HiveException + { + throw new HiveException("merge not supported"); + } + + @Override + public Object terminate(AggregationBuffer agg) throws HiveException + { + LastValueBuffer lb = (LastValueBuffer) agg; + return ObjectInspectorUtils.copyToStandardObject(lb.val, inputOI, + ObjectInspectorCopyOption.WRITABLE); + + } + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFNTile.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFNTile.java new file mode 100644 index 0000000..8ad4979 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFNTile.java @@ -0,0 +1,162 @@ +package org.apache.hadoop.hive.ql.udf.generic; + +import java.util.ArrayList; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.exec.WindowFunctionDescription; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AggregationBuffer; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.apache.hadoop.io.IntWritable; + +@WindowFunctionDescription +( + description = @Description( + name = "rank", + value = "_FUNC_(x) NTILE allows easy calculation of tertiles, quartiles, deciles and other " + + "common summary statistics. This function divides an ordered partition into a specified " + + "number of groups called buckets and assigns a bucket number to each row in the partition." + ), + supportsWindow = false, + pivotResult = true +) +public class GenericUDAFNTile extends AbstractGenericUDAFResolver +{ + static final Log LOG = LogFactory.getLog(GenericUDAFNTile.class.getName()); + + @Override + public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticException + { + if (parameters.length != 1) + { + throw new UDFArgumentTypeException(parameters.length - 1, "Exactly one argument is expected."); + } + ObjectInspector oi = TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(parameters[0]); + + boolean c = ObjectInspectorUtils.compareTypes(oi, PrimitiveObjectInspectorFactory.writableIntObjectInspector); + if (!c) + { + throw new UDFArgumentTypeException(0, "Number of tiles must be an int expression"); + } + + return new GenericUDAFNTileEvaluator(); + } + + static class NTileBuffer implements AggregationBuffer + { + Integer numBuckets; + int numRows; + + void init() + { + numBuckets = null; + numRows = 0; + } + + NTileBuffer() + { + init(); + } + } + + public static class GenericUDAFNTileEvaluator extends GenericUDAFEvaluator + { + PrimitiveObjectInspector inputOI; + + @Override + public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException + { + assert (parameters.length == 1); + super.init(m, parameters); + if (m != Mode.COMPLETE) + { + throw new HiveException( + "Only COMPLETE mode supported for NTile function"); + } + inputOI = (PrimitiveObjectInspector) parameters[0]; + return ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableIntObjectInspector); + } + + @Override + public AggregationBuffer getNewAggregationBuffer() throws HiveException + { + return new NTileBuffer(); + } + + @Override + public void reset(AggregationBuffer agg) throws HiveException + { + ((NTileBuffer) agg).init(); + } + + @Override + public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveException + { + NTileBuffer rb = (NTileBuffer) agg; + if ( rb.numBuckets == null) + { + rb.numBuckets = PrimitiveObjectInspectorUtils.getInt(parameters[0], inputOI); + } + rb.numRows++; + } + + @Override + public Object terminatePartial(AggregationBuffer agg) throws HiveException + { + throw new HiveException("terminatePartial not supported"); + } + + @Override + public void merge(AggregationBuffer agg, Object partial) throws HiveException + { + throw new HiveException("merge not supported"); + } + + @Override + public Object terminate(AggregationBuffer agg) throws HiveException + { + NTileBuffer rb = (NTileBuffer) agg; + ArrayList res = new ArrayList(rb.numRows); + + /* + * if there is a remainder from numRows/numBuckets; then distribute increase the size of the first 'rem' buckets by 1. + */ + + int bucketsz = rb.numRows / rb.numBuckets; + int rem = rb.numRows % rb.numBuckets; + int start = 0; + int bucket = 1; + while ( start < rb.numRows) + { + int end = start + bucketsz; + if (rem > 0) + { + end++; rem--; + } + end = Math.min(rb.numRows, end); + for(int i = start; i < end; i++) + { + res.add(new IntWritable(bucket)); + } + start = end; + bucket++; + } + + return res; + } + + } + + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentRank.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentRank.java new file mode 100644 index 0000000..b7db49b --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentRank.java @@ -0,0 +1,65 @@ +package org.apache.hadoop.hive.ql.udf.generic; + +import java.util.ArrayList; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.WindowFunctionDescription; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.IntWritable; + +@WindowFunctionDescription +( + description = @Description( + name = "percentrank", + value = "_FUNC_(x) PERCENT_RANK is similar to CUME_DIST, but it uses rank values rather " + + "than row counts in its numerator. PERCENT_RANK of a row is calculated as: " + + "(rank of row in its partition - 1) / (number of rows in the partition - 1)" + ), + supportsWindow = false, + pivotResult = true +) +public class GenericUDAFPercentRank extends GenericUDAFRank +{ + static final Log LOG = LogFactory.getLog(GenericUDAFPercentRank.class.getName()); + + @Override + protected GenericUDAFRankEvaluator createEvaluator() + { + return new GenericUDAFPercentRankEvaluator(); + } + + public static class GenericUDAFPercentRankEvaluator extends GenericUDAFRankEvaluator + { + @Override + public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException + { + super.init(m, parameters); + return ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector); + } + + @Override + public Object terminate(AggregationBuffer agg) throws HiveException + { + ArrayList ranks = ((RankBuffer) agg).rowNums; + double sz = ranks.size(); + if ( sz > 1 ) { + sz = sz - 1; + } + ArrayList pranks = new ArrayList(ranks.size()); + + for(IntWritable i : ranks) + { + double pr = ((double)i.get() - 1)/sz; + pranks.add(new DoubleWritable(pr)); + } + + return pranks; + } + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFRank.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFRank.java new file mode 100644 index 0000000..5b7c71f --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFRank.java @@ -0,0 +1,168 @@ +package org.apache.hadoop.hive.ql.udf.generic; + +import java.util.ArrayList; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.exec.WindowFunctionDescription; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.parse.PTFTranslator; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AggregationBuffer; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.apache.hadoop.io.IntWritable; + +@WindowFunctionDescription +( + description = @Description( + name = "rank", + value = "_FUNC_(x)" + ), + supportsWindow = false, + pivotResult = true +) +public class GenericUDAFRank extends AbstractGenericUDAFResolver +{ + static final Log LOG = LogFactory.getLog(GenericUDAFRank.class.getName()); + + @Override + public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticException + { + if (parameters.length < 1) + { + throw new UDFArgumentTypeException(parameters.length - 1, "One or more arguments are expected."); + } + for(int i=0; i type or complex type containing map<>."); + } + } + return createEvaluator(); + } + + protected GenericUDAFRankEvaluator createEvaluator() + { + return new GenericUDAFRankEvaluator(); + } + + static class RankBuffer implements AggregationBuffer + { + ArrayList rowNums; + int currentRowNum; + Object[] currVal; + int currentRank; + int numParams; + + RankBuffer(int numParams) + { + this.numParams = numParams; + init(); + } + + void init() + { + rowNums = new ArrayList(); + currentRowNum = 0; + currentRank = 0; + currVal = new Object[numParams]; + } + + void incrRowNum() { currentRowNum++; } + + void addRank() + { + rowNums.add(new IntWritable(currentRank)); + } + } + + public static class GenericUDAFRankEvaluator extends GenericUDAFEvaluator + { + ObjectInspector[] inputOI; + ObjectInspector[] outputOI; + + @Override + public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException + { + super.init(m, parameters); + if (m != Mode.COMPLETE) + { + throw new HiveException( + "Only COMPLETE mode supported for Rank function"); + } + inputOI = parameters; + outputOI = new ObjectInspector[inputOI.length]; + for(int i=0; i < inputOI.length; i++) + { + outputOI[i] = ObjectInspectorUtils.getStandardObjectInspector(inputOI[i], ObjectInspectorCopyOption.JAVA); + } + return ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableIntObjectInspector); + } + + @Override + public AggregationBuffer getNewAggregationBuffer() throws HiveException + { + return new RankBuffer(inputOI.length); + } + + @Override + public void reset(AggregationBuffer agg) throws HiveException + { + ((RankBuffer) agg).init(); + } + + @Override + public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveException + { + RankBuffer rb = (RankBuffer) agg; + int c = PTFTranslator.compare(rb.currVal, outputOI, parameters, inputOI); + rb.incrRowNum(); + if ( rb.currentRowNum == 1 || c != 0 ) + { + nextRank(rb); + rb.currVal = PTFTranslator.copyToStandardObject(parameters, inputOI, ObjectInspectorCopyOption.JAVA); + } + rb.addRank(); + } + + /* + * Called when the value in the partition has changed. Update the currentRank + */ + protected void nextRank(RankBuffer rb) + { + rb.currentRank = rb.currentRowNum; + } + + @Override + public Object terminatePartial(AggregationBuffer agg) throws HiveException + { + throw new HiveException("terminatePartial not supported"); + } + + @Override + public void merge(AggregationBuffer agg, Object partial) throws HiveException + { + throw new HiveException("merge not supported"); + } + + @Override + public Object terminate(AggregationBuffer agg) throws HiveException + { + return ((RankBuffer) agg).rowNums; + } + + } + + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFRowNumber.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFRowNumber.java new file mode 100644 index 0000000..ac32d34 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFRowNumber.java @@ -0,0 +1,124 @@ +package org.apache.hadoop.hive.ql.udf.generic; + +import java.util.ArrayList; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.exec.WindowFunctionDescription; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AggregationBuffer; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.io.IntWritable; + +@WindowFunctionDescription +( + description = @Description( + name = "rowNumber", + value = "_FUNC_() - The ROW_NUMBER function assigns a unique number (sequentially, starting from 1, as defined by ORDER BY) to each row within the partition." + ), + supportsWindow = false, + pivotResult = true +) +public class GenericUDAFRowNumber extends AbstractGenericUDAFResolver +{ + static final Log LOG = LogFactory.getLog(GenericUDAFRowNumber.class.getName()); + + @Override + public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) + throws SemanticException + { + if (parameters.length != 0) + { + throw new UDFArgumentTypeException(parameters.length - 1, + "No argument is expected."); + } + return new GenericUDAFRowNumberEvaluator(); + } + + static class RowNumberBuffer implements AggregationBuffer + { + ArrayList rowNums; + int nextRow; + + void init() + { + rowNums = new ArrayList(); + } + + RowNumberBuffer() + { + init(); + nextRow = 1; + } + + void incr() + { + rowNums.add(new IntWritable(nextRow++)); + } + } + + public static class GenericUDAFRowNumberEvaluator extends + GenericUDAFEvaluator + { + + @Override + public ObjectInspector init(Mode m, ObjectInspector[] parameters) + throws HiveException + { + super.init(m, parameters); + if (m != Mode.COMPLETE) + { + throw new HiveException("Only COMPLETE mode supported for RowNumber function"); + } + + return ObjectInspectorFactory.getStandardListObjectInspector( + PrimitiveObjectInspectorFactory.writableIntObjectInspector); + } + + @Override + public AggregationBuffer getNewAggregationBuffer() throws HiveException + { + return new RowNumberBuffer(); + } + + @Override + public void reset(AggregationBuffer agg) throws HiveException + { + ((RowNumberBuffer) agg).init(); + } + + @Override + public void iterate(AggregationBuffer agg, Object[] parameters) + throws HiveException + { + ((RowNumberBuffer) agg).incr(); + } + + @Override + public Object terminatePartial(AggregationBuffer agg) + throws HiveException + { + throw new HiveException("terminatePartial not supported"); + } + + @Override + public void merge(AggregationBuffer agg, Object partial) + throws HiveException + { + throw new HiveException("merge not supported"); + } + + @Override + public Object terminate(AggregationBuffer agg) throws HiveException + { + return ((RowNumberBuffer) agg).rowNums; + } + + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLeadLag.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLeadLag.java new file mode 100644 index 0000000..5e6a4e9 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLeadLag.java @@ -0,0 +1,158 @@ +package org.apache.hadoop.hive.ql.udf.generic; + +import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; +import org.apache.hadoop.hive.ql.exec.PTFPartition.PTFPartitionIterator; +import org.apache.hadoop.hive.ql.exec.PTFUtils; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; + +public abstract class GenericUDFLeadLag extends GenericUDF +{ + transient ExprNodeEvaluator exprEvaluator; + transient PTFPartitionIterator pItr; + ObjectInspector firstArgOI; + + private PrimitiveObjectInspector amtOI; + + static{ + PTFUtils.makeTransient(GenericUDFLeadLag.class, "exprEvaluator"); + PTFUtils.makeTransient(GenericUDFLeadLag.class, "pItr"); + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException + { + DeferredObject amt = arguments[1]; + int intAmt = 0; + try + { + intAmt = PrimitiveObjectInspectorUtils.getInt(amt.get(), amtOI); + } + catch (NullPointerException e) + { + intAmt = Integer.MAX_VALUE; + } + catch (NumberFormatException e) + { + intAmt = Integer.MAX_VALUE; + } + + int idx = pItr.getIndex() - 1; + try + { + Object row = getRow(intAmt); + Object ret = exprEvaluator.evaluate(row); + ret = ObjectInspectorUtils.copyToStandardObject(ret, firstArgOI, ObjectInspectorCopyOption.WRITABLE); + return ret; + } + finally + { + Object currRow = pItr.resetToIndex(idx); + // reevaluate expression on current Row, to trigger the Lazy object + // caches to be reset to the current row. + exprEvaluator.evaluate(currRow); + } + + } + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) + throws UDFArgumentException + { + // index has to be a primitive + if (arguments[1] instanceof PrimitiveObjectInspector) + { + amtOI = (PrimitiveObjectInspector) arguments[1]; + } + else + { + throw new UDFArgumentTypeException(1, + "Primitive Type is expected but " + + arguments[1].getTypeName() + "\" is found"); + } + + firstArgOI = arguments[0]; + return ObjectInspectorUtils.getStandardObjectInspector(firstArgOI, + ObjectInspectorCopyOption.WRITABLE); + } + + + + public ExprNodeEvaluator getExprEvaluator() + { + return exprEvaluator; + } + + public void setExprEvaluator(ExprNodeEvaluator exprEvaluator) + { + this.exprEvaluator = exprEvaluator; + } + + public PTFPartitionIterator getpItr() + { + return pItr; + } + + public void setpItr(PTFPartitionIterator pItr) + { + this.pItr = pItr; + } + + @Override + public String getDisplayString(String[] children) + { + assert (children.length == 2); + StringBuilder sb = new StringBuilder(); + sb.append(_getFnName()); + sb.append("("); + sb.append(children[0]); + sb.append(", "); + sb.append(children[1]); + sb.append(")"); + return sb.toString(); + } + + protected abstract String _getFnName(); + + protected abstract Object getRow(int amt); + + public static class GenericUDFLead extends GenericUDFLeadLag + { + + @Override + protected String _getFnName() + { + return "lead"; + } + + @Override + protected Object getRow(int amt) + { + return pItr.lead(amt - 1); + } + + } + + public static class GenericUDFLag extends GenericUDFLeadLag + { + @Override + protected String _getFnName() + { + return "lag"; + } + + @Override + protected Object getRow(int amt) + { + return pItr.lag(amt + 1); + } + + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NPath.java ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NPath.java new file mode 100644 index 0000000..91d105d --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NPath.java @@ -0,0 +1,820 @@ +package org.apache.hadoop.hive.ql.udf.ptf; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; + +import org.apache.hadoop.hive.ql.exec.ColumnInfo; +import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; +import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory; +import org.apache.hadoop.hive.ql.exec.PTFPartition; +import org.apache.hadoop.hive.ql.exec.PTFPartition.PTFPartitionIterator; +import org.apache.hadoop.hive.ql.exec.PTFUtils; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.PTFSpec.SelectSpec; +import org.apache.hadoop.hive.ql.parse.PTFTranslator; +import org.apache.hadoop.hive.ql.parse.RowResolver; +import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.parse.TypeCheckCtx; +import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.PTFDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.ArgDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.PTFInputDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.TableFuncDef; +import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; + +/** + * return rows that meet a specified pattern. Use symbols to specify a list of expressions to match. + * Pattern is used to specify a Path. The results list can contain expressions based on the input columns + * and also the matched Path. + *
    + *
  1. pattern: pattern for the Path. Path is 'dot' separated list of symbols. + * Each element is treated as a symbol. Elements that end in '*' or '+' are interpreted with the + * usual meaning of zero or more, one or more respectively. For e.g. "LATE.EARLY*.ONTIMEOREARLY" implies a sequence of flights + * where the first occurrence was LATE, followed by zero or more EARLY flights, followed by a ONTIME or EARLY flight. + *
  2. symbols specify a list of name, expression pairs. For e.g. + * 'LATE', arrival_delay > 0, 'EARLY', arrival_delay < 0 , 'ONTIME', arrival_delay == 0. + * These symbols can be used in the Pattern defined above. + *
  3. resultSelectList specified as a select list. + * The expressions in the selectList are evaluated in the context where all the input columns are available, plus the attribute + * "tpath" is available. Path is a collection of rows that represents the matching Path. + *
+ */ +public class NPath extends TableFunctionEvaluator +{ + private transient String patternStr; + private transient SymbolsInfo symInfo; + private transient String resultExprStr; + private transient SymbolFunction syFn; + private ResultExprInfo resultExprInfo; + /* + * the names of the Columns of the input to NPath. Used to setup the tpath Struct column. + */ + private ArrayList inputColumnNames; + + @Override + public void execute(PTFPartitionIterator pItr, PTFPartition outP) throws HiveException + { + while (pItr.hasNext()) + { + Object iRow = pItr.next(); + + SymbolFunctionResult syFnRes = SymbolFunction.match(syFn, iRow, pItr); + if (syFnRes.matches ) + { + int sz = syFnRes.nextRow - (pItr.getIndex() - 1); + Object selectListInput = NPath.getSelectListInput(iRow, tDef.getInput().getOI(), pItr, sz); + ArrayList oRow = new ArrayList(); + for(ExprNodeEvaluator resExprEval : resultExprInfo.resultExprEvals) + { + oRow.add(resExprEval.evaluate(selectListInput)); + } + outP.append(oRow); + } + } + } + + static void throwErrorWithSignature(String message) throws SemanticException + { + throw new SemanticException(PTFUtils.sprintf( + "NPath signature is: SymbolPattern, one or more SymbolName, expression pairs, the result expression as a select list. Error %s", + message)); + } + public static class NPathResolver extends TableFunctionResolver + { + + @Override + protected TableFunctionEvaluator createEvaluator(PTFDef qDef, TableFuncDef tDef) + { + + return new NPath(); + } + + /** + *
    + *
  • check structure of Arguments: + *
      + *
    1. First arg should be a String + *
    2. then there should be an even number of Arguments: String, expression; expression should be Convertible to Boolean. + *
    3. finally there should be a String. + *
    + *
  • convert pattern into a NNode chain. + *
  • convert symbol args into a Symbol Map. + *
  • parse selectList into SelectList struct. The inputOI used to translate these expressions should be based on the + * columns in the Input, the 'path.attr' + *
+ */ + @Override + public void setupOutputOI() throws SemanticException + { + NPath evaluator = (NPath) getEvaluator(); + TableFuncDef tDef = evaluator.getTableDef(); + + ArrayList args = tDef.getArgs(); + int argsNum = args == null ? 0 : args.size(); + + if ( argsNum < 4 ) + { + throwErrorWithSignature("at least 4 arguments required"); + } + + validateAndSetupPatternStr(evaluator, args); + validateAndSetupSymbolInfo(evaluator, args, argsNum); + validateAndSetupResultExprStr(evaluator, args, argsNum); + setupSymbolFunctionChain(evaluator); + + /* + * setup OI for input to resultExpr select list + */ + RowResolver selectListInputRR = NPath.createSelectListRR(evaluator, tDef.getInput()); + + /* + * parse ResultExpr Str and setup OI. + */ + ResultExpressionParser resultExprParser = new ResultExpressionParser(evaluator.resultExprStr, selectListInputRR); + try { + resultExprParser.translate(); + } + catch(HiveException he) { + throw new SemanticException(he); + } + evaluator.resultExprInfo = resultExprParser.getResultExprInfo(); + StructObjectInspector OI = evaluator.resultExprInfo.resultOI; + setOutputOI(OI); + } + /* + * validate and setup patternStr + */ + private void validateAndSetupPatternStr(NPath evaluator, ArrayList args) throws SemanticException { + ArgDef symboPatternArg = args.get(0); + ObjectInspector symbolPatternArgOI = symboPatternArg.getOI(); + + if ( !ObjectInspectorUtils.isConstantObjectInspector(symbolPatternArgOI) || + (symbolPatternArgOI.getCategory() != ObjectInspector.Category.PRIMITIVE) || + ((PrimitiveObjectInspector)symbolPatternArgOI).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.STRING ) + { + throwErrorWithSignature("Currently the symbol Pattern must be a Constant String."); + } + + evaluator.patternStr = ((ConstantObjectInspector)symbolPatternArgOI).getWritableConstantValue().toString(); + } + + /* + * validate and setup SymbolInfo + */ + private void validateAndSetupSymbolInfo(NPath evaluator, ArrayList args, int argsNum) throws SemanticException { + int symbolArgsSz = argsNum - 2; + if ( symbolArgsSz % 2 != 0) + { + throwErrorWithSignature("Symbol Name, Expression need to be specified in pairs: there are odd number of symbol args"); + } + + evaluator.symInfo = new SymbolsInfo(symbolArgsSz/2); + for(int i=1; i <= symbolArgsSz; i += 2) + { + ArgDef symbolNameArg = args.get(i); + ObjectInspector symbolNameArgOI = symbolNameArg.getOI(); + + if ( !ObjectInspectorUtils.isConstantObjectInspector(symbolNameArgOI) || + (symbolNameArgOI.getCategory() != ObjectInspector.Category.PRIMITIVE) || + ((PrimitiveObjectInspector)symbolNameArgOI).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.STRING ) + { + throwErrorWithSignature(PTFUtils.sprintf("Currently a Symbol Name(%s) must be a Constant String", symbolNameArg.getExpression().toStringTree())); + } + String symbolName = ((ConstantObjectInspector)symbolNameArgOI).getWritableConstantValue().toString(); + + ArgDef symolExprArg = args.get(i+1); + ObjectInspector symolExprArgOI = symolExprArg.getOI(); + if ( (symolExprArgOI.getCategory() != ObjectInspector.Category.PRIMITIVE) || + ((PrimitiveObjectInspector)symolExprArgOI).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.BOOLEAN ) + { + throwErrorWithSignature(PTFUtils.sprintf("Currently a Symbol Expression(%s) must be a boolean expression", symolExprArg.getExpression().toStringTree())); + } + evaluator.symInfo.add(symbolName, symolExprArg); + } + } + + /* + * validate and setup resultExprStr + */ + private void validateAndSetupResultExprStr(NPath evaluator, ArrayList args, int argsNum) throws SemanticException { + ArgDef resultExprArg = args.get(argsNum - 1); + ObjectInspector resultExprArgOI = resultExprArg.getOI(); + + if ( !ObjectInspectorUtils.isConstantObjectInspector(resultExprArgOI) || + (resultExprArgOI.getCategory() != ObjectInspector.Category.PRIMITIVE) || + ((PrimitiveObjectInspector)resultExprArgOI).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.STRING ) + { + throwErrorWithSignature("Currently the result Expr parameter must be a Constant String."); + } + + evaluator.resultExprStr = ((ConstantObjectInspector)resultExprArgOI).getWritableConstantValue().toString(); + } + + /* + * setup SymbolFunction chain. + */ + private void setupSymbolFunctionChain(NPath evaluator) throws SemanticException { + SymbolParser syP = new SymbolParser(evaluator.patternStr, + evaluator.symInfo.symbolExprsNames, + evaluator.symInfo.symbolExprsEvaluators, evaluator.symInfo.symbolExprsOIs); + syP.parse(); + evaluator.syFn = syP.getSymbolFunction(); + } + + @Override + public boolean transformsRawInput() + { + return false; + } + + @Override + public void initializeOutputOI() throws HiveException { + try { + NPath evaluator = (NPath) getEvaluator(); + TableFuncDef tDef = evaluator.getTableDef(); + + ArrayList args = tDef.getArgs(); + int argsNum = args.size(); + + validateAndSetupPatternStr(evaluator, args); + validateAndSetupSymbolInfo(evaluator, args, argsNum); + validateAndSetupResultExprStr(evaluator, args, argsNum); + setupSymbolFunctionChain(evaluator); + + /* + * setup OI for input to resultExpr select list + */ + RowResolver selectListInputRR = NPath.createSelectListRR(evaluator, tDef.getInput()); + StructObjectInspector selectListInputOI = (StructObjectInspector) PTFTranslator.getInputOI(selectListInputRR); + ResultExprInfo resultExprInfo = evaluator.resultExprInfo; + ArrayList selectListExprOIs = new ArrayList(); + resultExprInfo.resultExprEvals = new ArrayList(); + + for(int i=0 ; i < resultExprInfo.resultExprNodes.size(); i++) { + ExprNodeDesc selectColumnExprNode =resultExprInfo.resultExprNodes.get(i); + ExprNodeEvaluator selectColumnExprEval = ExprNodeEvaluatorFactory.get(selectColumnExprNode); + ObjectInspector selectColumnOI = selectColumnExprEval.initialize(selectListInputOI); + resultExprInfo.resultExprEvals.add(selectColumnExprEval); + selectListExprOIs.add(selectColumnOI); + } + + resultExprInfo.resultOI = ObjectInspectorFactory.getStandardStructObjectInspector( + resultExprInfo.resultExprNames, selectListExprOIs); + setOutputOI(resultExprInfo.resultOI); + } + catch(SemanticException se) { + throw new HiveException(se); + } + } + + @Override + public ArrayList getOutputColumnNames() { + NPath evaluator = (NPath) getEvaluator(); + return evaluator.resultExprInfo.getResultExprNames(); + } + + } + + public ResultExprInfo getResultExprInfo() { + return resultExprInfo; + } + + public void setResultExprInfo(ResultExprInfo resultExprInfo) { + this.resultExprInfo = resultExprInfo; + } + + static class SymbolsInfo { + int sz; + ArrayList symbolExprsEvaluators; + ArrayList symbolExprsOIs; + ArrayList symbolExprsNames; + + SymbolsInfo(int sz) + { + this.sz = sz; + symbolExprsEvaluators = new ArrayList(sz); + symbolExprsOIs = new ArrayList(sz); + symbolExprsNames = new ArrayList(sz); + } + + void add(String name, ArgDef arg) + { + symbolExprsNames.add(name); + symbolExprsEvaluators.add(arg.getExprEvaluator()); + symbolExprsOIs.add(arg.getOI()); + } + } + + public static class ResultExprInfo { + ArrayList resultExprNames; + ArrayList resultExprNodes; + private transient ArrayList resultExprEvals; + private transient StructObjectInspector resultOI; + + public ArrayList getResultExprNames() { + return resultExprNames; + } + public void setResultExprNames(ArrayList resultExprNames) { + this.resultExprNames = resultExprNames; + } + public ArrayList getResultExprNodes() { + return resultExprNodes; + } + public void setResultExprNodes(ArrayList resultExprNodes) { + this.resultExprNodes = resultExprNodes; + } + } + + public static abstract class SymbolFunction + { + SymbolFunctionResult result; + + public SymbolFunction() + { + result = new SymbolFunctionResult(); + } + + public static SymbolFunctionResult match(SymbolFunction syFn, Object row, + PTFPartitionIterator pItr) throws HiveException + { + int resetToIdx = pItr.getIndex() - 1; + try + { + return syFn.match(row, pItr); + } finally + { + pItr.resetToIndex(resetToIdx); + } + } + + protected abstract SymbolFunctionResult match(Object row, PTFPartitionIterator pItr) + throws HiveException; + + protected abstract boolean isOptional(); + } + + public static class Symbol extends SymbolFunction { + ExprNodeEvaluator symbolExprEval; + Converter converter; + + public Symbol(ExprNodeEvaluator symbolExprEval, ObjectInspector symbolOI) + { + this.symbolExprEval = symbolExprEval; + converter = ObjectInspectorConverters.getConverter( + symbolOI, + PrimitiveObjectInspectorFactory.javaBooleanObjectInspector); + } + + @Override + protected SymbolFunctionResult match(Object row, PTFPartitionIterator pItr) + throws HiveException + { + Object val = null; + val = symbolExprEval.evaluate(row); + val = converter.convert(val); + result.matches = ((Boolean) val).booleanValue(); + result.nextRow = pItr.getIndex(); + + return result; + } + + @Override + protected boolean isOptional() + { + return false; + } + } + + public static class Star extends SymbolFunction { + SymbolFunction symbolFn; + + public Star(SymbolFunction symbolFn) + { + this.symbolFn = symbolFn; + } + + @Override + protected SymbolFunctionResult match(Object row, PTFPartitionIterator pItr) + throws HiveException + { + result.matches = true; + SymbolFunctionResult rowResult = symbolFn.match(row, pItr); + + while (rowResult.matches && pItr.hasNext()) + { + row = pItr.next(); + rowResult = symbolFn.match(row, pItr); + } + + result.nextRow = pItr.getIndex() - 1; + return result; + } + + @Override + protected boolean isOptional() + { + return true; + } + } + + public static class Plus extends SymbolFunction { + SymbolFunction symbolFn; + + public Plus(SymbolFunction symbolFn) + { + this.symbolFn = symbolFn; + } + + @Override + protected SymbolFunctionResult match(Object row, PTFPartitionIterator pItr) + throws HiveException + { + SymbolFunctionResult rowResult = symbolFn.match(row, pItr); + + if (!rowResult.matches) + { + result.matches = false; + result.nextRow = pItr.getIndex() - 1; + return result; + } + + result.matches = true; + while (rowResult.matches && pItr.hasNext()) + { + row = pItr.next(); + rowResult = symbolFn.match(row, pItr); + } + + result.nextRow = pItr.getIndex() - 1; + return result; + } + + @Override + protected boolean isOptional() + { + return false; + } + } + + public static class Chain extends SymbolFunction + { + ArrayList components; + + public Chain(ArrayList components) + { + this.components = components; + } + + /* + * Iterate over the Symbol Functions in the Chain: + * - If we are not at the end of the Iterator (i.e. row != null ) + * - match the current componentFn + * - if it returns false, then return false + * - otherwise set row to the next row from the Iterator. + * - if we are at the end of the Iterator + * - skip any optional Symbol Fns (star patterns) at the end. + * - but if we come to a non optional Symbol Fn, return false. + * - if we match all Fns in the chain return true. + */ + @Override + protected SymbolFunctionResult match(Object row, PTFPartitionIterator pItr) + throws HiveException + { + SymbolFunctionResult componentResult = null; + for (SymbolFunction sFn : components) + { + if (row != null) + { + componentResult = sFn.match(row, pItr); + if (!componentResult.matches) + { + result.matches = false; + result.nextRow = componentResult.nextRow; + return result; + } + row = pItr.resetToIndex(componentResult.nextRow); + } + else + { + if (!sFn.isOptional()) + { + result.matches = false; + result.nextRow = componentResult.nextRow; + return result; + } + } + } + + result.matches = true; + result.nextRow = componentResult.nextRow; + return result; + } + + @Override + protected boolean isOptional() + { + return false; + } + } + + + public static class SymbolFunctionResult + { + /* + * does the row match the pattern represented by this SymbolFunction + */ + public boolean matches; + /* + * what is the index of the row beyond the set of rows that match this pattern. + */ + public int nextRow; + } + + public static class SymbolParser + { + String patternStr; + String[] symbols; + HashMap symbolExprEvalMap; + ArrayList symbolFunctions; + Chain symbolFnChain; + + + public SymbolParser(String patternStr, ArrayList symbolNames, + ArrayList symbolExprEvals, ArrayList symbolExprOIs) + { + super(); + this.patternStr = patternStr; + symbolExprEvalMap = new HashMap(); + int sz = symbolNames.size(); + for(int i=0; i < sz; i++) + { + String symbolName = symbolNames.get(i); + ExprNodeEvaluator symbolExprEval = symbolExprEvals.get(i); + ObjectInspector symbolExprOI = symbolExprOIs.get(i); + symbolExprEvalMap.put(symbolName.toLowerCase(), new Object[] {symbolExprEval, symbolExprOI}); + } + } + + public SymbolFunction getSymbolFunction() + { + return symbolFnChain; + } + + public void parse() throws SemanticException + { + symbols = patternStr.split("\\."); + symbolFunctions = new ArrayList(); + + for(String symbol : symbols) + { + boolean isStar = symbol.endsWith("*"); + boolean isPlus = symbol.endsWith("+"); + + symbol = (isStar || isPlus) ? symbol.substring(0, symbol.length() - 1) : symbol; + Object[] symbolDetails = symbolExprEvalMap.get(symbol.toLowerCase()); + if ( symbolDetails == null ) + { + throw new SemanticException(PTFUtils.sprintf("Unknown Symbol %s", symbol)); + } + + ExprNodeEvaluator symbolExprEval = (ExprNodeEvaluator) symbolDetails[0]; + ObjectInspector symbolExprOI = (ObjectInspector) symbolDetails[1]; + SymbolFunction sFn = new Symbol(symbolExprEval, symbolExprOI); + + if ( isStar ) + { + sFn = new Star(sFn); + } + else if ( isPlus ) + { + sFn = new Plus(sFn); + } + symbolFunctions.add(sFn); + } + symbolFnChain = new Chain(symbolFunctions); + } + } + + /* + * ResultExpression is a Select List with the following variation: + * - the select keyword is optional. The parser checks if the expression doesn't start with + * select; if not it prefixes it. + * - Window Fn clauses are not permitted. + * - expressions can operate on the input columns plus the psuedo column 'path' which is array of + * structs. The shape of the struct is + * the same as the input. + */ + public static class ResultExpressionParser { + String resultExprString; + + RowResolver selectListInputRowResolver; + TypeCheckCtx selectListInputTypeCheckCtx; + StructObjectInspector selectListInputOI; + + SelectSpec selectSpec; + + ResultExprInfo resultExprInfo; + + public ResultExpressionParser(String resultExprString, RowResolver selectListInputRowResolver) + { + this.resultExprString = resultExprString; + this.selectListInputRowResolver = selectListInputRowResolver; + } + + public void translate() throws SemanticException, HiveException + { + setupSelectListInputInfo(); + fixResultExprString(); + parse(); + validateSelectExpr(); + buildSelectListEvaluators(); + } + + public ResultExprInfo getResultExprInfo() { + return resultExprInfo; + } + + private void buildSelectListEvaluators() throws SemanticException, HiveException + { + resultExprInfo = new ResultExprInfo(); + resultExprInfo.resultExprEvals = new ArrayList(); + resultExprInfo.resultExprNames = new ArrayList(); + resultExprInfo.resultExprNodes = new ArrayList(); + //result + ArrayList selectListExprOIs = new ArrayList(); + int i = 0; + Iterator it = selectSpec.getColumnListAndAlias(); + while (it.hasNext()) + { + Object[] selectColDetails = (Object[]) it.next(); + String selectColName = (String) selectColDetails[1]; + ASTNode selectColumnNode = (ASTNode) selectColDetails[2]; + ExprNodeDesc selectColumnExprNode = PTFTranslator.buildExprNode(selectColumnNode, + selectListInputTypeCheckCtx); + ExprNodeEvaluator selectColumnExprEval = ExprNodeEvaluatorFactory.get(selectColumnExprNode); + ObjectInspector selectColumnOI = null; + selectColumnOI = selectColumnExprEval.initialize(selectListInputOI); + + selectColName = getColumnName(selectColName, selectColumnExprNode, i); + + resultExprInfo.resultExprEvals.add(selectColumnExprEval); + selectListExprOIs.add(selectColumnOI); + resultExprInfo.resultExprNodes.add(selectColumnExprNode); + resultExprInfo.resultExprNames.add(selectColName); + i++; + } + + resultExprInfo.resultOI = ObjectInspectorFactory.getStandardStructObjectInspector( + resultExprInfo.resultExprNames, selectListExprOIs); + } + + private void setupSelectListInputInfo() throws SemanticException + { + selectListInputTypeCheckCtx = new TypeCheckCtx(selectListInputRowResolver); + selectListInputTypeCheckCtx.setUnparseTranslator(null); + /* + * create SelectListOI + */ + selectListInputOI = (StructObjectInspector) PTFTranslator.getInputOI(selectListInputRowResolver); + } + + private void fixResultExprString() + { + String r = resultExprString.trim(); + String prefix = r.substring(0, 6); + if (!prefix.toLowerCase().equals("select")) + { + r = "select " + r; + } + resultExprString = r; + } + + private void parse() throws SemanticException + { + selectSpec = SemanticAnalyzer.parseSelect(resultExprString); + } + + private void validateSelectExpr() throws SemanticException + { + if (selectSpec.getWindowFuncs() != null) + { + throw new SemanticException( + "NPath Result Expression cannot have Windowing Function expressions"); + } + + for (ASTNode node : selectSpec.getExpressions()) + { + PTFTranslator.validateNoLeadLagInValueBoundarySpec(node, + "Lead/Lag not allowed in NPath Result Expression"); + } + } + + private String getColumnName(String alias, ExprNodeDesc exprNode, int colIdx) + { + if (alias != null) + { + return alias; + } + else if (exprNode instanceof ExprNodeColumnDesc) + { + ExprNodeColumnDesc colDesc = (ExprNodeColumnDesc) exprNode; + return colDesc.getColumn(); + } + return "npath_col_" + colIdx; + } + } + + public static final String PATHATTR_NAME = "tpath"; + + /* + * add array to the list of columns + */ + protected static RowResolver createSelectListRR(NPath evaluator, PTFInputDef inpDef) throws SemanticException { + RowResolver rr = new RowResolver(); + RowResolver inputRR = inpDef.getInputInfo().getRowResolver(); + boolean inputColNamesKnown = evaluator.inputColumnNames != null; + + if ( !inputColNamesKnown ) { + evaluator.inputColumnNames = new ArrayList(); + } + + ArrayList inpColOIs = new ArrayList(); + + for (ColumnInfo inpCInfo : inputRR.getColumnInfos()) { + ColumnInfo cInfo = new ColumnInfo(inpCInfo); + String colAlias = cInfo.getAlias(); + + String[] tabColAlias = inputRR.reverseLookup(inpCInfo.getInternalName()); + if (tabColAlias != null) { + colAlias = tabColAlias[1]; + } + ASTNode inExpr = null; + inExpr = PTFTranslator.getASTNode(inpCInfo, inputRR); + if ( inExpr != null ) { + rr.putExpression(inExpr, cInfo); + } + else { + colAlias = colAlias == null ? cInfo.getInternalName() : colAlias; + rr.put(cInfo.getTabAlias(), colAlias, cInfo); + } + + if ( !inputColNamesKnown ) { + evaluator.inputColumnNames.add(colAlias); + } + inpColOIs.add(cInfo.getObjectInspector()); + } + + StandardListObjectInspector pathAttrOI = ObjectInspectorFactory.getStandardListObjectInspector( + ObjectInspectorFactory.getStandardStructObjectInspector(evaluator.inputColumnNames, inpColOIs)); + + ColumnInfo pathColumn = new ColumnInfo(PATHATTR_NAME, + TypeInfoUtils.getTypeInfoFromObjectInspector(pathAttrOI), + null, + false, false); + rr.put(null, PATHATTR_NAME, pathColumn); + + return rr; + } + + public static Object getSelectListInput(Object currRow, ObjectInspector rowOI, + PTFPartitionIterator pItr, int sz) { + ArrayList oRow = new ArrayList(); + List currRowAsStdObject = (List) ObjectInspectorUtils + .copyToStandardObject(currRow, rowOI); + oRow.addAll(currRowAsStdObject); + oRow.add(getPath(currRow, rowOI, pItr, sz)); + return oRow; + } + + public static ArrayList getPath(Object currRow, ObjectInspector rowOI, + PTFPartitionIterator pItr, int sz) { + int idx = pItr.getIndex() - 1; + ArrayList path = new ArrayList(); + path.add(ObjectInspectorUtils.copyToStandardObject(currRow, rowOI)); + int pSz = 1; + + while (pSz < sz && pItr.hasNext()) + { + currRow = pItr.next(); + path.add(ObjectInspectorUtils.copyToStandardObject(currRow, rowOI)); + pSz++; + } + pItr.resetToIndex(idx); + return path; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/Noop.java ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/Noop.java new file mode 100644 index 0000000..fa109bb --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/Noop.java @@ -0,0 +1,78 @@ +package org.apache.hadoop.hive.ql.udf.ptf; + +import java.util.ArrayList; + +import org.apache.hadoop.hive.ql.exec.PTFPartition; +import org.apache.hadoop.hive.ql.exec.PTFPartition.PTFPartitionIterator; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.PTFDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.TableFuncDef; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; + +public class Noop extends TableFunctionEvaluator +{ + + @Override + public PTFPartition execute(PTFPartition iPart) throws HiveException + { + return iPart; + } + + @Override + protected void execute(PTFPartitionIterator pItr, PTFPartition oPart) + { + throw new UnsupportedOperationException(); + } + + public static class NoopResolver extends TableFunctionResolver + { + + @Override + protected TableFunctionEvaluator createEvaluator(PTFDef qDef, TableFuncDef tDef) + { + return new Noop(); + } + + @Override + public void setupOutputOI() throws SemanticException + { + StructObjectInspector OI = getEvaluator().getTableDef().getInput().getOI(); + setOutputOI(OI); + } + + /* + * (non-Javadoc) + * @see org.apache.hadoop.hive.ql.udf.ptf.TableFunctionResolver#carryForwardNames() + * Setting to true is correct only for special internal Functions. + */ + @Override + public boolean carryForwardNames() { + return true; + } + + /* + * (non-Javadoc) + * @see org.apache.hadoop.hive.ql.udf.ptf.TableFunctionResolver#getOutputNames() + * Set to null only because carryForwardNames is true. + */ + @Override + public ArrayList getOutputColumnNames() { + return null; + } + + @Override + public boolean transformsRawInput() + { + return false; + } + + @Override + public void initializeOutputOI() throws HiveException { + setupOutputOI(); + + } + + } + +} \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NoopWithMap.java ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NoopWithMap.java new file mode 100644 index 0000000..e2cb526 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NoopWithMap.java @@ -0,0 +1,98 @@ +package org.apache.hadoop.hive.ql.udf.ptf; + +import java.util.ArrayList; + +import org.apache.hadoop.hive.ql.exec.PTFPartition; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.PTFDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.TableFuncDef; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; + +public class NoopWithMap extends Noop +{ + @Override + public PTFPartition execute(PTFPartition iPart) throws HiveException + { + return iPart; + } + + @Override + protected PTFPartition _transformRawInput(PTFPartition iPart) throws HiveException + { + return iPart; + } + + public static class NoopWithMapResolver extends TableFunctionResolver + { + + @Override + protected TableFunctionEvaluator createEvaluator(PTFDef qDef, TableFuncDef tDef) + { + return new NoopWithMap(); + } + + @Override + public void setupOutputOI() throws SemanticException + { + StructObjectInspector OI = getEvaluator().getTableDef().getInput().getOI(); + setOutputOI(OI); + } + + /* + * (non-Javadoc) + * @see org.apache.hadoop.hive.ql.udf.ptf.TableFunctionResolver#carryForwardNames() + * Setting to true is correct only for special internal Functions. + */ + @Override + public boolean carryForwardNames() { + return true; + } + + /* + * (non-Javadoc) + * @see org.apache.hadoop.hive.ql.udf.ptf.TableFunctionResolver#getOutputNames() + * Set to null only because carryForwardNames is true. + */ + @Override + public ArrayList getOutputColumnNames() { + return null; + } + + @Override + public void setupRawInputOI() throws SemanticException + { + StructObjectInspector OI = getEvaluator().getTableDef().getInput().getOI(); + setRawInputOI(OI); + } + + /* + * (non-Javadoc) + * @see org.apache.hadoop.hive.ql.udf.ptf.TableFunctionResolver#getOutputNames() + * Set to null only because carryForwardNames is true. + */ + @Override + public ArrayList getRawInputColumnNames() throws SemanticException { + return null; + } + + @Override + public boolean transformsRawInput() + { + return true; + } + + @Override + public void initializeOutputOI() throws HiveException { + setupOutputOI(); + } + + @Override + public void initializeRawInputOI() throws HiveException { + setupRawInputOI(); + } + + } + + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java new file mode 100644 index 0000000..5d038e8 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java @@ -0,0 +1,143 @@ +package org.apache.hadoop.hive.ql.udf.ptf; + +import static org.apache.hadoop.hive.ql.exec.PTFUtils.sprintf; + +import org.apache.hadoop.hive.ql.exec.PTFOperator; +import org.apache.hadoop.hive.ql.exec.PTFPartition; +import org.apache.hadoop.hive.ql.exec.PTFPartition.PTFPartitionIterator; +import org.apache.hadoop.hive.ql.exec.PTFUtils; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.plan.PTFDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.TableFuncDef; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; + +/** + * Based on Hive {@link GenericUDAFEvaluator}. Break up the responsibility of the old AsbtractTableFunction + * class into a Resolver and Evaluator. + *

+ * The Evaluator also holds onto the {@link TableFunctionDef}. This provides information + * about the arguments to the function, the shape of the Input partition and the Partitioning details. + * The Evaluator is responsible for providing the 2 execute methods: + *

    + *
  1. execute: which is invoked after the input is partitioned; the contract + * is, it is given an input Partition and must return an output Partition. The shape of the output + * Partition is obtained from the getOutputOI call. + *
  2. transformRawInput: In the case where this function indicates that it will transform the raw input + * before it is fed through the partitioning mechanics, this function is called. Again the contract is + * t is given an input Partition and must return an Partition. The shape of the output Partition is + * obtained from getRawInputOI() call. + *
+ * + */ +public abstract class TableFunctionEvaluator +{ + transient protected StructObjectInspector OI; + transient protected StructObjectInspector rawInputOI; + protected TableFuncDef tDef; + protected PTFDef qDef; + String partitionClass; + int partitionMemSize; + boolean transformsRawInput; + + static{ + PTFUtils.makeTransient(TableFunctionEvaluator.class, "OI"); + PTFUtils.makeTransient(TableFunctionEvaluator.class, "rawInputOI"); + } + + + public StructObjectInspector getOutputOI() + { + return OI; + } + + protected void setOutputOI(StructObjectInspector outputOI) + { + OI = outputOI; + } + + public TableFuncDef getTableDef() + { + return tDef; + } + + public void setTableDef(TableFuncDef tDef) + { + this.tDef = tDef; + } + + protected PTFDef getQueryDef() + { + return qDef; + } + + protected void setQueryDef(PTFDef qDef) + { + this.qDef = qDef; + } + + public String getPartitionClass() + { + return partitionClass; + } + + public void setPartitionClass(String partitionClass) + { + this.partitionClass = partitionClass; + } + + public int getPartitionMemSize() + { + return partitionMemSize; + } + + public void setPartitionMemSize(int partitionMemSize) + { + this.partitionMemSize = partitionMemSize; + } + + public StructObjectInspector getRawInputOI() + { + return rawInputOI; + } + + protected void setRawInputOI(StructObjectInspector rawInputOI) + { + this.rawInputOI = rawInputOI; + } + + public boolean isTransformsRawInput() { + return transformsRawInput; + } + + public void setTransformsRawInput(boolean transformsRawInput) { + this.transformsRawInput = transformsRawInput; + } + + public PTFPartition execute(PTFPartition iPart) + throws HiveException + { + PTFPartitionIterator pItr = iPart.iterator(); + PTFOperator.connectLeadLagFunctionsToPartition(qDef, pItr); + PTFPartition outP = new PTFPartition(getPartitionClass(), + getPartitionMemSize(), tDef.getSerde(), OI); + execute(pItr, outP); + return outP; + } + + protected abstract void execute(PTFPartitionIterator pItr, PTFPartition oPart) throws HiveException; + + public PTFPartition transformRawInput(PTFPartition iPart) throws HiveException + { + if ( !isTransformsRawInput()) + { + throw new HiveException(sprintf("Internal Error: mapExecute called on function (%s)that has no Map Phase", tDef.getName())); + } + return _transformRawInput(iPart); + } + + protected PTFPartition _transformRawInput(PTFPartition iPart) throws HiveException + { + return null; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java new file mode 100644 index 0000000..ce284ac --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java @@ -0,0 +1,198 @@ +package org.apache.hadoop.hive.ql.udf.ptf; + +import java.util.ArrayList; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; +import org.apache.hadoop.hive.ql.exec.FunctionRegistry; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.PTFDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.TableFuncDef; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFResolver; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; + +/** + * Based on Hive {@link GenericUDAFResolver}. Break up the responsibility of the + * old AbstractTableFunction class into a Resolver and Evaluator. + * The Resolver is responsible for: + *
    + *
  1. setting up the {@link tableFunctionEvaluator} + *
  2. Setting up the The raw and output ObjectInspectors of the Evaluator. + *
  3. The Evaluator also holds onto the {@link TableFunctionDef}. This provides information + * about the arguments to the function, the shape of the Input partition and the Partitioning details. + *
+ * The Resolver for a function is obtained from the {@link FunctionRegistry}. The Resolver is initialized + * by the following 4 step process: + *
    + *
  1. The initialize method is called; which is passed the {@link PTFDef} and the {@link TableFunctionDef}. + *
  2. The resolver is then asked to setup the Raw ObjectInspector. This is only required if the Function reshapes + * the raw input. + *
  3. Once the Resolver has had a chance to compute the shape of the Raw Input that is fed to the partitioning + * machinery; the translator sets up the partitioning details on the tableFuncDef. + *
  4. finally the resolver is asked to setup the output ObjectInspector. + *
+ */ +@SuppressWarnings("deprecation") +public abstract class TableFunctionResolver +{ + TableFunctionEvaluator evaluator; + PTFDef qDef; + + /* + * - called during translation. + * - invokes createEvaluator which must be implemented by a subclass + * - sets up the evaluator with references to the TableDef, PartitionClass, PartitonMemsize and + * the transformsRawInput boolean. + */ + public void initialize(PTFDef qDef, TableFuncDef tDef) + throws SemanticException + { + this.qDef = qDef; + HiveConf cfg = qDef.getTranslationInfo().getHiveCfg(); + String partitionClass = HiveConf.getVar(cfg, ConfVars.HIVE_PTF_PARTITION_PERSISTENCE_CLASS); + int partitionMemSize = HiveConf.getIntVar(cfg, ConfVars.HIVE_PTF_PARTITION_PERSISTENT_SIZE); + + evaluator = createEvaluator(qDef, tDef); + evaluator.setTransformsRawInput(transformsRawInput()); + evaluator.setTableDef(tDef); + evaluator.setQueryDef(qDef); + evaluator.setPartitionClass(partitionClass); + evaluator.setPartitionMemSize(partitionMemSize); + + } + + /* + * called during deserialization of a QueryDef during runtime. + */ + public void initialize(PTFDef qDef, TableFuncDef tDef, TableFunctionEvaluator evaluator) + throws HiveException + { + this.evaluator = evaluator; + this.qDef = qDef; + evaluator.setTableDef(tDef); + evaluator.setQueryDef(qDef); + } + + public TableFunctionEvaluator getEvaluator() + { + return evaluator; + } + + /* + * - a subclass must provide this method. + * - this method is invoked during translation and also when the Operator is initialized during runtime. + * - a subclass must use this call to setup the shape of its output. + * - subsequent to this call, a call to getOutputOI call on the {@link TableFunctionEvaluator} must return the OI + * of the output of this function. + */ + public abstract void setupOutputOI() throws SemanticException; + + /* + * A PTF Function must provide the 'external' names of the columns in its Output. + * + */ + public abstract ArrayList getOutputColumnNames() throws SemanticException; + + + /** + * This method is invoked during runtime(during deserialization of theQueryDef). + * At this point the TableFunction can assume that the {@link ExprNodeDesc Expression Nodes} + * exist for all the Def (ArgDef, ColumnDef, WindowDef..). It is the responsibility of + * the TableFunction to construct the {@link ExprNodeEvaluator evaluators} and setup the OI. + * + * @param tblFuncDef + * @param qDef + * @throws HiveException + */ + public abstract void initializeOutputOI() throws HiveException; + + /* + * - Called on functions that transform the raw input. + * - this method is invoked during translation and also when the Operator is initialized during runtime. + * - a subclass must use this call to setup the shape of the raw input, that is fed to the partitioning mechanics. + * - subsequent to this call, a call to getRawInputOI call on the {@link TableFunctionEvaluator} must return the OI + * of the output of this function. + */ + public void setupRawInputOI() throws SemanticException + { + if (!transformsRawInput()) + { + return; + } + throw new SemanticException( + "Function has map phase, must extend setupMapOI"); + } + + /* + * A PTF Function must provide the 'external' names of the columns in the transformed Raw Input. + * + */ + public ArrayList getRawInputColumnNames() throws SemanticException { + if (!transformsRawInput()) + { + return null; + } + throw new SemanticException( + "Function transforms Raw Input; must extend getRawColumnInputNames"); + } + + /* + * Same responsibility as initializeOI, but for the RawInput. + */ + public void initializeRawInputOI() throws HiveException + { + if (!transformsRawInput()) + { + return; + } + throw new HiveException( + "Function has map phase, must extend initializeRawInputOI"); + } + + /* + * callback method used by subclasses to set the RawInputOI on the Evaluator. + */ + protected void setRawInputOI(StructObjectInspector rawInputOI) + { + evaluator.setRawInputOI(rawInputOI); + } + + /* + * callback method used by subclasses to set the OutputOI on the Evaluator. + */ + protected void setOutputOI(StructObjectInspector outputOI) + { + evaluator.setOutputOI(outputOI); + } + + public PTFDef getQueryDef() + { + return qDef; + } + + /* + * This is used during translation to decide if the internalName -> alias mapping from the Input to the PTF is carried + * forward when building the Output RR for this PTF. + * This is used by internal PTFs: NOOP, WindowingTableFunction to make names in its input available in the Output. + * In general this should be false; and the names used for the Output Columns must be provided by the PTF Writer in the + * function getOutputNames. + */ + public boolean carryForwardNames() { + return false; + } + + /* + * a subclass must indicate whether it will transform the raw input before it is fed through the + * partitioning mechanics. + */ + public abstract boolean transformsRawInput(); + + /* + * a subclass must provide the {@link TableFunctionEvaluator} instance. + */ + protected abstract TableFunctionEvaluator createEvaluator(PTFDef qDef, + TableFuncDef tDef); +} diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java new file mode 100644 index 0000000..d1814db --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java @@ -0,0 +1,505 @@ +package org.apache.hadoop.hive.ql.udf.ptf; + +import java.util.AbstractList; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hive.ql.exec.FunctionRegistry; +import org.apache.hadoop.hive.ql.exec.PTFOperator; +import org.apache.hadoop.hive.ql.exec.PTFPartition; +import org.apache.hadoop.hive.ql.exec.PTFPartition.PTFPartitionIterator; +import org.apache.hadoop.hive.ql.exec.WindowFunctionInfo; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowFrameSpec.BoundarySpec; +import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowFrameSpec.Direction; +import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowFunctionSpec; +import org.apache.hadoop.hive.ql.parse.PTFTranslator; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.PTFDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.ArgDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.SelectDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.TableFuncDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.WindowFrameDef.BoundaryDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.WindowFrameDef.CurrentRowDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.WindowFrameDef.RangeBoundaryDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.WindowFrameDef.ValueBoundaryDef; +import org.apache.hadoop.hive.ql.plan.PTFDef.WindowFunctionDef; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AggregationBuffer; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; + +public class WindowingTableFunction extends TableFunctionEvaluator +{ + ArrayList wFnDefs; + + @SuppressWarnings({ "unchecked", "rawtypes" }) + @Override + public void execute(PTFPartitionIterator pItr, PTFPartition outP) throws HiveException + { + ArrayList> oColumns = new ArrayList>(); + PTFPartition iPart = pItr.getPartition(); + StructObjectInspector inputOI; + try { + inputOI = (StructObjectInspector) iPart.getSerDe().getObjectInspector(); + } catch (SerDeException se) { + throw new HiveException(se); + } + + for(WindowFunctionDef wFn : wFnDefs) + { + boolean processWindow = wFn.getWindow() != null && wFn.getWindow().getWindow() != null; + pItr.reset(); + if ( !processWindow ) + { + GenericUDAFEvaluator fEval = wFn.getEvaluator(); + Object[] args = new Object[wFn.getArgs() == null ? 0 : wFn.getArgs().size()]; + AggregationBuffer aggBuffer = fEval.getNewAggregationBuffer(); + while(pItr.hasNext()) + { + Object row = pItr.next(); + int i =0; + if ( wFn.getArgs() != null ) { + for(ArgDef arg : wFn.getArgs()) + { + args[i++] = arg.getExprEvaluator().evaluate(row); + } + } + fEval.aggregate(aggBuffer, args); + } + Object out = fEval.evaluate(aggBuffer); + WindowFunctionInfo wFnInfo = FunctionRegistry.getWindowFunctionInfo(wFn.getSpec().getName()); + if ( !wFnInfo.isPivotResult()) + { + out = new SameList(iPart.size(), out); + } + oColumns.add((List)out); + } + else + { + oColumns.add(executeFnwithWindow(getQueryDef(), wFn, iPart)); + } + } + + /* + * Output Columns in the following order + * - the columns representing the output from Window Fns + * - the input Rows columns + */ + + for(int i=0; i < iPart.size(); i++) + { + ArrayList oRow = new ArrayList(); + Object iRow = iPart.getAt(i); + + for(int j=0; j < oColumns.size(); j++) + { + oRow.add(oColumns.get(j).get(i)); + } + + for(StructField f : inputOI.getAllStructFieldRefs()) + { + oRow.add(inputOI.getStructFieldData(iRow, f)); + } + + outP.append(oRow); + } + } + + public static class WindowingTableFunctionResolver extends TableFunctionResolver + { + + @Override + protected TableFunctionEvaluator createEvaluator(PTFDef qDef, TableFuncDef tDef) + { + + return new WindowingTableFunction(); + } + + @Override + public void setupOutputOI() throws SemanticException + { + ArrayList wFnDefs = new ArrayList(); + PTFDef qDef = getQueryDef(); + SelectDef select = qDef.getSelectList(); + ArrayList wFnSpecs = qDef.getSpec().getSelectList().getWindowFuncs(); + ArrayList aliases = new ArrayList(); + ArrayList fieldOIs = new ArrayList(); + + /* + * Setup the columns in the OI in the following order + * - the columns representing the Window Fns + * - the columns from the input + * Why? + * - during translation the input contains Virtual columns that are not represent during runtime + * - this messes with the Column Numbers (and hence internal Names) if we add the columns in a different order. + */ + + for(WindowFunctionSpec wFnS : wFnSpecs) + { + WindowFunctionDef wFnDef = PTFTranslator.translate(qDef, getEvaluator().getTableDef(), wFnS); + WindowFunctionInfo wFnInfo = FunctionRegistry.getWindowFunctionInfo(wFnS.getName()); + wFnDefs.add(wFnDef); + aliases.add(wFnS.getAlias()); + if ( wFnInfo.isPivotResult()) + { + ListObjectInspector lOI = (ListObjectInspector) wFnDef.getOI(); + fieldOIs.add(lOI.getListElementObjectInspector()); + } + else + { + fieldOIs.add(wFnDef.getOI()); + } + } + + PTFTranslator.addInputColumnsToList(qDef, getEvaluator().getTableDef(), aliases, fieldOIs); + + select.setWindowFuncs(wFnDefs); + WindowingTableFunction wTFn = (WindowingTableFunction) getEvaluator(); + wTFn.wFnDefs = wFnDefs; + + StructObjectInspector OI = ObjectInspectorFactory.getStandardStructObjectInspector(aliases, fieldOIs); + setOutputOI(OI); + } + + /* + * Setup the OI based on the: + * - Input TableDef's columns + * - the Window Functions. + */ + @Override + public void initializeOutputOI() throws HiveException + { + PTFDef qDef = getQueryDef(); + TableFuncDef tblFuncDef = evaluator.getTableDef(); + WindowingTableFunction wTFn = (WindowingTableFunction) tblFuncDef.getFunction(); + ArrayList wFnDefs = qDef.getSelectList().getWindowFuncs(); + ArrayList aliases = new ArrayList(); + ArrayList fieldOIs = new ArrayList(); + + /* + * Setup the columns in the OI in the following order + * - the columns representing the Window Fns + * - the columns from the input + * Why? + * - during translation the input contains Virtual columns that are not present during runtime + * - this messes with the Column Numbers (and hence internal Names) if we add the columns in a different order. + */ + + for (WindowFunctionDef wFnDef : wFnDefs) { + WindowFunctionSpec wFnS = wFnDef.getSpec(); + WindowFunctionInfo wFnInfo = FunctionRegistry.getWindowFunctionInfo(wFnS.getName()); + aliases.add(wFnS.getAlias()); + if ( wFnInfo.isPivotResult()) + { + ListObjectInspector lOI = (ListObjectInspector) wFnDef.getOI(); + fieldOIs.add(lOI.getListElementObjectInspector()); + } + else + { + fieldOIs.add(wFnDef.getOI()); + } + + } + PTFTranslator.addInputColumnsToList(qDef, getEvaluator().getTableDef(), aliases, fieldOIs); + + wTFn.wFnDefs = wFnDefs; + StructObjectInspector OI = ObjectInspectorFactory.getStandardStructObjectInspector(aliases, fieldOIs); + setOutputOI(OI); + } + + + @Override + public boolean transformsRawInput() + { + return false; + } + + /* + * (non-Javadoc) + * @see org.apache.hadoop.hive.ql.udf.ptf.TableFunctionResolver#carryForwardNames() + * Setting to true is correct only for special internal Functions. + */ + @Override + public boolean carryForwardNames() { + return true; + } + + /* + * (non-Javadoc) + * @see org.apache.hadoop.hive.ql.udf.ptf.TableFunctionResolver#getOutputNames() + * Set to null only because carryForwardNames is true. + */ + @Override + public ArrayList getOutputColumnNames() { + return null; + } + + } + + static ArrayList executeFnwithWindow(PTFDef qDef, WindowFunctionDef wFnDef, PTFPartition iPart) + throws HiveException + { + ArrayList vals = new ArrayList(); + + GenericUDAFEvaluator fEval = wFnDef.getEvaluator(); + Object[] args = new Object[wFnDef.getArgs().size()]; + for(int i=0; i < iPart.size(); i++) + { + AggregationBuffer aggBuffer = fEval.getNewAggregationBuffer(); + Range rng = getRange(wFnDef, i, iPart); + PTFPartitionIterator rItr = rng.iterator(); + PTFOperator.connectLeadLagFunctionsToPartition(qDef, rItr); + while(rItr.hasNext()) + { + Object row = rItr.next(); + int j = 0; + for(ArgDef arg : wFnDef.getArgs()) + { + args[j++] = arg.getExprEvaluator().evaluate(row); + } + fEval.aggregate(aggBuffer, args); + } + Object out = fEval.evaluate(aggBuffer); + out = ObjectInspectorUtils.copyToStandardObject(out, wFnDef.getOI()); + vals.add(out); + } + return vals; + } + + static Range getRange(WindowFunctionDef wFnDef, int currRow, PTFPartition p) throws HiveException + { + BoundaryDef startB = wFnDef.getWindow().getWindow().getStart(); + BoundaryDef endB = wFnDef.getWindow().getWindow().getEnd(); + + int start = getIndex(startB, currRow, p, false); + int end = getIndex(endB, currRow, p, true); + + return new Range(start, end, p); + } + + static int getIndex(BoundaryDef bDef, int currRow, PTFPartition p, boolean end) throws HiveException + { + if ( bDef instanceof CurrentRowDef) + { + return currRow + (end ? 1 : 0); + } + else if ( bDef instanceof RangeBoundaryDef) + { + RangeBoundaryDef rbDef = (RangeBoundaryDef) bDef; + int amt = rbDef.getAmt(); + + if ( amt == BoundarySpec.UNBOUNDED_AMOUNT ) + { + return rbDef.getDirection() == Direction.PRECEDING ? 0 : p.size(); + } + + amt = rbDef.getDirection() == Direction.PRECEDING ? -amt : amt; + int idx = currRow + amt; + idx = idx < 0 ? 0 : (idx > p.size() ? p.size() : idx); + return idx + (end && idx < p.size() ? 1 : 0); + } + else + { + ValueBoundaryScanner vbs = ValueBoundaryScanner.getScanner((ValueBoundaryDef)bDef); + return vbs.computeBoundaryRange(currRow, p); + } + } + + static class Range + { + int start; + int end; + PTFPartition p; + + public Range(int start, int end, PTFPartition p) + { + super(); + this.start = start; + this.end = end; + this.p = p; + } + + public PTFPartitionIterator iterator() + { + return p.range(start, end); + } + } + + /* + * - starting from the given rowIdx scan in the given direction until a row's expr + * evaluates to an amt that crosses the 'amt' threshold specified in the ValueBoundaryDef. + */ + static abstract class ValueBoundaryScanner + { + ValueBoundaryDef bndDef; + + public ValueBoundaryScanner(ValueBoundaryDef bndDef) + { + this.bndDef = bndDef; + } + + /* + * return the other end of the Boundary + * - when scanning backwards: go back until you reach a row where the + * startingValue - rowValue >= amt + * - when scanning forward: go forward go back until you reach a row where the + * rowValue - startingValue >= amt + */ + public int computeBoundaryRange(int rowIdx, PTFPartition p) throws HiveException + { + int r = rowIdx; + Object rowValue = computeValue(p.getAt(r)); + int amt = bndDef.getAmt(); + + if ( amt == BoundarySpec.UNBOUNDED_AMOUNT ) + { + return bndDef.getDirection() == Direction.PRECEDING ? 0 : p.size(); + } + + Direction d = bndDef.getDirection(); + boolean scanNext = rowValue != null; + while ( scanNext ) + { + if ( d == Direction.PRECEDING ) { + r = r - 1; + } + else { + r = r + 1; + } + + if ( r < 0 || r >= p.size() ) + { + scanNext = false; + break; + } + + Object currVal = computeValue(p.getAt(r)); + if ( currVal == null ) + { + scanNext = false; + break; + } + + switch(d) + { + case PRECEDING: + scanNext = !isGreater(rowValue, currVal, amt); + break; + case FOLLOWING: + scanNext = !isGreater(currVal, rowValue, amt); + case CURRENT: + default: + break; + } + } + /* + * if moving backwards, then r is at a row that failed the range test. So incr r, so that + * Range starts from a row where the test succeeds. + * Whereas when moving forward, leave r as is; because the Range's end value should be the + * row idx not in the Range. + */ + if ( d == Direction.PRECEDING ) { + r = r + 1; + } + r = r < 0 ? 0 : (r >= p.size() ? p.size() : r); + return r; + } + + public Object computeValue(Object row) throws HiveException + { + Object o = bndDef.getExprEvaluator().evaluate(row); + return ObjectInspectorUtils.copyToStandardObject(o, bndDef.getOI()); + } + + public abstract boolean isGreater(Object v1, Object v2, int amt); + + + public static ValueBoundaryScanner getScanner(ValueBoundaryDef vbDef) + { + PrimitiveObjectInspector pOI = (PrimitiveObjectInspector) vbDef.getOI(); + switch(pOI.getPrimitiveCategory()) + { + case BYTE: + case INT: + case LONG: + case SHORT: + case TIMESTAMP: + return new LongValueBoundaryScanner(vbDef); + case DOUBLE: + case FLOAT: + return new DoubleValueBoundaryScanner(vbDef); + } + return null; + } + } + + public static class LongValueBoundaryScanner extends ValueBoundaryScanner + { + public LongValueBoundaryScanner(ValueBoundaryDef bndDef) + { + super(bndDef); + } + + @Override + public boolean isGreater(Object v1, Object v2, int amt) + { + long l1 = PrimitiveObjectInspectorUtils.getLong(v1, + (PrimitiveObjectInspector) bndDef.getOI()); + long l2 = PrimitiveObjectInspectorUtils.getLong(v2, + (PrimitiveObjectInspector) bndDef.getOI()); + return (l1 -l2) >= amt; + } + } + + public static class DoubleValueBoundaryScanner extends ValueBoundaryScanner + { + public DoubleValueBoundaryScanner(ValueBoundaryDef bndDef) + { + super(bndDef); + } + + @Override + public boolean isGreater(Object v1, Object v2, int amt) + { + double d1 = PrimitiveObjectInspectorUtils.getDouble(v1, + (PrimitiveObjectInspector) bndDef.getOI()); + double d2 = PrimitiveObjectInspectorUtils.getDouble(v2, + (PrimitiveObjectInspector) bndDef.getOI()); + return (d1 -d2) >= amt; + } + } + + public static class SameList extends AbstractList + { + int sz; + E val; + + public SameList(int sz, E val) + { + this.sz = sz; + this.val = val; + } + + @Override + public E get(int index) + { + return val; + } + + @Override + public int size() + { + return sz; + } + + } + +} diff --git ql/src/test/queries/clientnegative/ptf_negative_AggrFuncsWithNoGBYNoPartDef.q ql/src/test/queries/clientnegative/ptf_negative_AggrFuncsWithNoGBYNoPartDef.q new file mode 100644 index 0000000..542367a --- /dev/null +++ ql/src/test/queries/clientnegative/ptf_negative_AggrFuncsWithNoGBYNoPartDef.q @@ -0,0 +1,20 @@ +DROP TABLE part; + +CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +); + +LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part; + +-- testAggrFuncsWithNoGBYNoPartDef +select p_mfgr, +sum(p_retailprice) as s1 +from part; \ No newline at end of file diff --git ql/src/test/queries/clientnegative/ptf_negative_AmbiguousWindowDefn.q ql/src/test/queries/clientnegative/ptf_negative_AmbiguousWindowDefn.q new file mode 100644 index 0000000..593b9fa --- /dev/null +++ ql/src/test/queries/clientnegative/ptf_negative_AmbiguousWindowDefn.q @@ -0,0 +1,28 @@ +DROP TABLE part; + +CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +); + +LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part; + +-- testAmbiguousWindowDefn +select p_mfgr, p_name, p_size, +sum(p_size) as s1 over (w1), +sum(p_size) as s2 over (w2), +sum(p_size) as s3 over (w3) +from part +distribute by p_mfgr +sort by p_mfgr +window w1 as rows between 2 preceding and 2 following, + w2 as rows between unbounded preceding and current row, + w3 as w3; + diff --git ql/src/test/queries/clientnegative/ptf_negative_DuplicateWindowAlias.q ql/src/test/queries/clientnegative/ptf_negative_DuplicateWindowAlias.q new file mode 100644 index 0000000..7f50476 --- /dev/null +++ ql/src/test/queries/clientnegative/ptf_negative_DuplicateWindowAlias.q @@ -0,0 +1,24 @@ +DROP TABLE part; + +CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +); + +LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part; + +-- testDuplicateWindowAlias +select p_mfgr, p_name, p_size, +sum(p_size) as s1 over (w1), +sum(p_size) as s2 over (w2) +from part +window w1 as distribute by p_mfgr sort by p_mfgr rows between 2 preceding and 2 following, + w2 as w1, + w2 as rows between unbounded preceding and current row; diff --git ql/src/test/queries/clientnegative/ptf_negative_HavingLead.q ql/src/test/queries/clientnegative/ptf_negative_HavingLead.q new file mode 100644 index 0000000..3b27fe4 --- /dev/null +++ ql/src/test/queries/clientnegative/ptf_negative_HavingLead.q @@ -0,0 +1,22 @@ +DROP TABLE part; + +CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +); + +LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part; + +-- testHavingLead +select p_mfgr,p_name, p_size +from part +having lead(p_size, 1) <= p_size +distribute by p_mfgr +sort by p_name; \ No newline at end of file diff --git ql/src/test/queries/clientnegative/ptf_negative_HavingLeadWithPTF.q ql/src/test/queries/clientnegative/ptf_negative_HavingLeadWithPTF.q new file mode 100644 index 0000000..6533814 --- /dev/null +++ ql/src/test/queries/clientnegative/ptf_negative_HavingLeadWithPTF.q @@ -0,0 +1,24 @@ +DROP TABLE part; + +CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +); + +LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part; + +-- testHavingLeadWithPTF +select p_mfgr,p_name, p_size +from noop(part +distribute by p_mfgr +sort by p_name) +having lead(p_size, 1) <= p_size +distribute by p_mfgr +sort by p_name; \ No newline at end of file diff --git ql/src/test/queries/clientnegative/ptf_negative_IncompatibleDistributeClause.q ql/src/test/queries/clientnegative/ptf_negative_IncompatibleDistributeClause.q new file mode 100644 index 0000000..9229c14 --- /dev/null +++ ql/src/test/queries/clientnegative/ptf_negative_IncompatibleDistributeClause.q @@ -0,0 +1,23 @@ +DROP TABLE part; + +CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +); + +LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part; + +-- testIncompatibleDistributeClause +select p_mfgr,p_name, p_size, +rank() as r, denserank() as dr, +sum(p_size) as s over (w1) +from part +distribute by p_mfgr +window w1 as distribute by p_name rows between 2 preceding and 2 following; \ No newline at end of file diff --git ql/src/test/queries/clientnegative/ptf_negative_IncompatibleOrderInWindowDefs.q ql/src/test/queries/clientnegative/ptf_negative_IncompatibleOrderInWindowDefs.q new file mode 100644 index 0000000..1cfa650 --- /dev/null +++ ql/src/test/queries/clientnegative/ptf_negative_IncompatibleOrderInWindowDefs.q @@ -0,0 +1,25 @@ +DROP TABLE part; + +CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +); + +LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part; + +-- testIncompatibleOrderInWindowDefs +select p_mfgr, p_name, p_size, +sum(p_size) as s1 over (w1), +sum(p_size) as s2 over (w2) +from part +distribute by p_mfgr +sort by p_mfgr +window w1 as distribute by p_mfgr sort by p_mfgr rows between 2 preceding and 2 following, + w2 as distribute by p_mfgr sort by p_name rows between unbounded preceding and current row; diff --git ql/src/test/queries/clientnegative/ptf_negative_IncompatiblePartitionInWindowDefs.q ql/src/test/queries/clientnegative/ptf_negative_IncompatiblePartitionInWindowDefs.q new file mode 100644 index 0000000..670a162 --- /dev/null +++ ql/src/test/queries/clientnegative/ptf_negative_IncompatiblePartitionInWindowDefs.q @@ -0,0 +1,25 @@ +DROP TABLE part; + +CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +); + +LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part; + +-- testIncompatiblePartitionInWindowDefs +select p_mfgr, p_name, p_size, +sum(p_size) as s1 over (w1), +sum(p_size) as s2 over (w2) +from part +distribute by p_mfgr +sort by p_mfgr +window w1 as distribute by p_mfgr sort by p_mfgr rows between 2 preceding and 2 following, + w2 as distribute by p_name sort by p_name rows between unbounded preceding and current row; diff --git ql/src/test/queries/clientnegative/ptf_negative_IncompatibleSortClause.q ql/src/test/queries/clientnegative/ptf_negative_IncompatibleSortClause.q new file mode 100644 index 0000000..af567c0 --- /dev/null +++ ql/src/test/queries/clientnegative/ptf_negative_IncompatibleSortClause.q @@ -0,0 +1,23 @@ +DROP TABLE part; + +CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +); + +LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part; + +-- testIncompatibleSortClause +select p_mfgr,p_name, p_size, +rank() as r, denserank() as dr, +sum(p_size) as s over (w1) +from part +distribute by p_mfgr +window w1 as distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following; \ No newline at end of file diff --git ql/src/test/queries/clientnegative/ptf_negative_InvalidValueBoundary.q ql/src/test/queries/clientnegative/ptf_negative_InvalidValueBoundary.q new file mode 100644 index 0000000..1003266 --- /dev/null +++ ql/src/test/queries/clientnegative/ptf_negative_InvalidValueBoundary.q @@ -0,0 +1,24 @@ +DROP TABLE part; + +CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +); + +LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part; + +-- testInvalidValueBoundary +select p_mfgr,p_name, p_size, +sum(p_size) as s over (w1) , +denserank() as dr +from part +distribute by p_mfgr +sort by p_name +window w1 as range between p_name 2 less and current row; \ No newline at end of file diff --git ql/src/test/queries/clientnegative/ptf_negative_JoinWithAmbigousAlias.q ql/src/test/queries/clientnegative/ptf_negative_JoinWithAmbigousAlias.q new file mode 100644 index 0000000..971d66b --- /dev/null +++ ql/src/test/queries/clientnegative/ptf_negative_JoinWithAmbigousAlias.q @@ -0,0 +1,22 @@ +DROP TABLE part; + +CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +); + +LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part; + +-- testJoinWithAmbigousAlias +select abc.* +from noop(part +distribute by p_mfgr +sort by p_name +) abc join part on abc.p_partkey = p1.p_partkey; \ No newline at end of file diff --git ql/src/test/queries/clientnegative/ptf_negative_NoSortNoDistByClause.q ql/src/test/queries/clientnegative/ptf_negative_NoSortNoDistByClause.q new file mode 100644 index 0000000..fe05ab9 --- /dev/null +++ ql/src/test/queries/clientnegative/ptf_negative_NoSortNoDistByClause.q @@ -0,0 +1,21 @@ +DROP TABLE part; + +CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +); + +LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part; + +-- testNoSortNoDistByClause +select p_mfgr,p_name, p_size, +rank() as r, denserank() as dr +from part +window w1 as rows between 2 preceding and 2 following; \ No newline at end of file diff --git ql/src/test/queries/clientnegative/ptf_negative_WhereLead.q ql/src/test/queries/clientnegative/ptf_negative_WhereLead.q new file mode 100644 index 0000000..38cbb7d --- /dev/null +++ ql/src/test/queries/clientnegative/ptf_negative_WhereLead.q @@ -0,0 +1,22 @@ +DROP TABLE part; + +CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +); + +LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part; + +-- testWhereLead +select p_mfgr,p_name, p_size +from part +where lead(p_size, 1) <= p_size +distribute by p_mfgr +sort by p_name; diff --git ql/src/test/queries/clientnegative/ptf_negative_WhereWithRankCond.q ql/src/test/queries/clientnegative/ptf_negative_WhereWithRankCond.q new file mode 100644 index 0000000..d773ef0 --- /dev/null +++ ql/src/test/queries/clientnegative/ptf_negative_WhereWithRankCond.q @@ -0,0 +1,23 @@ +DROP TABLE part; + +CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +); + +LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part; + +-- testWhereWithRankCond +select p_mfgr,p_name, p_size, +rank() as r +from part +where r < 4 +distribute by p_mfgr +sort by p_mfgr; diff --git ql/src/test/queries/clientpositive/ptf_general_queries.q ql/src/test/queries/clientpositive/ptf_general_queries.q new file mode 100644 index 0000000..1db9762 --- /dev/null +++ ql/src/test/queries/clientpositive/ptf_general_queries.q @@ -0,0 +1,866 @@ +DROP TABLE part; +DROP TABLE part_rc; +DROP TABLE part_seq; +DROP TABLE flights_tiny; + +-- data setup +CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +); + +LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part; + +CREATE TABLE part_rc( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) STORED AS RCFILE ; + +LOAD DATA LOCAL INPATH '../data/files/part.rc' overwrite into table part_rc; + +CREATE TABLE part_seq( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) STORED AS SEQUENCEFILE ; + +LOAD DATA LOCAL INPATH '../data/files/part.seq' overwrite into table part_seq; + +create table flights_tiny ( +ORIGIN_CITY_NAME string, +DEST_CITY_NAME string, +YEAR int, +MONTH int, +DAY_OF_MONTH int, +ARR_DELAY float, +FL_NUM string +); + +LOAD DATA LOCAL INPATH '../data/files/flights_tiny.txt' OVERWRITE INTO TABLE flights_tiny; + +--1. test1 +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from noop(part + distribute by p_mfgr + sort by p_name + ); + +-- 2. test1NoPTF +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from part +distribute by p_mfgr +sort by p_name; + +--3. testLeadLag +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +p_retailprice, sum(p_retailprice) as s1 over (rows between unbounded preceding and current row), +p_size, p_size - lag(p_size,1) as deltaSz +from noop(part +distribute by p_mfgr +sort by p_name +); + +-- 4. testLeadLagNoPTF +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +p_retailprice, sum(p_retailprice) as s1 over (rows between unbounded preceding and current row), +p_size, p_size - lag(p_size,1) as deltaSz +from part +distribute by p_mfgr +sort by p_name ; + +-- 5. testLeadLagNoPTFNoWindowing +select p_mfgr, p_name, p_size +from part +distribute by p_mfgr +sort by p_name ; + +-- 6. testJoinWithLeadLag +select p1.p_mfgr, p1.p_name, +p1.p_size, p1.p_size - lag(p1.p_size,1) as deltaSz +from part p1 join part p2 on p1.p_partkey = p2.p_partkey +distribute by p1.p_mfgr +sort by p1.p_name ; + +-- 7. testJoinWithNoop +select p_mfgr, p_name, +p_size, p_size - lag(p_size,1) as deltaSz +from noop ( (select p1.* from part p1 join part p2 on p1.p_partkey = p2.p_partkey) j +distribute by j.p_mfgr +sort by j.p_name); + +-- 8. testGroupByWithSWQ +select p_mfgr, p_name, p_size, min(p_retailprice), +rank() as r, +denserank() as dr, +p_size, p_size - lag(p_size,1) as deltaSz +from part +group by p_mfgr, p_name, p_size +distribute by p_mfgr +sort by p_name ; + +-- 9. testGroupByHavingWithSWQ +select p_mfgr, p_name, p_size, min(p_retailprice), +rank() as r, +denserank() as dr, +p_size, p_size - lag(p_size,1) as deltaSz +from part +group by p_mfgr, p_name, p_size +having p_size > 0 +distribute by p_mfgr +sort by p_name ; + +-- 10. testOnlyPTF +select p_mfgr, p_name, p_size +from noop(part +distribute by p_mfgr +sort by p_name); + +-- 11. testAlias +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from noop(part + distribute by p_mfgr + sort by p_name + ) abc; + +-- 12. testSWQAndPTFAndWhere +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +p_size, p_size - lag(p_size,1) as deltaSz +from noop(part + distribute by p_mfgr + sort by p_name + ) +where p_size > 0 +distribute by p_mfgr +sort by p_name; + +-- 13. testSWQAndPTFAndGBy +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +p_size, p_size - lag(p_size,1) as deltaSz +from noop(part + distribute by p_mfgr + sort by p_name + ) +group by p_mfgr, p_name, p_size +distribute by p_mfgr +sort by p_name; + +-- 14. testCountNoWindowing +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +count(p_size) as cd +from noop(part +distribute by p_mfgr +sort by p_name); + +-- 15. testCountWithWindowing +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +count(p_size) as cd, +p_retailprice, sum(p_retailprice) as s1 over (rows between unbounded preceding and current row), +p_size, p_size - lag(p_size,1) as deltaSz +from noop(part +distribute by p_mfgr +sort by p_name); + + +-- 16. testCountInSubQ +select sub1.r, sub1.dr, sub1.cd, sub1.s1, sub1.deltaSz +from (select p_mfgr, p_name, +rank() as r, +denserank() as dr, +count(p_size) as cd, +p_retailprice, sum(p_retailprice) as s1 over (rows between unbounded preceding and current row), +p_size, p_size - lag(p_size,1) as deltaSz +from noop(part +distribute by p_mfgr +sort by p_name) +) sub1; + +-- 17. testJoin +select abc.* +from noop(part +distribute by p_mfgr +sort by p_name +) abc join part p1 on abc.p_partkey = p1.p_partkey; + +-- 18. testJoinRight +select abc.* +from part p1 join noop(part +distribute by p_mfgr +sort by p_name +) abc on abc.p_partkey = p1.p_partkey; + +-- 19. testJoinWithWindowing +select abc.p_mfgr, abc.p_name, +rank() as r, +denserank() as dr, +abc.p_retailprice, sum(abc.p_retailprice) as s1 over (rows between unbounded preceding and current row), +abc.p_size, abc.p_size - lag(abc.p_size,1) as deltaSz +from noop(part +distribute by p_mfgr +sort by p_name +) abc join part p1 on abc.p_partkey = p1.p_partkey +distribute by abc.p_mfgr +sort by abc.p_name ; + +-- 20. testMixedCaseAlias +select p_mfgr, p_name, p_size, rank() as R +from noop(part +distribute by p_mfgr +sort by p_name, p_size desc); + +-- 21. testNoopWithMap +select p_mfgr, p_name, p_size, rank() as r +from noopwithmap(part +distribute by p_mfgr +sort by p_name, p_size desc); + +-- 22. testNoopWithMapWithWindowing +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from noopwithmap(part + distribute by p_mfgr + sort by p_name); + +-- 23. testHavingWithWindowingNoGBY +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from part +having p_size > 5 +distribute by p_mfgr +sort by p_name; + +-- 24. testHavingWithWindowingCondRankNoGBY +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from part +having r < 4 +distribute by p_mfgr +sort by p_name; + +-- 25. testHavingWithWindowingPTFNoGBY +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from noop(part +distribute by p_mfgr +sort by p_name) +having r < 4; + +-- 26. testFirstLast +select p_mfgr,p_name, p_size, +sum(p_size) as s2 over (rows between current row and current row), +first_value(p_size) as f over (w1) , +last_value(p_size, false) as l over (w1) +from part +distribute by p_mfgr +sort by p_mfgr +window w1 as rows between 2 preceding and 2 following; + +-- 27. testFirstLastWithWhere +select p_mfgr,p_name, p_size, +rank() as r, +sum(p_size) as s2 over (rows between current row and current row), +first_value(p_size) as f over (w1), +last_value(p_size, false) as l over (w1) +from part +where p_mfgr = 'Manufacturer#3' +distribute by p_mfgr +sort by p_mfgr +window w1 as rows between 2 preceding and 2 following; + +-- 28. testSumDelta +select p_mfgr,p_name, p_size, +sum(p_size - lag(p_size,1)) as deltaSum over (w1) +from part +distribute by p_mfgr +sort by p_mfgr +window w1 as rows between 2 preceding and 2 following ; + +-- 29. testSumWindow +select p_mfgr,p_name, p_size, +sum(p_size) as s1 over (w1), +sum(p_size) as s2 over (rows between current row and current row) +from part +distribute by p_mfgr +sort by p_mfgr +window w1 as rows between 2 preceding and 2 following; + +-- 30. testNoSortClause +select p_mfgr,p_name, p_size, +rank() as r, denserank() as dr +from part +distribute by p_mfgr +window w1 as rows between 2 preceding and 2 following; + +-- 31. testExpressions +select p_mfgr,p_name, p_size, +rank() as r, +denserank() as dr, +cumedist() as cud, +percentrank() as pr, +ntile(3) as nt, +count(p_size) as ca, +avg(p_size) as avg, +stddev(p_size) as st, +first_value(p_size % 5) as fv, +last_value(p_size) as lv, +first_value(p_size, true) as fvW1 over (w1) +from part +having p_size > 5 +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following; + +-- 32. testMultipleWindows +select p_mfgr,p_name, p_size, + rank() as r, denserank() as dr, +cumedist() as cud, +sum(p_size) as s1 over (rows between unbounded preceding and current row), +sum(p_size) as s2 over (range between p_size 5 less and current row), +first_value(p_size, true) as fv1 over (w1) +from part +having p_size > 5 +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following; + +-- 33. testFunctionChain +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from noop(noopwithmap(noop(part +distribute by p_mfgr +sort by p_mfgr, p_name +))); + +-- 34. testPTFAndWindowingInSubQ +select p_mfgr, p_name, +sub1.cd, sub1.s1 +from (select p_mfgr, p_name, +count(p_size) as cd, +p_retailprice, +sum(p_retailprice) as s1 over (w1) +from noop(part +distribute by p_mfgr +sort by p_name) +window w1 as rows between 2 preceding and 2 following +) sub1 ; + +-- 35. testCountStar +select p_mfgr,p_name, p_size, +count(*) as c, +count(p_size) as ca, +first_value(p_size, true) as fvW1 over (w1) +from part +having p_size > 5 +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following; + +-- 36. testJoinWithWindowingWithCount +select abc.p_mfgr, abc.p_name, +rank() as r, +denserank() as dr, +count(abc.p_name) as cd, +abc.p_retailprice, sum(abc.p_retailprice) as s1 over (rows between unbounded preceding and current row), +abc.p_size, abc.p_size - lag(abc.p_size,1) as deltaSz +from noop(part +distribute by p_mfgr +sort by p_name +) abc join part p1 on abc.p_partkey = p1.p_partkey +distribute by abc.p_mfgr +sort by abc.p_name; + +-- 37. testSumDelta +select p_mfgr,p_name, p_size, +sum(p_size - lag(p_size,1)) as deltaSum +from part +distribute by p_mfgr +sort by p_mfgr +window w1 as rows between 2 preceding and 2 following ; + +-- 38. testUDAFs +select p_mfgr,p_name, p_size, +sum(p_retailprice) as s over (w1), +min(p_retailprice) as mi over (w1), +max(p_retailprice) as ma over (w1), +avg(p_retailprice) as ag over (w1) +from part +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following; + +-- 39. testUDAFsWithPTFWithGBY +select p_mfgr,p_name, p_size, p_retailprice, +sum(p_retailprice) as s over (w1), +min(p_retailprice) as mi , +max(p_retailprice) as ma , +avg(p_retailprice) as ag over (w1) +from part +group by p_mfgr,p_name, p_size, p_retailprice +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following; + +-- 40. testSTATs +select p_mfgr,p_name, p_size, +stddev(p_retailprice) as sdev over (w1), +stddev_pop(p_retailprice) as sdev_pop over (w1), +collect_set(p_size) as uniq_size over (w1), +variance(p_retailprice) as var over (w1), +corr(p_size, p_retailprice) as cor over (w1), +covar_pop(p_size, p_retailprice) as covarp over (w1) +from part +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following; + +-- 41. testDISTs +select p_mfgr,p_name, p_size, +histogram_numeric(p_retailprice, 5) as hist over (w1), +percentile(p_partkey, 0.5) as per over (w1), +rownumber() as rn +from part +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following; + +-- 42. testDistinctInSelectWithPTF +select DISTINCT p_mfgr, p_name, p_size +from noop(part +distribute by p_mfgr +sort by p_name); + +-- 43. testUDAFsNoWindowingNoPTFNoGBY +select p_mfgr,p_name, p_retailprice, +sum(p_retailprice) as s, +min(p_retailprice) as mi, +max(p_retailprice) as ma, +avg(p_retailprice) as av +from part +distribute by p_mfgr +sort by p_mfgr, p_name; + +-- 44. testViewAsTableInputWithWindowing +create view IF NOT EXISTS mfgr_price_view as +select p_mfgr, p_brand, +sum(p_retailprice) as s +from part +group by p_mfgr, p_brand; + +select p_mfgr, p_brand, s, +sum(s) as s1 over (w1) +from mfgr_price_view +distribute by p_mfgr +sort by p_mfgr +window w1 as rows between 2 preceding and current row; + +-- 45. testViewAsTableInputToPTF +select p_mfgr, p_brand, s, +sum(s) as s1 over (w1) +from noop(mfgr_price_view +distribute by p_mfgr +sort by p_mfgr) +window w1 as rows between 2 preceding and current row; + +-- 46. testCreateViewWithWindowingQuery +create view IF NOT EXISTS mfgr_brand_price_view as +select p_mfgr, p_brand, +sum(p_retailprice) as s over (w1) +from part +distribute by p_mfgr +sort by p_mfgr +window w1 as rows between 2 preceding and current row; + +select * from mfgr_brand_price_view; + +-- 47. testLateralViews +select p_mfgr, p_name, +lv_col, p_size, sum(p_size) as s over (w1) +from (select p_mfgr, p_name, p_size, array(1,2,3) arr from part) p +lateral view explode(arr) part_lv as lv_col +distribute by p_mfgr +sort by p_name +window w1 as rows between 2 preceding and current row; + +-- 48. testWindowingPTFWithPartRC +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from noop(part_rc +distribute by p_mfgr +sort by p_name); + +-- 49. testWindowingPTFWithPartSeqFile +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from noop(part_seq +distribute by p_mfgr +sort by p_name); + +-- 50. testConstExprInSelect +select 'tst1' as key, count(1) as value from part; + +-- 51. testMultipleInserts3SWQs +CREATE TABLE part_1( +p_mfgr STRING, +p_name STRING, +p_size INT, +r INT, +dr INT, +s DOUBLE); + +CREATE TABLE part_2( +p_mfgr STRING, +p_name STRING, +p_size INT, +r INT, +dr INT, +cud INT, +s1 DOUBLE, +s2 DOUBLE, +fv1 INT); + +CREATE TABLE part_3( +p_mfgr STRING, +p_name STRING, +p_size INT, +c INT, +ca INT, +fv INT); + +from part +INSERT OVERWRITE TABLE part_1 +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s over (rows between unbounded preceding and current row) +distribute by p_mfgr +sort by p_name +INSERT OVERWRITE TABLE part_2 +select p_mfgr,p_name, p_size, +rank() as r, denserank() as dr, +cumedist() as cud, +sum(p_size) as s1 over (rows between unbounded preceding and current row), +sum(p_size) as s2 over (range between p_size 5 less and current row), +first_value(p_size, true) as fv1 over (w1) +having p_size > 5 +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following +INSERT OVERWRITE TABLE part_3 +select p_mfgr,p_name, p_size, +count(*) as c, +count(p_size) as ca, +first_value(p_size, true) as fv over (w1) +having p_size > 5 +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following; + +select * from part_1; + +select * from part_2; + +select * from part_3; + +-- 52. testGroupByHavingWithSWQAndAlias +select p_mfgr, p_name, p_size, min(p_retailprice) as mi, +rank() as r, +denserank() as dr, +p_size, p_size - lag(p_size,1) as deltaSz +from part +group by p_mfgr, p_name, p_size +having p_size > 0 +distribute by p_mfgr +sort by p_name; + +-- 53. testMultipleRangeWindows +select p_mfgr,p_name, p_size, +sum(p_size) as s2 over (range between p_size 10 less and current row), +sum(p_size) as s1 over (range between current row and p_size 10 more ) +from part +distribute by p_mfgr +sort by p_mfgr, p_size +window w1 as rows between 2 preceding and 2 following; + +-- 54. testMultipleInserts2SWQsWithPTF +CREATE TABLE part_4( +p_mfgr STRING, +p_name STRING, +p_size INT, +r INT, +dr INT, +s DOUBLE); + +CREATE TABLE part_5( +p_mfgr STRING, +p_name STRING, +p_size INT, +s1 INT, +s2 INT, +r INT, +dr INT, +cud DOUBLE, +fv1 INT); + +from noop(part +distribute by p_mfgr +sort by p_name) +INSERT OVERWRITE TABLE part_4 select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s over (rows between unbounded preceding and current row) +distribute by p_mfgr +sort by p_name +INSERT OVERWRITE TABLE part_5 select p_mfgr,p_name, p_size, +sum(p_size) as s1 over (rows between unbounded preceding and current row), +sum(p_size) as s2 over (range between p_size 5 less and current row), +rank() as r, +denserank() as dr, +cumedist() as cud, +first_value(p_size, true) as fv1 over (w1) +having p_size > 5 +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following; + +select * from part_4; + +select * from part_5; + +-- 55. testPartOrderInUDAFInvoke +select p_mfgr, p_name, p_size, +sum(p_size) as s over (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following) +from part; + +-- 56. testPartOrderInWdwDef +select p_mfgr, p_name, p_size, +sum(p_size) as s over (w1) +from part +window w1 as distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following; + +-- 57. testDefaultPartitioningSpecRules +select p_mfgr, p_name, p_size, +sum(p_size) as s over (w1), + sum(p_size) as s2 over(w2) +from part +sort by p_name +window w1 as distribute by p_mfgr rows between 2 preceding and 2 following, + w2 as distribute by p_mfgr sort by p_name; + +-- 58. testWindowCrossReference +select p_mfgr, p_name, p_size, +sum(p_size) as s1 over (w1), +sum(p_size) as s2 over (w2) +from part +window w1 as distribute by p_mfgr sort by p_mfgr rows between 2 preceding and 2 following, + w2 as w1; + + +-- 59. testWindowInheritance +select p_mfgr, p_name, p_size, +sum(p_size) as s1 over (w1), +sum(p_size) as s2 over (w2) +from part +window w1 as distribute by p_mfgr sort by p_mfgr rows between 2 preceding and 2 following, + w2 as w1 rows between unbounded preceding and current row; + + +-- 60. testWindowForwardReference +select p_mfgr, p_name, p_size, +sum(p_size) as s1 over (w1), +sum(p_size) as s2 over (w2), +sum(p_size) as s3 over (w3) +from part +distribute by p_mfgr +sort by p_mfgr +window w1 as rows between 2 preceding and 2 following, + w2 as w3, + w3 as rows between unbounded preceding and current row; + + +-- 61. testWindowDefinitionPropagation +select p_mfgr, p_name, p_size, +sum(p_size) as s1 over (w1), +sum(p_size) as s2 over (w2), +sum(p_size) as s3 over (w3 rows between 2 preceding and 2 following) +from part +distribute by p_mfgr +sort by p_mfgr +window w1 as rows between 2 preceding and 2 following, + w2 as w3, + w3 as rows between unbounded preceding and current row; + +-- 62. testDistinctWithWindowing +select DISTINCT p_mfgr, p_name, p_size, +sum(p_size) as s over (w1) +from part +distribute by p_mfgr +sort by p_name +window w1 as rows between 2 preceding and 2 following; + +-- 63. testMulti2OperatorsFunctionChainWithMap +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +p_size, sum(p_size) as s1 over (rows between unbounded preceding and current row) +from noop( + noopwithmap( + noop( + noop(part + distribute by p_mfgr + sort by p_mfgr) + ) + distribute by p_mfgr,p_name + sort by p_mfgr,p_name) + distribute by p_mfgr,p_name + sort by p_mfgr,p_name) ; + +-- 64. testMulti3OperatorsFunctionChain +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +p_size, sum(p_size) as s1 over (rows between unbounded preceding and current row) +from noop( + noop( + noop( + noop(part + distribute by p_mfgr + sort by p_mfgr) + ) + distribute by p_mfgr,p_name + sort by p_mfgr,p_name) + distribute by p_mfgr + sort by p_mfgr ) ; + +-- 65. testMultiOperatorChainWithNoWindowing +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +p_size, sum(p_size) as s1 +from noop( + noop( + noop( + noop(part + distribute by p_mfgr,p_name + sort by p_mfgr,p_name) + ) + distribute by p_mfgr + sort by p_mfgr)); + + +-- 66. testMultiOperatorChainEndsWithNoopMap +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +p_size, sum(p_size) as s1 over (rows between unbounded preceding and current row) +from noopwithmap( + noop( + noop( + noop(part + distribute by p_mfgr,p_name + sort by p_mfgr,p_name) + ) + distribute by p_mfgr + sort by p_mfgr) + distribute by p_mfgr,p_name + sort by p_mfgr,p_name); + +--67. testMultiOperatorChainWithDiffPartitionForWindow1 +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +p_size, +sum(p_size) as s1 over (rows between unbounded preceding and current row), +sum(p_size) as s2 over (distribute by p_mfgr,p_name sort by p_mfgr,p_name rows between unbounded preceding and current row) +from noop( + noopwithmap( + noop(part + distribute by p_mfgr, p_name + sort by p_mfgr, p_name) + distribute by p_mfgr + sort by p_mfgr + )); + +--68. testMultiOperatorChainWithDiffPartitionForWindow2 +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +p_size, +sum(p_size) as s1 over (rows between unbounded preceding and current row), +sum(p_size) as s2 over (distribute by p_mfgr sort by p_mfgr rows between unbounded preceding and current row) +from noopwithmap( + noop( + noop(part + distribute by p_mfgr, p_name + sort by p_mfgr, p_name) + )); + +-- 69. basic Npath test +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from npath( + 'LATE.LATE+', + 'LATE', arr_delay > 15, + 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath as tpath' + on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + ); + +-- 70. testRankWithPartitioning +select p_mfgr, p_name, p_size, +rank() as r over (distribute by p_mfgr sort by p_name ) +from part; diff --git ql/src/test/results/clientnegative/ptf_negative_AggrFuncsWithNoGBYNoPartDef.q.out ql/src/test/results/clientnegative/ptf_negative_AggrFuncsWithNoGBYNoPartDef.q.out new file mode 100644 index 0000000..81d37f5 --- /dev/null +++ ql/src/test/results/clientnegative/ptf_negative_AggrFuncsWithNoGBYNoPartDef.q.out @@ -0,0 +1,36 @@ +PREHOOK: query: DROP TABLE part +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE part +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@part +PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +PREHOOK: type: LOAD +PREHOOK: Output: default@part +POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +POSTHOOK: type: LOAD +POSTHOOK: Output: default@part +FAILED: SemanticException 6:5 No partition specification associated with start of PTF chain . Error encountered near token 's1' diff --git ql/src/test/results/clientnegative/ptf_negative_AmbiguousWindowDefn.q.out ql/src/test/results/clientnegative/ptf_negative_AmbiguousWindowDefn.q.out new file mode 100644 index 0000000..ee473b9 --- /dev/null +++ ql/src/test/results/clientnegative/ptf_negative_AmbiguousWindowDefn.q.out @@ -0,0 +1,36 @@ +PREHOOK: query: DROP TABLE part +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE part +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@part +PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +PREHOOK: type: LOAD +PREHOOK: Output: default@part +POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +POSTHOOK: type: LOAD +POSTHOOK: Output: default@part +FAILED: SemanticException Window Spec w3 refers to an unknown source diff --git ql/src/test/results/clientnegative/ptf_negative_DuplicateWindowAlias.q.out ql/src/test/results/clientnegative/ptf_negative_DuplicateWindowAlias.q.out new file mode 100644 index 0000000..8ada2df --- /dev/null +++ ql/src/test/results/clientnegative/ptf_negative_DuplicateWindowAlias.q.out @@ -0,0 +1,36 @@ +PREHOOK: query: DROP TABLE part +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE part +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@part +PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +PREHOOK: type: LOAD +PREHOOK: Output: default@part +POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +POSTHOOK: type: LOAD +POSTHOOK: Output: default@part +FAILED: SemanticException 10:7 Duplicate definition of window w2 is not allowed. Error encountered near token 'w2' diff --git ql/src/test/results/clientnegative/ptf_negative_HavingLead.q.out ql/src/test/results/clientnegative/ptf_negative_HavingLead.q.out new file mode 100644 index 0000000..1d5594d --- /dev/null +++ ql/src/test/results/clientnegative/ptf_negative_HavingLead.q.out @@ -0,0 +1,36 @@ +PREHOOK: query: DROP TABLE part +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE part +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@part +PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +PREHOOK: type: LOAD +PREHOOK: Output: default@part +POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +POSTHOOK: type: LOAD +POSTHOOK: Output: default@part +FAILED: SemanticException 6:23 Query has no windowing or group by clause: Unsupported place for having. Error encountered near token 'p_size' diff --git ql/src/test/results/clientnegative/ptf_negative_HavingLeadWithPTF.q.out ql/src/test/results/clientnegative/ptf_negative_HavingLeadWithPTF.q.out new file mode 100644 index 0000000..3689cae --- /dev/null +++ ql/src/test/results/clientnegative/ptf_negative_HavingLeadWithPTF.q.out @@ -0,0 +1,36 @@ +PREHOOK: query: DROP TABLE part +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE part +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@part +PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +PREHOOK: type: LOAD +PREHOOK: Output: default@part +POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +POSTHOOK: type: LOAD +POSTHOOK: Output: default@part +FAILED: SemanticException 8:23 Query has no windowing or group by clause: Unsupported place for having. Error encountered near token 'p_size' diff --git ql/src/test/results/clientnegative/ptf_negative_IncompatibleDistributeClause.q.out ql/src/test/results/clientnegative/ptf_negative_IncompatibleDistributeClause.q.out new file mode 100644 index 0000000..9a50c4d --- /dev/null +++ ql/src/test/results/clientnegative/ptf_negative_IncompatibleDistributeClause.q.out @@ -0,0 +1,36 @@ +PREHOOK: query: DROP TABLE part +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE part +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@part +PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +PREHOOK: type: LOAD +PREHOOK: Output: default@part +POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +POSTHOOK: type: LOAD +POSTHOOK: Output: default@part +FAILED: SemanticException Window Function 'sum((TOK_TABLE_OR_COL p_size)) w1 partitionColumns=[(TOK_TABLE_OR_COL p_name)] window(start=range(2 PRECEDING), end=range(2 FOLLOWING)) as s' has an incompatible partition clause diff --git ql/src/test/results/clientnegative/ptf_negative_IncompatibleOrderInWindowDefs.q.out ql/src/test/results/clientnegative/ptf_negative_IncompatibleOrderInWindowDefs.q.out new file mode 100644 index 0000000..df00a35 --- /dev/null +++ ql/src/test/results/clientnegative/ptf_negative_IncompatibleOrderInWindowDefs.q.out @@ -0,0 +1,36 @@ +PREHOOK: query: DROP TABLE part +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE part +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@part +PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +PREHOOK: type: LOAD +PREHOOK: Output: default@part +POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +POSTHOOK: type: LOAD +POSTHOOK: Output: default@part +FAILED: SemanticException Window Function 'sum((TOK_TABLE_OR_COL p_size)) w2 partitionColumns=[(TOK_TABLE_OR_COL p_mfgr)] orderColumns=[(TOK_TABLE_OR_COL p_name) ASC] window(start=range(Unbounded PRECEDING), end=currentRow) as s2' has an incompatible order clause diff --git ql/src/test/results/clientnegative/ptf_negative_IncompatiblePartitionInWindowDefs.q.out ql/src/test/results/clientnegative/ptf_negative_IncompatiblePartitionInWindowDefs.q.out new file mode 100644 index 0000000..e9b943b --- /dev/null +++ ql/src/test/results/clientnegative/ptf_negative_IncompatiblePartitionInWindowDefs.q.out @@ -0,0 +1,36 @@ +PREHOOK: query: DROP TABLE part +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE part +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@part +PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +PREHOOK: type: LOAD +PREHOOK: Output: default@part +POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +POSTHOOK: type: LOAD +POSTHOOK: Output: default@part +FAILED: SemanticException Window Function 'sum((TOK_TABLE_OR_COL p_size)) w2 partitionColumns=[(TOK_TABLE_OR_COL p_name)] orderColumns=[(TOK_TABLE_OR_COL p_name) ASC] window(start=range(Unbounded PRECEDING), end=currentRow) as s2' has an incompatible partition clause diff --git ql/src/test/results/clientnegative/ptf_negative_IncompatibleSortClause.q.out ql/src/test/results/clientnegative/ptf_negative_IncompatibleSortClause.q.out new file mode 100644 index 0000000..529f19f --- /dev/null +++ ql/src/test/results/clientnegative/ptf_negative_IncompatibleSortClause.q.out @@ -0,0 +1,36 @@ +PREHOOK: query: DROP TABLE part +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE part +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@part +PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +PREHOOK: type: LOAD +PREHOOK: Output: default@part +POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +POSTHOOK: type: LOAD +POSTHOOK: Output: default@part +FAILED: SemanticException Window Function 'sum((TOK_TABLE_OR_COL p_size)) w1 partitionColumns=[(TOK_TABLE_OR_COL p_mfgr)] orderColumns=[(TOK_TABLE_OR_COL p_name) ASC] window(start=range(2 PRECEDING), end=range(2 FOLLOWING)) as s' has an incompatible order clause diff --git ql/src/test/results/clientnegative/ptf_negative_InvalidValueBoundary.q.out ql/src/test/results/clientnegative/ptf_negative_InvalidValueBoundary.q.out new file mode 100644 index 0000000..961a979 --- /dev/null +++ ql/src/test/results/clientnegative/ptf_negative_InvalidValueBoundary.q.out @@ -0,0 +1,36 @@ +PREHOOK: query: DROP TABLE part +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE part +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@part +PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +PREHOOK: type: LOAD +PREHOOK: Output: default@part +POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +POSTHOOK: type: LOAD +POSTHOOK: Output: default@part +FAILED: SemanticException Primitve type STRING not supported in Value Boundary expression diff --git ql/src/test/results/clientnegative/ptf_negative_JoinWithAmbigousAlias.q.out ql/src/test/results/clientnegative/ptf_negative_JoinWithAmbigousAlias.q.out new file mode 100644 index 0000000..cc950ef --- /dev/null +++ ql/src/test/results/clientnegative/ptf_negative_JoinWithAmbigousAlias.q.out @@ -0,0 +1,36 @@ +PREHOOK: query: DROP TABLE part +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE part +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@part +PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +PREHOOK: type: LOAD +PREHOOK: Output: default@part +POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +POSTHOOK: type: LOAD +POSTHOOK: Output: default@part +FAILED: SemanticException [Error 10008]: Line 8:11 Ambiguous table alias 'part' diff --git ql/src/test/results/clientnegative/ptf_negative_NoSortNoDistByClause.q.out ql/src/test/results/clientnegative/ptf_negative_NoSortNoDistByClause.q.out new file mode 100644 index 0000000..4f241a4 --- /dev/null +++ ql/src/test/results/clientnegative/ptf_negative_NoSortNoDistByClause.q.out @@ -0,0 +1,36 @@ +PREHOOK: query: DROP TABLE part +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE part +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@part +PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +PREHOOK: type: LOAD +PREHOOK: Output: default@part +POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +POSTHOOK: type: LOAD +POSTHOOK: Output: default@part +FAILED: SemanticException 6:5 No partition specification associated with start of PTF chain . Error encountered near token '2' diff --git ql/src/test/results/clientnegative/ptf_negative_WhereLead.q.out ql/src/test/results/clientnegative/ptf_negative_WhereLead.q.out new file mode 100644 index 0000000..d490b7b --- /dev/null +++ ql/src/test/results/clientnegative/ptf_negative_WhereLead.q.out @@ -0,0 +1,55 @@ +PREHOOK: query: DROP TABLE part +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE part +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@part +PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +PREHOOK: type: LOAD +PREHOOK: Output: default@part +POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +POSTHOOK: type: LOAD +POSTHOOK: Output: default@part +PREHOOK: query: -- testWhereLead +select p_mfgr,p_name, p_size +from part +where lead(p_size, 1) <= p_size +distribute by p_mfgr +sort by p_name +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +Execution failed with exit status: 2 +Obtaining error information + +Task failed! +Task ID: + Stage-1 + +Logs: + +#### A masked pattern was here #### +FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.MapRedTask diff --git ql/src/test/results/clientnegative/ptf_negative_WhereWithRankCond.q.out ql/src/test/results/clientnegative/ptf_negative_WhereWithRankCond.q.out new file mode 100644 index 0000000..277b07a --- /dev/null +++ ql/src/test/results/clientnegative/ptf_negative_WhereWithRankCond.q.out @@ -0,0 +1,36 @@ +PREHOOK: query: DROP TABLE part +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE part +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@part +PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +PREHOOK: type: LOAD +PREHOOK: Output: default@part +POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +POSTHOOK: type: LOAD +POSTHOOK: Output: default@part +FAILED: SemanticException [Error 10004]: Line 7:6 Invalid table alias or column reference 'r': (possible column names are: p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment) diff --git ql/src/test/results/clientpositive/ptf_general_queries.q.out ql/src/test/results/clientpositive/ptf_general_queries.q.out new file mode 100644 index 0000000..8f354b2 --- /dev/null +++ ql/src/test/results/clientpositive/ptf_general_queries.q.out @@ -0,0 +1,4739 @@ +PREHOOK: query: DROP TABLE part +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE part +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE part_rc +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE part_rc +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE part_seq +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE part_seq +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE flights_tiny +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE flights_tiny +POSTHOOK: type: DROPTABLE +PREHOOK: query: -- data setup +CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- data setup +CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@part +PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +PREHOOK: type: LOAD +PREHOOK: Output: default@part +POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part +POSTHOOK: type: LOAD +POSTHOOK: Output: default@part +PREHOOK: query: CREATE TABLE part_rc( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) STORED AS RCFILE +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE part_rc( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) STORED AS RCFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@part_rc +PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part.rc' overwrite into table part_rc +PREHOOK: type: LOAD +PREHOOK: Output: default@part_rc +POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part.rc' overwrite into table part_rc +POSTHOOK: type: LOAD +POSTHOOK: Output: default@part_rc +PREHOOK: query: CREATE TABLE part_seq( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) STORED AS SEQUENCEFILE +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE part_seq( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) STORED AS SEQUENCEFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@part_seq +PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part.seq' overwrite into table part_seq +PREHOOK: type: LOAD +PREHOOK: Output: default@part_seq +POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part.seq' overwrite into table part_seq +POSTHOOK: type: LOAD +POSTHOOK: Output: default@part_seq +PREHOOK: query: create table flights_tiny ( +ORIGIN_CITY_NAME string, +DEST_CITY_NAME string, +YEAR int, +MONTH int, +DAY_OF_MONTH int, +ARR_DELAY float, +FL_NUM string +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table flights_tiny ( +ORIGIN_CITY_NAME string, +DEST_CITY_NAME string, +YEAR int, +MONTH int, +DAY_OF_MONTH int, +ARR_DELAY float, +FL_NUM string +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@flights_tiny +PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/flights_tiny.txt' OVERWRITE INTO TABLE flights_tiny +PREHOOK: type: LOAD +PREHOOK: Output: default@flights_tiny +POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/flights_tiny.txt' OVERWRITE INTO TABLE flights_tiny +POSTHOOK: type: LOAD +POSTHOOK: Output: default@flights_tiny +PREHOOK: query: --1. test1 +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from noop(part + distribute by p_mfgr + sort by p_name + ) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: --1. test1 +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from noop(part + distribute by p_mfgr + sort by p_name + ) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15 +Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3 +Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 5702.650000000001 +Manufacturer#1 almond aquamarine burnished black steel 28 5 4 7117.070000000001 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 8749.730000000001 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68 +Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38 +Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001 +Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 7222.02 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 8923.62 +Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68 +Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95 +Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34 +Manufacturer#3 almond antique misty red olive 1 4 4 6195.32 +Manufacturer#3 almond antique olive coral navajo 45 5 5 7532.61 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67 +Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09 +Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35 +Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 6047.27 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 7337.620000000001 +Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69 +Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004 +Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6208.18 +Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 7672.66 +PREHOOK: query: -- 2. test1NoPTF +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from part +distribute by p_mfgr +sort by p_name +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 2. test1NoPTF +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from part +distribute by p_mfgr +sort by p_name +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15 +Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3 +Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 5702.650000000001 +Manufacturer#1 almond aquamarine burnished black steel 28 5 4 7117.070000000001 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 8749.730000000001 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68 +Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38 +Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001 +Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 7222.02 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 8923.62 +Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68 +Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95 +Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34 +Manufacturer#3 almond antique misty red olive 1 4 4 6195.32 +Manufacturer#3 almond antique olive coral navajo 45 5 5 7532.61 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67 +Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09 +Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35 +Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 6047.27 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 7337.620000000001 +Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69 +Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004 +Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6208.18 +Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 7672.66 +PREHOOK: query: --3. testLeadLag +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +p_retailprice, sum(p_retailprice) as s1 over (rows between unbounded preceding and current row), +p_size, p_size - lag(p_size,1) as deltaSz +from noop(part +distribute by p_mfgr +sort by p_name +) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: --3. testLeadLag +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +p_retailprice, sum(p_retailprice) as s1 over (rows between unbounded preceding and current row), +p_size, p_size - lag(p_size,1) as deltaSz +from noop(part +distribute by p_mfgr +sort by p_name +) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 1 1 1173.15 1173.15 2 0 +Manufacturer#1 almond antique burnished rose metallic 1 1 1173.15 2346.3 2 0 +Manufacturer#1 almond antique chartreuse lavender yellow 3 2 1753.76 4100.06 34 32 +Manufacturer#1 almond antique salmon chartreuse burlywood 4 3 1602.59 5702.650000000001 6 -28 +Manufacturer#1 almond aquamarine burnished black steel 5 4 1414.42 7117.070000000001 28 22 +Manufacturer#1 almond aquamarine pink moccasin thistle 6 5 1632.66 8749.730000000001 42 14 +Manufacturer#2 almond antique violet chocolate turquoise 1 1 1690.68 1690.68 14 0 +Manufacturer#2 almond antique violet turquoise frosted 2 2 1800.7 3491.38 40 26 +Manufacturer#2 almond aquamarine midnight light salmon 3 3 2031.98 5523.360000000001 2 -38 +Manufacturer#2 almond aquamarine rose maroon antique 4 4 1698.66 7222.02 25 23 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 5 5 1701.6 8923.62 18 -7 +Manufacturer#3 almond antique chartreuse khaki white 1 1 1671.68 1671.68 17 0 +Manufacturer#3 almond antique forest lavender goldenrod 2 2 1190.27 2861.95 14 -3 +Manufacturer#3 almond antique metallic orange dim 3 3 1410.39 4272.34 19 5 +Manufacturer#3 almond antique misty red olive 4 4 1922.98 6195.32 1 -18 +Manufacturer#3 almond antique olive coral navajo 5 5 1337.29 7532.61 45 44 +Manufacturer#4 almond antique gainsboro frosted violet 1 1 1620.67 1620.67 10 0 +Manufacturer#4 almond antique violet mint lemon 2 2 1375.42 2996.09 39 29 +Manufacturer#4 almond aquamarine floral ivory bisque 3 3 1206.26 4202.35 27 -12 +Manufacturer#4 almond aquamarine yellow dodger mint 4 4 1844.92 6047.27 7 -20 +Manufacturer#4 almond azure aquamarine papaya violet 5 5 1290.35 7337.620000000001 12 5 +Manufacturer#5 almond antique blue firebrick mint 1 1 1789.69 1789.69 31 0 +Manufacturer#5 almond antique medium spring khaki 2 2 1611.66 3401.3500000000004 6 -25 +Manufacturer#5 almond antique sky peru orange 3 3 1788.73 5190.08 2 -4 +Manufacturer#5 almond aquamarine dodger light gainsboro 4 4 1018.1 6208.18 46 44 +Manufacturer#5 almond azure blanched chiffon midnight 5 5 1464.48 7672.66 23 -23 +PREHOOK: query: -- 4. testLeadLagNoPTF +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +p_retailprice, sum(p_retailprice) as s1 over (rows between unbounded preceding and current row), +p_size, p_size - lag(p_size,1) as deltaSz +from part +distribute by p_mfgr +sort by p_name +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 4. testLeadLagNoPTF +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +p_retailprice, sum(p_retailprice) as s1 over (rows between unbounded preceding and current row), +p_size, p_size - lag(p_size,1) as deltaSz +from part +distribute by p_mfgr +sort by p_name +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 1 1 1173.15 1173.15 2 0 +Manufacturer#1 almond antique burnished rose metallic 1 1 1173.15 2346.3 2 0 +Manufacturer#1 almond antique chartreuse lavender yellow 3 2 1753.76 4100.06 34 32 +Manufacturer#1 almond antique salmon chartreuse burlywood 4 3 1602.59 5702.650000000001 6 -28 +Manufacturer#1 almond aquamarine burnished black steel 5 4 1414.42 7117.070000000001 28 22 +Manufacturer#1 almond aquamarine pink moccasin thistle 6 5 1632.66 8749.730000000001 42 14 +Manufacturer#2 almond antique violet chocolate turquoise 1 1 1690.68 1690.68 14 0 +Manufacturer#2 almond antique violet turquoise frosted 2 2 1800.7 3491.38 40 26 +Manufacturer#2 almond aquamarine midnight light salmon 3 3 2031.98 5523.360000000001 2 -38 +Manufacturer#2 almond aquamarine rose maroon antique 4 4 1698.66 7222.02 25 23 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 5 5 1701.6 8923.62 18 -7 +Manufacturer#3 almond antique chartreuse khaki white 1 1 1671.68 1671.68 17 0 +Manufacturer#3 almond antique forest lavender goldenrod 2 2 1190.27 2861.95 14 -3 +Manufacturer#3 almond antique metallic orange dim 3 3 1410.39 4272.34 19 5 +Manufacturer#3 almond antique misty red olive 4 4 1922.98 6195.32 1 -18 +Manufacturer#3 almond antique olive coral navajo 5 5 1337.29 7532.61 45 44 +Manufacturer#4 almond antique gainsboro frosted violet 1 1 1620.67 1620.67 10 0 +Manufacturer#4 almond antique violet mint lemon 2 2 1375.42 2996.09 39 29 +Manufacturer#4 almond aquamarine floral ivory bisque 3 3 1206.26 4202.35 27 -12 +Manufacturer#4 almond aquamarine yellow dodger mint 4 4 1844.92 6047.27 7 -20 +Manufacturer#4 almond azure aquamarine papaya violet 5 5 1290.35 7337.620000000001 12 5 +Manufacturer#5 almond antique blue firebrick mint 1 1 1789.69 1789.69 31 0 +Manufacturer#5 almond antique medium spring khaki 2 2 1611.66 3401.3500000000004 6 -25 +Manufacturer#5 almond antique sky peru orange 3 3 1788.73 5190.08 2 -4 +Manufacturer#5 almond aquamarine dodger light gainsboro 4 4 1018.1 6208.18 46 44 +Manufacturer#5 almond azure blanched chiffon midnight 5 5 1464.48 7672.66 23 -23 +PREHOOK: query: -- 5. testLeadLagNoPTFNoWindowing +select p_mfgr, p_name, p_size +from part +distribute by p_mfgr +sort by p_name +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 5. testLeadLagNoPTFNoWindowing +select p_mfgr, p_name, p_size +from part +distribute by p_mfgr +sort by p_name +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#5 almond antique blue firebrick mint 31 +Manufacturer#1 almond antique burnished rose metallic 2 +Manufacturer#1 almond antique burnished rose metallic 2 +Manufacturer#3 almond antique chartreuse khaki white 17 +Manufacturer#1 almond antique chartreuse lavender yellow 34 +Manufacturer#3 almond antique forest lavender goldenrod 14 +Manufacturer#4 almond antique gainsboro frosted violet 10 +Manufacturer#5 almond antique medium spring khaki 6 +Manufacturer#3 almond antique metallic orange dim 19 +Manufacturer#3 almond antique misty red olive 1 +Manufacturer#3 almond antique olive coral navajo 45 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 +Manufacturer#5 almond antique sky peru orange 2 +Manufacturer#2 almond antique violet chocolate turquoise 14 +Manufacturer#4 almond antique violet mint lemon 39 +Manufacturer#2 almond antique violet turquoise frosted 40 +Manufacturer#1 almond aquamarine burnished black steel 28 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 +Manufacturer#4 almond aquamarine floral ivory bisque 27 +Manufacturer#2 almond aquamarine midnight light salmon 2 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 +Manufacturer#2 almond aquamarine rose maroon antique 25 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 +Manufacturer#4 almond aquamarine yellow dodger mint 7 +Manufacturer#4 almond azure aquamarine papaya violet 12 +Manufacturer#5 almond azure blanched chiffon midnight 23 +PREHOOK: query: -- 6. testJoinWithLeadLag +select p1.p_mfgr, p1.p_name, +p1.p_size, p1.p_size - lag(p1.p_size,1) as deltaSz +from part p1 join part p2 on p1.p_partkey = p2.p_partkey +distribute by p1.p_mfgr +sort by p1.p_name +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 6. testJoinWithLeadLag +select p1.p_mfgr, p1.p_name, +p1.p_size, p1.p_size - lag(p1.p_size,1) as deltaSz +from part p1 join part p2 on p1.p_partkey = p2.p_partkey +distribute by p1.p_mfgr +sort by p1.p_name +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 0 +Manufacturer#1 almond antique burnished rose metallic 2 0 +Manufacturer#1 almond antique burnished rose metallic 2 0 +Manufacturer#1 almond antique burnished rose metallic 2 0 +Manufacturer#1 almond antique chartreuse lavender yellow 34 32 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 -28 +Manufacturer#1 almond aquamarine burnished black steel 28 22 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 14 +Manufacturer#2 almond antique violet chocolate turquoise 14 0 +Manufacturer#2 almond antique violet turquoise frosted 40 26 +Manufacturer#2 almond aquamarine midnight light salmon 2 -38 +Manufacturer#2 almond aquamarine rose maroon antique 25 23 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 -7 +Manufacturer#3 almond antique chartreuse khaki white 17 0 +Manufacturer#3 almond antique forest lavender goldenrod 14 -3 +Manufacturer#3 almond antique metallic orange dim 19 5 +Manufacturer#3 almond antique misty red olive 1 -18 +Manufacturer#3 almond antique olive coral navajo 45 44 +Manufacturer#4 almond antique gainsboro frosted violet 10 0 +Manufacturer#4 almond antique violet mint lemon 39 29 +Manufacturer#4 almond aquamarine floral ivory bisque 27 -12 +Manufacturer#4 almond aquamarine yellow dodger mint 7 -20 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 +Manufacturer#5 almond antique blue firebrick mint 31 0 +Manufacturer#5 almond antique medium spring khaki 6 -25 +Manufacturer#5 almond antique sky peru orange 2 -4 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 44 +Manufacturer#5 almond azure blanched chiffon midnight 23 -23 +PREHOOK: query: -- 7. testJoinWithNoop +select p_mfgr, p_name, +p_size, p_size - lag(p_size,1) as deltaSz +from noop ( (select p1.* from part p1 join part p2 on p1.p_partkey = p2.p_partkey) j +distribute by j.p_mfgr +sort by j.p_name) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 7. testJoinWithNoop +select p_mfgr, p_name, +p_size, p_size - lag(p_size,1) as deltaSz +from noop ( (select p1.* from part p1 join part p2 on p1.p_partkey = p2.p_partkey) j +distribute by j.p_mfgr +sort by j.p_name) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 0 +Manufacturer#1 almond antique burnished rose metallic 2 0 +Manufacturer#1 almond antique burnished rose metallic 2 0 +Manufacturer#1 almond antique burnished rose metallic 2 0 +Manufacturer#1 almond antique chartreuse lavender yellow 34 32 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 -28 +Manufacturer#1 almond aquamarine burnished black steel 28 22 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 14 +Manufacturer#2 almond antique violet chocolate turquoise 14 0 +Manufacturer#2 almond antique violet turquoise frosted 40 26 +Manufacturer#2 almond aquamarine midnight light salmon 2 -38 +Manufacturer#2 almond aquamarine rose maroon antique 25 23 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 -7 +Manufacturer#3 almond antique chartreuse khaki white 17 0 +Manufacturer#3 almond antique forest lavender goldenrod 14 -3 +Manufacturer#3 almond antique metallic orange dim 19 5 +Manufacturer#3 almond antique misty red olive 1 -18 +Manufacturer#3 almond antique olive coral navajo 45 44 +Manufacturer#4 almond antique gainsboro frosted violet 10 0 +Manufacturer#4 almond antique violet mint lemon 39 29 +Manufacturer#4 almond aquamarine floral ivory bisque 27 -12 +Manufacturer#4 almond aquamarine yellow dodger mint 7 -20 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 +Manufacturer#5 almond antique blue firebrick mint 31 0 +Manufacturer#5 almond antique medium spring khaki 6 -25 +Manufacturer#5 almond antique sky peru orange 2 -4 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 44 +Manufacturer#5 almond azure blanched chiffon midnight 23 -23 +PREHOOK: query: -- 8. testGroupByWithSWQ +select p_mfgr, p_name, p_size, min(p_retailprice), +rank() as r, +denserank() as dr, +p_size, p_size - lag(p_size,1) as deltaSz +from part +group by p_mfgr, p_name, p_size +distribute by p_mfgr +sort by p_name +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 8. testGroupByWithSWQ +select p_mfgr, p_name, p_size, min(p_retailprice), +rank() as r, +denserank() as dr, +p_size, p_size - lag(p_size,1) as deltaSz +from part +group by p_mfgr, p_name, p_size +distribute by p_mfgr +sort by p_name +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 1173.15 1 1 2 0 +Manufacturer#1 almond antique chartreuse lavender yellow 34 1753.76 2 2 34 32 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 1602.59 3 3 6 -28 +Manufacturer#1 almond aquamarine burnished black steel 28 1414.42 4 4 28 22 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 1632.66 5 5 42 14 +Manufacturer#2 almond antique violet chocolate turquoise 14 1690.68 1 1 14 0 +Manufacturer#2 almond antique violet turquoise frosted 40 1800.7 2 2 40 26 +Manufacturer#2 almond aquamarine midnight light salmon 2 2031.98 3 3 2 -38 +Manufacturer#2 almond aquamarine rose maroon antique 25 1698.66 4 4 25 23 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 1701.6 5 5 18 -7 +Manufacturer#3 almond antique chartreuse khaki white 17 1671.68 1 1 17 0 +Manufacturer#3 almond antique forest lavender goldenrod 14 1190.27 2 2 14 -3 +Manufacturer#3 almond antique metallic orange dim 19 1410.39 3 3 19 5 +Manufacturer#3 almond antique misty red olive 1 1922.98 4 4 1 -18 +Manufacturer#3 almond antique olive coral navajo 45 1337.29 5 5 45 44 +Manufacturer#4 almond antique gainsboro frosted violet 10 1620.67 1 1 10 0 +Manufacturer#4 almond antique violet mint lemon 39 1375.42 2 2 39 29 +Manufacturer#4 almond aquamarine floral ivory bisque 27 1206.26 3 3 27 -12 +Manufacturer#4 almond aquamarine yellow dodger mint 7 1844.92 4 4 7 -20 +Manufacturer#4 almond azure aquamarine papaya violet 12 1290.35 5 5 12 5 +Manufacturer#5 almond antique blue firebrick mint 31 1789.69 1 1 31 0 +Manufacturer#5 almond antique medium spring khaki 6 1611.66 2 2 6 -25 +Manufacturer#5 almond antique sky peru orange 2 1788.73 3 3 2 -4 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 1018.1 4 4 46 44 +Manufacturer#5 almond azure blanched chiffon midnight 23 1464.48 5 5 23 -23 +PREHOOK: query: -- 9. testGroupByHavingWithSWQ +select p_mfgr, p_name, p_size, min(p_retailprice), +rank() as r, +denserank() as dr, +p_size, p_size - lag(p_size,1) as deltaSz +from part +group by p_mfgr, p_name, p_size +having p_size > 0 +distribute by p_mfgr +sort by p_name +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 9. testGroupByHavingWithSWQ +select p_mfgr, p_name, p_size, min(p_retailprice), +rank() as r, +denserank() as dr, +p_size, p_size - lag(p_size,1) as deltaSz +from part +group by p_mfgr, p_name, p_size +having p_size > 0 +distribute by p_mfgr +sort by p_name +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 1173.15 1 1 2 0 +Manufacturer#1 almond antique chartreuse lavender yellow 34 1753.76 2 2 34 32 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 1602.59 3 3 6 -28 +Manufacturer#1 almond aquamarine burnished black steel 28 1414.42 4 4 28 22 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 1632.66 5 5 42 14 +Manufacturer#2 almond antique violet chocolate turquoise 14 1690.68 1 1 14 0 +Manufacturer#2 almond antique violet turquoise frosted 40 1800.7 2 2 40 26 +Manufacturer#2 almond aquamarine midnight light salmon 2 2031.98 3 3 2 -38 +Manufacturer#2 almond aquamarine rose maroon antique 25 1698.66 4 4 25 23 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 1701.6 5 5 18 -7 +Manufacturer#3 almond antique chartreuse khaki white 17 1671.68 1 1 17 0 +Manufacturer#3 almond antique forest lavender goldenrod 14 1190.27 2 2 14 -3 +Manufacturer#3 almond antique metallic orange dim 19 1410.39 3 3 19 5 +Manufacturer#3 almond antique misty red olive 1 1922.98 4 4 1 -18 +Manufacturer#3 almond antique olive coral navajo 45 1337.29 5 5 45 44 +Manufacturer#4 almond antique gainsboro frosted violet 10 1620.67 1 1 10 0 +Manufacturer#4 almond antique violet mint lemon 39 1375.42 2 2 39 29 +Manufacturer#4 almond aquamarine floral ivory bisque 27 1206.26 3 3 27 -12 +Manufacturer#4 almond aquamarine yellow dodger mint 7 1844.92 4 4 7 -20 +Manufacturer#4 almond azure aquamarine papaya violet 12 1290.35 5 5 12 5 +Manufacturer#5 almond antique blue firebrick mint 31 1789.69 1 1 31 0 +Manufacturer#5 almond antique medium spring khaki 6 1611.66 2 2 6 -25 +Manufacturer#5 almond antique sky peru orange 2 1788.73 3 3 2 -4 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 1018.1 4 4 46 44 +Manufacturer#5 almond azure blanched chiffon midnight 23 1464.48 5 5 23 -23 +PREHOOK: query: -- 10. testOnlyPTF +select p_mfgr, p_name, p_size +from noop(part +distribute by p_mfgr +sort by p_name) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 10. testOnlyPTF +select p_mfgr, p_name, p_size +from noop(part +distribute by p_mfgr +sort by p_name) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 +Manufacturer#1 almond antique burnished rose metallic 2 +Manufacturer#1 almond antique chartreuse lavender yellow 34 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 +Manufacturer#1 almond aquamarine burnished black steel 28 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 +Manufacturer#2 almond antique violet chocolate turquoise 14 +Manufacturer#2 almond antique violet turquoise frosted 40 +Manufacturer#2 almond aquamarine midnight light salmon 2 +Manufacturer#2 almond aquamarine rose maroon antique 25 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 +Manufacturer#3 almond antique chartreuse khaki white 17 +Manufacturer#3 almond antique forest lavender goldenrod 14 +Manufacturer#3 almond antique metallic orange dim 19 +Manufacturer#3 almond antique misty red olive 1 +Manufacturer#3 almond antique olive coral navajo 45 +Manufacturer#4 almond antique gainsboro frosted violet 10 +Manufacturer#4 almond antique violet mint lemon 39 +Manufacturer#4 almond aquamarine floral ivory bisque 27 +Manufacturer#4 almond aquamarine yellow dodger mint 7 +Manufacturer#4 almond azure aquamarine papaya violet 12 +Manufacturer#5 almond antique blue firebrick mint 31 +Manufacturer#5 almond antique medium spring khaki 6 +Manufacturer#5 almond antique sky peru orange 2 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 +Manufacturer#5 almond azure blanched chiffon midnight 23 +PREHOOK: query: -- 11. testAlias +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from noop(part + distribute by p_mfgr + sort by p_name + ) abc +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 11. testAlias +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from noop(part + distribute by p_mfgr + sort by p_name + ) abc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15 +Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3 +Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 5702.650000000001 +Manufacturer#1 almond aquamarine burnished black steel 28 5 4 7117.070000000001 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 8749.730000000001 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68 +Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38 +Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001 +Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 7222.02 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 8923.62 +Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68 +Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95 +Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34 +Manufacturer#3 almond antique misty red olive 1 4 4 6195.32 +Manufacturer#3 almond antique olive coral navajo 45 5 5 7532.61 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67 +Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09 +Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35 +Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 6047.27 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 7337.620000000001 +Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69 +Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004 +Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6208.18 +Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 7672.66 +PREHOOK: query: -- 12. testSWQAndPTFAndWhere +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +p_size, p_size - lag(p_size,1) as deltaSz +from noop(part + distribute by p_mfgr + sort by p_name + ) +where p_size > 0 +distribute by p_mfgr +sort by p_name +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 12. testSWQAndPTFAndWhere +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +p_size, p_size - lag(p_size,1) as deltaSz +from noop(part + distribute by p_mfgr + sort by p_name + ) +where p_size > 0 +distribute by p_mfgr +sort by p_name +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 1 1 2 0 +Manufacturer#1 almond antique burnished rose metallic 2 1 1 2 0 +Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 34 32 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 6 -28 +Manufacturer#1 almond aquamarine burnished black steel 28 5 4 28 22 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 42 14 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 14 0 +Manufacturer#2 almond antique violet turquoise frosted 40 2 2 40 26 +Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 2 -38 +Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 25 23 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 18 -7 +Manufacturer#3 almond antique chartreuse khaki white 17 1 1 17 0 +Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 14 -3 +Manufacturer#3 almond antique metallic orange dim 19 3 3 19 5 +Manufacturer#3 almond antique misty red olive 1 4 4 1 -18 +Manufacturer#3 almond antique olive coral navajo 45 5 5 45 44 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 10 0 +Manufacturer#4 almond antique violet mint lemon 39 2 2 39 29 +Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 27 -12 +Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 7 -20 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 12 5 +Manufacturer#5 almond antique blue firebrick mint 31 1 1 31 0 +Manufacturer#5 almond antique medium spring khaki 6 2 2 6 -25 +Manufacturer#5 almond antique sky peru orange 2 3 3 2 -4 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 46 44 +Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 23 -23 +PREHOOK: query: -- 13. testSWQAndPTFAndGBy +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +p_size, p_size - lag(p_size,1) as deltaSz +from noop(part + distribute by p_mfgr + sort by p_name + ) +group by p_mfgr, p_name, p_size +distribute by p_mfgr +sort by p_name +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 13. testSWQAndPTFAndGBy +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +p_size, p_size - lag(p_size,1) as deltaSz +from noop(part + distribute by p_mfgr + sort by p_name + ) +group by p_mfgr, p_name, p_size +distribute by p_mfgr +sort by p_name +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 1 1 2 0 +Manufacturer#1 almond antique chartreuse lavender yellow 34 2 2 34 32 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 3 3 6 -28 +Manufacturer#1 almond aquamarine burnished black steel 28 4 4 28 22 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 5 5 42 14 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 14 0 +Manufacturer#2 almond antique violet turquoise frosted 40 2 2 40 26 +Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 2 -38 +Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 25 23 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 18 -7 +Manufacturer#3 almond antique chartreuse khaki white 17 1 1 17 0 +Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 14 -3 +Manufacturer#3 almond antique metallic orange dim 19 3 3 19 5 +Manufacturer#3 almond antique misty red olive 1 4 4 1 -18 +Manufacturer#3 almond antique olive coral navajo 45 5 5 45 44 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 10 0 +Manufacturer#4 almond antique violet mint lemon 39 2 2 39 29 +Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 27 -12 +Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 7 -20 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 12 5 +Manufacturer#5 almond antique blue firebrick mint 31 1 1 31 0 +Manufacturer#5 almond antique medium spring khaki 6 2 2 6 -25 +Manufacturer#5 almond antique sky peru orange 2 3 3 2 -4 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 46 44 +Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 23 -23 +PREHOOK: query: -- 14. testCountNoWindowing +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +count(p_size) as cd +from noop(part +distribute by p_mfgr +sort by p_name) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 14. testCountNoWindowing +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +count(p_size) as cd +from noop(part +distribute by p_mfgr +sort by p_name) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 1 1 6 +Manufacturer#1 almond antique burnished rose metallic 1 1 6 +Manufacturer#1 almond antique chartreuse lavender yellow 3 2 6 +Manufacturer#1 almond antique salmon chartreuse burlywood 4 3 6 +Manufacturer#1 almond aquamarine burnished black steel 5 4 6 +Manufacturer#1 almond aquamarine pink moccasin thistle 6 5 6 +Manufacturer#2 almond antique violet chocolate turquoise 1 1 5 +Manufacturer#2 almond antique violet turquoise frosted 2 2 5 +Manufacturer#2 almond aquamarine midnight light salmon 3 3 5 +Manufacturer#2 almond aquamarine rose maroon antique 4 4 5 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 5 5 5 +Manufacturer#3 almond antique chartreuse khaki white 1 1 5 +Manufacturer#3 almond antique forest lavender goldenrod 2 2 5 +Manufacturer#3 almond antique metallic orange dim 3 3 5 +Manufacturer#3 almond antique misty red olive 4 4 5 +Manufacturer#3 almond antique olive coral navajo 5 5 5 +Manufacturer#4 almond antique gainsboro frosted violet 1 1 5 +Manufacturer#4 almond antique violet mint lemon 2 2 5 +Manufacturer#4 almond aquamarine floral ivory bisque 3 3 5 +Manufacturer#4 almond aquamarine yellow dodger mint 4 4 5 +Manufacturer#4 almond azure aquamarine papaya violet 5 5 5 +Manufacturer#5 almond antique blue firebrick mint 1 1 5 +Manufacturer#5 almond antique medium spring khaki 2 2 5 +Manufacturer#5 almond antique sky peru orange 3 3 5 +Manufacturer#5 almond aquamarine dodger light gainsboro 4 4 5 +Manufacturer#5 almond azure blanched chiffon midnight 5 5 5 +PREHOOK: query: -- 15. testCountWithWindowing +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +count(p_size) as cd, +p_retailprice, sum(p_retailprice) as s1 over (rows between unbounded preceding and current row), +p_size, p_size - lag(p_size,1) as deltaSz +from noop(part +distribute by p_mfgr +sort by p_name) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 15. testCountWithWindowing +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +count(p_size) as cd, +p_retailprice, sum(p_retailprice) as s1 over (rows between unbounded preceding and current row), +p_size, p_size - lag(p_size,1) as deltaSz +from noop(part +distribute by p_mfgr +sort by p_name) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 1 1 6 1173.15 1173.15 2 0 +Manufacturer#1 almond antique burnished rose metallic 1 1 6 1173.15 2346.3 2 0 +Manufacturer#1 almond antique chartreuse lavender yellow 3 2 6 1753.76 4100.06 34 32 +Manufacturer#1 almond antique salmon chartreuse burlywood 4 3 6 1602.59 5702.650000000001 6 -28 +Manufacturer#1 almond aquamarine burnished black steel 5 4 6 1414.42 7117.070000000001 28 22 +Manufacturer#1 almond aquamarine pink moccasin thistle 6 5 6 1632.66 8749.730000000001 42 14 +Manufacturer#2 almond antique violet chocolate turquoise 1 1 5 1690.68 1690.68 14 0 +Manufacturer#2 almond antique violet turquoise frosted 2 2 5 1800.7 3491.38 40 26 +Manufacturer#2 almond aquamarine midnight light salmon 3 3 5 2031.98 5523.360000000001 2 -38 +Manufacturer#2 almond aquamarine rose maroon antique 4 4 5 1698.66 7222.02 25 23 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 5 5 5 1701.6 8923.62 18 -7 +Manufacturer#3 almond antique chartreuse khaki white 1 1 5 1671.68 1671.68 17 0 +Manufacturer#3 almond antique forest lavender goldenrod 2 2 5 1190.27 2861.95 14 -3 +Manufacturer#3 almond antique metallic orange dim 3 3 5 1410.39 4272.34 19 5 +Manufacturer#3 almond antique misty red olive 4 4 5 1922.98 6195.32 1 -18 +Manufacturer#3 almond antique olive coral navajo 5 5 5 1337.29 7532.61 45 44 +Manufacturer#4 almond antique gainsboro frosted violet 1 1 5 1620.67 1620.67 10 0 +Manufacturer#4 almond antique violet mint lemon 2 2 5 1375.42 2996.09 39 29 +Manufacturer#4 almond aquamarine floral ivory bisque 3 3 5 1206.26 4202.35 27 -12 +Manufacturer#4 almond aquamarine yellow dodger mint 4 4 5 1844.92 6047.27 7 -20 +Manufacturer#4 almond azure aquamarine papaya violet 5 5 5 1290.35 7337.620000000001 12 5 +Manufacturer#5 almond antique blue firebrick mint 1 1 5 1789.69 1789.69 31 0 +Manufacturer#5 almond antique medium spring khaki 2 2 5 1611.66 3401.3500000000004 6 -25 +Manufacturer#5 almond antique sky peru orange 3 3 5 1788.73 5190.08 2 -4 +Manufacturer#5 almond aquamarine dodger light gainsboro 4 4 5 1018.1 6208.18 46 44 +Manufacturer#5 almond azure blanched chiffon midnight 5 5 5 1464.48 7672.66 23 -23 +PREHOOK: query: -- 16. testCountInSubQ +select sub1.r, sub1.dr, sub1.cd, sub1.s1, sub1.deltaSz +from (select p_mfgr, p_name, +rank() as r, +denserank() as dr, +count(p_size) as cd, +p_retailprice, sum(p_retailprice) as s1 over (rows between unbounded preceding and current row), +p_size, p_size - lag(p_size,1) as deltaSz +from noop(part +distribute by p_mfgr +sort by p_name) +) sub1 +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 16. testCountInSubQ +select sub1.r, sub1.dr, sub1.cd, sub1.s1, sub1.deltaSz +from (select p_mfgr, p_name, +rank() as r, +denserank() as dr, +count(p_size) as cd, +p_retailprice, sum(p_retailprice) as s1 over (rows between unbounded preceding and current row), +p_size, p_size - lag(p_size,1) as deltaSz +from noop(part +distribute by p_mfgr +sort by p_name) +) sub1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +1 1 6 1173.15 0 +1 1 6 2346.3 0 +3 2 6 4100.06 32 +4 3 6 5702.650000000001 -28 +5 4 6 7117.070000000001 22 +6 5 6 8749.730000000001 14 +1 1 5 1690.68 0 +2 2 5 3491.38 26 +3 3 5 5523.360000000001 -38 +4 4 5 7222.02 23 +5 5 5 8923.62 -7 +1 1 5 1671.68 0 +2 2 5 2861.95 -3 +3 3 5 4272.34 5 +4 4 5 6195.32 -18 +5 5 5 7532.61 44 +1 1 5 1620.67 0 +2 2 5 2996.09 29 +3 3 5 4202.35 -12 +4 4 5 6047.27 -20 +5 5 5 7337.620000000001 5 +1 1 5 1789.69 0 +2 2 5 3401.3500000000004 -25 +3 3 5 5190.08 -4 +4 4 5 6208.18 44 +5 5 5 7672.66 -23 +PREHOOK: query: -- 17. testJoin +select abc.* +from noop(part +distribute by p_mfgr +sort by p_name +) abc join part p1 on abc.p_partkey = p1.p_partkey +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 17. testJoin +select abc.* +from noop(part +distribute by p_mfgr +sort by p_name +) abc join part p1 on abc.p_partkey = p1.p_partkey +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +15103 almond aquamarine dodger light gainsboro Manufacturer#5 Brand#53 ECONOMY BURNISHED STEEL 46 LG PACK 1018.1 packages hinder carefu +17273 almond antique forest lavender goldenrod Manufacturer#3 Brand#35 PROMO ANODIZED TIN 14 JUMBO CASE 1190.27 along the +17927 almond aquamarine yellow dodger mint Manufacturer#4 Brand#41 ECONOMY BRUSHED COPPER 7 SM PKG 1844.92 ites. eve +33357 almond azure aquamarine papaya violet Manufacturer#4 Brand#41 STANDARD ANODIZED TIN 12 WRAP CASE 1290.35 reful +40982 almond antique misty red olive Manufacturer#3 Brand#32 ECONOMY PLATED COPPER 1 LG PKG 1922.98 c foxes can s +42669 almond antique medium spring khaki Manufacturer#5 Brand#51 STANDARD BURNISHED TIN 6 MED CAN 1611.66 sits haggl +45261 almond aquamarine floral ivory bisque Manufacturer#4 Brand#42 SMALL PLATED STEEL 27 WRAP CASE 1206.26 careful +48427 almond antique violet mint lemon Manufacturer#4 Brand#42 PROMO POLISHED STEEL 39 SM CASE 1375.42 hely ironic i +49671 almond antique gainsboro frosted violet Manufacturer#4 Brand#41 SMALL BRUSHED BRASS 10 SM BOX 1620.67 ccounts run quick +65667 almond aquamarine pink moccasin thistle Manufacturer#1 Brand#12 LARGE BURNISHED STEEL 42 JUMBO CASE 1632.66 e across the expr +78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith +85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull +86428 almond aquamarine burnished black steel Manufacturer#1 Brand#12 STANDARD ANODIZED STEEL 28 WRAP BAG 1414.42 arefully +90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl +105685 almond antique violet chocolate turquoise Manufacturer#2 Brand#22 MEDIUM ANODIZED COPPER 14 MED CAN 1690.68 ly pending requ +110592 almond antique salmon chartreuse burlywood Manufacturer#1 Brand#15 PROMO BURNISHED NICKEL 6 JUMBO PKG 1602.59 to the furiously +112398 almond antique metallic orange dim Manufacturer#3 Brand#32 MEDIUM BURNISHED BRASS 19 JUMBO JAR 1410.39 ole car +121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h +121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h +121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h +121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h +132666 almond aquamarine rose maroon antique Manufacturer#2 Brand#24 SMALL POLISHED NICKEL 25 MED BOX 1698.66 even +144293 almond antique olive coral navajo Manufacturer#3 Brand#34 STANDARD POLISHED STEEL 45 JUMBO CAN 1337.29 ag furiously about +146985 almond aquamarine midnight light salmon Manufacturer#2 Brand#23 MEDIUM BURNISHED COPPER 2 SM CASE 2031.98 s cajole caref +155733 almond antique sky peru orange Manufacturer#5 Brand#53 SMALL PLATED BRASS 2 WRAP DRUM 1788.73 furiously. bra +191709 almond antique violet turquoise frosted Manufacturer#2 Brand#22 ECONOMY POLISHED STEEL 40 MED BOX 1800.7 haggle +192697 almond antique blue firebrick mint Manufacturer#5 Brand#52 MEDIUM BURNISHED TIN 31 LG DRUM 1789.69 ickly ir +195606 almond aquamarine sandy cyan gainsboro Manufacturer#2 Brand#25 STANDARD PLATED TIN 18 SM PKG 1701.6 ic de +PREHOOK: query: -- 18. testJoinRight +select abc.* +from part p1 join noop(part +distribute by p_mfgr +sort by p_name +) abc on abc.p_partkey = p1.p_partkey +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 18. testJoinRight +select abc.* +from part p1 join noop(part +distribute by p_mfgr +sort by p_name +) abc on abc.p_partkey = p1.p_partkey +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +15103 almond aquamarine dodger light gainsboro Manufacturer#5 Brand#53 ECONOMY BURNISHED STEEL 46 LG PACK 1018.1 packages hinder carefu +17273 almond antique forest lavender goldenrod Manufacturer#3 Brand#35 PROMO ANODIZED TIN 14 JUMBO CASE 1190.27 along the +17927 almond aquamarine yellow dodger mint Manufacturer#4 Brand#41 ECONOMY BRUSHED COPPER 7 SM PKG 1844.92 ites. eve +33357 almond azure aquamarine papaya violet Manufacturer#4 Brand#41 STANDARD ANODIZED TIN 12 WRAP CASE 1290.35 reful +40982 almond antique misty red olive Manufacturer#3 Brand#32 ECONOMY PLATED COPPER 1 LG PKG 1922.98 c foxes can s +42669 almond antique medium spring khaki Manufacturer#5 Brand#51 STANDARD BURNISHED TIN 6 MED CAN 1611.66 sits haggl +45261 almond aquamarine floral ivory bisque Manufacturer#4 Brand#42 SMALL PLATED STEEL 27 WRAP CASE 1206.26 careful +48427 almond antique violet mint lemon Manufacturer#4 Brand#42 PROMO POLISHED STEEL 39 SM CASE 1375.42 hely ironic i +49671 almond antique gainsboro frosted violet Manufacturer#4 Brand#41 SMALL BRUSHED BRASS 10 SM BOX 1620.67 ccounts run quick +65667 almond aquamarine pink moccasin thistle Manufacturer#1 Brand#12 LARGE BURNISHED STEEL 42 JUMBO CASE 1632.66 e across the expr +78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith +85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull +86428 almond aquamarine burnished black steel Manufacturer#1 Brand#12 STANDARD ANODIZED STEEL 28 WRAP BAG 1414.42 arefully +90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl +105685 almond antique violet chocolate turquoise Manufacturer#2 Brand#22 MEDIUM ANODIZED COPPER 14 MED CAN 1690.68 ly pending requ +110592 almond antique salmon chartreuse burlywood Manufacturer#1 Brand#15 PROMO BURNISHED NICKEL 6 JUMBO PKG 1602.59 to the furiously +112398 almond antique metallic orange dim Manufacturer#3 Brand#32 MEDIUM BURNISHED BRASS 19 JUMBO JAR 1410.39 ole car +121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h +121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h +121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h +121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h +132666 almond aquamarine rose maroon antique Manufacturer#2 Brand#24 SMALL POLISHED NICKEL 25 MED BOX 1698.66 even +144293 almond antique olive coral navajo Manufacturer#3 Brand#34 STANDARD POLISHED STEEL 45 JUMBO CAN 1337.29 ag furiously about +146985 almond aquamarine midnight light salmon Manufacturer#2 Brand#23 MEDIUM BURNISHED COPPER 2 SM CASE 2031.98 s cajole caref +155733 almond antique sky peru orange Manufacturer#5 Brand#53 SMALL PLATED BRASS 2 WRAP DRUM 1788.73 furiously. bra +191709 almond antique violet turquoise frosted Manufacturer#2 Brand#22 ECONOMY POLISHED STEEL 40 MED BOX 1800.7 haggle +192697 almond antique blue firebrick mint Manufacturer#5 Brand#52 MEDIUM BURNISHED TIN 31 LG DRUM 1789.69 ickly ir +195606 almond aquamarine sandy cyan gainsboro Manufacturer#2 Brand#25 STANDARD PLATED TIN 18 SM PKG 1701.6 ic de +PREHOOK: query: -- 19. testJoinWithWindowing +select abc.p_mfgr, abc.p_name, +rank() as r, +denserank() as dr, +abc.p_retailprice, sum(abc.p_retailprice) as s1 over (rows between unbounded preceding and current row), +abc.p_size, abc.p_size - lag(abc.p_size,1) as deltaSz +from noop(part +distribute by p_mfgr +sort by p_name +) abc join part p1 on abc.p_partkey = p1.p_partkey +distribute by abc.p_mfgr +sort by abc.p_name +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 19. testJoinWithWindowing +select abc.p_mfgr, abc.p_name, +rank() as r, +denserank() as dr, +abc.p_retailprice, sum(abc.p_retailprice) as s1 over (rows between unbounded preceding and current row), +abc.p_size, abc.p_size - lag(abc.p_size,1) as deltaSz +from noop(part +distribute by p_mfgr +sort by p_name +) abc join part p1 on abc.p_partkey = p1.p_partkey +distribute by abc.p_mfgr +sort by abc.p_name +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 1 1 1173.15 1173.15 2 0 +Manufacturer#1 almond antique burnished rose metallic 1 1 1173.15 2346.3 2 0 +Manufacturer#1 almond antique burnished rose metallic 1 1 1173.15 3519.4500000000003 2 0 +Manufacturer#1 almond antique burnished rose metallic 1 1 1173.15 4692.6 2 0 +Manufacturer#1 almond antique chartreuse lavender yellow 5 2 1753.76 6446.360000000001 34 32 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 3 1602.59 8048.950000000001 6 -28 +Manufacturer#1 almond aquamarine burnished black steel 7 4 1414.42 9463.37 28 22 +Manufacturer#1 almond aquamarine pink moccasin thistle 8 5 1632.66 11096.03 42 14 +Manufacturer#2 almond antique violet chocolate turquoise 1 1 1690.68 1690.68 14 0 +Manufacturer#2 almond antique violet turquoise frosted 2 2 1800.7 3491.38 40 26 +Manufacturer#2 almond aquamarine midnight light salmon 3 3 2031.98 5523.360000000001 2 -38 +Manufacturer#2 almond aquamarine rose maroon antique 4 4 1698.66 7222.02 25 23 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 5 5 1701.6 8923.62 18 -7 +Manufacturer#3 almond antique chartreuse khaki white 1 1 1671.68 1671.68 17 0 +Manufacturer#3 almond antique forest lavender goldenrod 2 2 1190.27 2861.95 14 -3 +Manufacturer#3 almond antique metallic orange dim 3 3 1410.39 4272.34 19 5 +Manufacturer#3 almond antique misty red olive 4 4 1922.98 6195.32 1 -18 +Manufacturer#3 almond antique olive coral navajo 5 5 1337.29 7532.61 45 44 +Manufacturer#4 almond antique gainsboro frosted violet 1 1 1620.67 1620.67 10 0 +Manufacturer#4 almond antique violet mint lemon 2 2 1375.42 2996.09 39 29 +Manufacturer#4 almond aquamarine floral ivory bisque 3 3 1206.26 4202.35 27 -12 +Manufacturer#4 almond aquamarine yellow dodger mint 4 4 1844.92 6047.27 7 -20 +Manufacturer#4 almond azure aquamarine papaya violet 5 5 1290.35 7337.620000000001 12 5 +Manufacturer#5 almond antique blue firebrick mint 1 1 1789.69 1789.69 31 0 +Manufacturer#5 almond antique medium spring khaki 2 2 1611.66 3401.3500000000004 6 -25 +Manufacturer#5 almond antique sky peru orange 3 3 1788.73 5190.08 2 -4 +Manufacturer#5 almond aquamarine dodger light gainsboro 4 4 1018.1 6208.18 46 44 +Manufacturer#5 almond azure blanched chiffon midnight 5 5 1464.48 7672.66 23 -23 +PREHOOK: query: -- 20. testMixedCaseAlias +select p_mfgr, p_name, p_size, rank() as R +from noop(part +distribute by p_mfgr +sort by p_name, p_size desc) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 20. testMixedCaseAlias +select p_mfgr, p_name, p_size, rank() as R +from noop(part +distribute by p_mfgr +sort by p_name, p_size desc) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 1 +Manufacturer#1 almond antique burnished rose metallic 2 1 +Manufacturer#1 almond antique chartreuse lavender yellow 34 3 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 +Manufacturer#1 almond aquamarine burnished black steel 28 5 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 +Manufacturer#2 almond antique violet turquoise frosted 40 2 +Manufacturer#2 almond aquamarine midnight light salmon 2 3 +Manufacturer#2 almond aquamarine rose maroon antique 25 4 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 +Manufacturer#3 almond antique chartreuse khaki white 17 1 +Manufacturer#3 almond antique forest lavender goldenrod 14 2 +Manufacturer#3 almond antique metallic orange dim 19 3 +Manufacturer#3 almond antique misty red olive 1 4 +Manufacturer#3 almond antique olive coral navajo 45 5 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 +Manufacturer#4 almond antique violet mint lemon 39 2 +Manufacturer#4 almond aquamarine floral ivory bisque 27 3 +Manufacturer#4 almond aquamarine yellow dodger mint 7 4 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 +Manufacturer#5 almond antique blue firebrick mint 31 1 +Manufacturer#5 almond antique medium spring khaki 6 2 +Manufacturer#5 almond antique sky peru orange 2 3 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 +Manufacturer#5 almond azure blanched chiffon midnight 23 5 +PREHOOK: query: -- 21. testNoopWithMap +select p_mfgr, p_name, p_size, rank() as r +from noopwithmap(part +distribute by p_mfgr +sort by p_name, p_size desc) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 21. testNoopWithMap +select p_mfgr, p_name, p_size, rank() as r +from noopwithmap(part +distribute by p_mfgr +sort by p_name, p_size desc) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 1 +Manufacturer#1 almond antique burnished rose metallic 2 1 +Manufacturer#1 almond antique chartreuse lavender yellow 34 3 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 +Manufacturer#1 almond aquamarine burnished black steel 28 5 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 +Manufacturer#2 almond antique violet turquoise frosted 40 2 +Manufacturer#2 almond aquamarine midnight light salmon 2 3 +Manufacturer#2 almond aquamarine rose maroon antique 25 4 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 +Manufacturer#3 almond antique chartreuse khaki white 17 1 +Manufacturer#3 almond antique forest lavender goldenrod 14 2 +Manufacturer#3 almond antique metallic orange dim 19 3 +Manufacturer#3 almond antique misty red olive 1 4 +Manufacturer#3 almond antique olive coral navajo 45 5 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 +Manufacturer#4 almond antique violet mint lemon 39 2 +Manufacturer#4 almond aquamarine floral ivory bisque 27 3 +Manufacturer#4 almond aquamarine yellow dodger mint 7 4 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 +Manufacturer#5 almond antique blue firebrick mint 31 1 +Manufacturer#5 almond antique medium spring khaki 6 2 +Manufacturer#5 almond antique sky peru orange 2 3 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 +Manufacturer#5 almond azure blanched chiffon midnight 23 5 +PREHOOK: query: -- 22. testNoopWithMapWithWindowing +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from noopwithmap(part + distribute by p_mfgr + sort by p_name) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 22. testNoopWithMapWithWindowing +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from noopwithmap(part + distribute by p_mfgr + sort by p_name) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15 +Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3 +Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 5702.650000000001 +Manufacturer#1 almond aquamarine burnished black steel 28 5 4 7117.070000000001 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 8749.730000000001 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68 +Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38 +Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001 +Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 7222.02 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 8923.62 +Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68 +Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95 +Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34 +Manufacturer#3 almond antique misty red olive 1 4 4 6195.32 +Manufacturer#3 almond antique olive coral navajo 45 5 5 7532.61 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67 +Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09 +Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35 +Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 6047.27 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 7337.620000000001 +Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69 +Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004 +Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6208.18 +Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 7672.66 +PREHOOK: query: -- 23. testHavingWithWindowingNoGBY +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from part +having p_size > 5 +distribute by p_mfgr +sort by p_name +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 23. testHavingWithWindowingNoGBY +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from part +having p_size > 5 +distribute by p_mfgr +sort by p_name +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 5702.650000000001 +Manufacturer#1 almond aquamarine burnished black steel 28 5 4 7117.070000000001 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 8749.730000000001 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68 +Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38 +Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 7222.02 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 8923.62 +Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68 +Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95 +Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34 +Manufacturer#3 almond antique olive coral navajo 45 5 5 7532.61 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67 +Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09 +Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35 +Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 6047.27 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 7337.620000000001 +Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69 +Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6208.18 +Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 7672.66 +PREHOOK: query: -- 24. testHavingWithWindowingCondRankNoGBY +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from part +having r < 4 +distribute by p_mfgr +sort by p_name +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 24. testHavingWithWindowingCondRankNoGBY +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from part +having r < 4 +distribute by p_mfgr +sort by p_name +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15 +Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3 +Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68 +Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38 +Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001 +Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68 +Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95 +Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67 +Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09 +Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35 +Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69 +Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004 +Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08 +PREHOOK: query: -- 25. testHavingWithWindowingPTFNoGBY +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from noop(part +distribute by p_mfgr +sort by p_name) +having r < 4 +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 25. testHavingWithWindowingPTFNoGBY +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from noop(part +distribute by p_mfgr +sort by p_name) +having r < 4 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15 +Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3 +Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68 +Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38 +Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001 +Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68 +Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95 +Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67 +Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09 +Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35 +Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69 +Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004 +Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08 +PREHOOK: query: -- 26. testFirstLast +select p_mfgr,p_name, p_size, +sum(p_size) as s2 over (rows between current row and current row), +first_value(p_size) as f over (w1) , +last_value(p_size, false) as l over (w1) +from part +distribute by p_mfgr +sort by p_mfgr +window w1 as rows between 2 preceding and 2 following +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 26. testFirstLast +select p_mfgr,p_name, p_size, +sum(p_size) as s2 over (rows between current row and current row), +first_value(p_size) as f over (w1) , +last_value(p_size, false) as l over (w1) +from part +distribute by p_mfgr +sort by p_mfgr +window w1 as rows between 2 preceding and 2 following +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 2 2 2 +Manufacturer#1 almond antique chartreuse lavender yellow 34 34 2 6 +Manufacturer#1 almond antique burnished rose metallic 2 2 2 28 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 6 34 42 +Manufacturer#1 almond aquamarine burnished black steel 28 28 2 42 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 42 6 42 +Manufacturer#2 almond antique violet chocolate turquoise 14 14 14 2 +Manufacturer#2 almond antique violet turquoise frosted 40 40 14 25 +Manufacturer#2 almond aquamarine midnight light salmon 2 2 14 18 +Manufacturer#2 almond aquamarine rose maroon antique 25 25 40 18 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 18 2 18 +Manufacturer#3 almond antique metallic orange dim 19 19 19 14 +Manufacturer#3 almond antique chartreuse khaki white 17 17 19 1 +Manufacturer#3 almond antique forest lavender goldenrod 14 14 19 45 +Manufacturer#3 almond antique misty red olive 1 1 17 45 +Manufacturer#3 almond antique olive coral navajo 45 45 14 45 +Manufacturer#4 almond antique gainsboro frosted violet 10 10 10 27 +Manufacturer#4 almond antique violet mint lemon 39 39 10 7 +Manufacturer#4 almond aquamarine floral ivory bisque 27 27 10 12 +Manufacturer#4 almond aquamarine yellow dodger mint 7 7 39 12 +Manufacturer#4 almond azure aquamarine papaya violet 12 12 27 12 +Manufacturer#5 almond antique blue firebrick mint 31 31 31 2 +Manufacturer#5 almond antique medium spring khaki 6 6 31 46 +Manufacturer#5 almond antique sky peru orange 2 2 31 23 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 46 6 23 +Manufacturer#5 almond azure blanched chiffon midnight 23 23 2 23 +PREHOOK: query: -- 27. testFirstLastWithWhere +select p_mfgr,p_name, p_size, +rank() as r, +sum(p_size) as s2 over (rows between current row and current row), +first_value(p_size) as f over (w1), +last_value(p_size, false) as l over (w1) +from part +where p_mfgr = 'Manufacturer#3' +distribute by p_mfgr +sort by p_mfgr +window w1 as rows between 2 preceding and 2 following +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 27. testFirstLastWithWhere +select p_mfgr,p_name, p_size, +rank() as r, +sum(p_size) as s2 over (rows between current row and current row), +first_value(p_size) as f over (w1), +last_value(p_size, false) as l over (w1) +from part +where p_mfgr = 'Manufacturer#3' +distribute by p_mfgr +sort by p_mfgr +window w1 as rows between 2 preceding and 2 following +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#3 almond antique chartreuse khaki white 17 1 17 17 19 +Manufacturer#3 almond antique forest lavender goldenrod 14 1 14 17 1 +Manufacturer#3 almond antique metallic orange dim 19 1 19 17 45 +Manufacturer#3 almond antique misty red olive 1 1 1 14 45 +Manufacturer#3 almond antique olive coral navajo 45 1 45 19 45 +PREHOOK: query: -- 28. testSumDelta +select p_mfgr,p_name, p_size, +sum(p_size - lag(p_size,1)) as deltaSum over (w1) +from part +distribute by p_mfgr +sort by p_mfgr +window w1 as rows between 2 preceding and 2 following +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 28. testSumDelta +select p_mfgr,p_name, p_size, +sum(p_size - lag(p_size,1)) as deltaSum over (w1) +from part +distribute by p_mfgr +sort by p_mfgr +window w1 as rows between 2 preceding and 2 following +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 0 +Manufacturer#1 almond antique chartreuse lavender yellow 34 4 +Manufacturer#1 almond antique burnished rose metallic 2 26 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 8 +Manufacturer#1 almond aquamarine burnished black steel 28 40 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 36 +Manufacturer#2 almond antique violet chocolate turquoise 14 -12 +Manufacturer#2 almond antique violet turquoise frosted 40 11 +Manufacturer#2 almond aquamarine midnight light salmon 2 4 +Manufacturer#2 almond aquamarine rose maroon antique 25 -22 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 16 +Manufacturer#3 almond antique metallic orange dim 19 -5 +Manufacturer#3 almond antique chartreuse khaki white 17 -18 +Manufacturer#3 almond antique forest lavender goldenrod 14 26 +Manufacturer#3 almond antique misty red olive 1 28 +Manufacturer#3 almond antique olive coral navajo 45 31 +Manufacturer#4 almond antique gainsboro frosted violet 10 17 +Manufacturer#4 almond antique violet mint lemon 39 -3 +Manufacturer#4 almond aquamarine floral ivory bisque 27 2 +Manufacturer#4 almond aquamarine yellow dodger mint 7 -27 +Manufacturer#4 almond azure aquamarine papaya violet 12 -15 +Manufacturer#5 almond antique blue firebrick mint 31 -29 +Manufacturer#5 almond antique medium spring khaki 6 15 +Manufacturer#5 almond antique sky peru orange 2 -8 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 17 +Manufacturer#5 almond azure blanched chiffon midnight 23 21 +PREHOOK: query: -- 29. testSumWindow +select p_mfgr,p_name, p_size, +sum(p_size) as s1 over (w1), +sum(p_size) as s2 over (rows between current row and current row) +from part +distribute by p_mfgr +sort by p_mfgr +window w1 as rows between 2 preceding and 2 following +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 29. testSumWindow +select p_mfgr,p_name, p_size, +sum(p_size) as s1 over (w1), +sum(p_size) as s2 over (rows between current row and current row) +from part +distribute by p_mfgr +sort by p_mfgr +window w1 as rows between 2 preceding and 2 following +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 38 2 +Manufacturer#1 almond antique chartreuse lavender yellow 34 44 34 +Manufacturer#1 almond antique burnished rose metallic 2 72 2 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 112 6 +Manufacturer#1 almond aquamarine burnished black steel 28 78 28 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 76 42 +Manufacturer#2 almond antique violet chocolate turquoise 14 56 14 +Manufacturer#2 almond antique violet turquoise frosted 40 81 40 +Manufacturer#2 almond aquamarine midnight light salmon 2 99 2 +Manufacturer#2 almond aquamarine rose maroon antique 25 85 25 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 45 18 +Manufacturer#3 almond antique metallic orange dim 19 50 19 +Manufacturer#3 almond antique chartreuse khaki white 17 51 17 +Manufacturer#3 almond antique forest lavender goldenrod 14 96 14 +Manufacturer#3 almond antique misty red olive 1 77 1 +Manufacturer#3 almond antique olive coral navajo 45 60 45 +Manufacturer#4 almond antique gainsboro frosted violet 10 76 10 +Manufacturer#4 almond antique violet mint lemon 39 83 39 +Manufacturer#4 almond aquamarine floral ivory bisque 27 95 27 +Manufacturer#4 almond aquamarine yellow dodger mint 7 85 7 +Manufacturer#4 almond azure aquamarine papaya violet 12 46 12 +Manufacturer#5 almond antique blue firebrick mint 31 39 31 +Manufacturer#5 almond antique medium spring khaki 6 85 6 +Manufacturer#5 almond antique sky peru orange 2 108 2 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 77 46 +Manufacturer#5 almond azure blanched chiffon midnight 23 71 23 +PREHOOK: query: -- 30. testNoSortClause +select p_mfgr,p_name, p_size, +rank() as r, denserank() as dr +from part +distribute by p_mfgr +window w1 as rows between 2 preceding and 2 following +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 30. testNoSortClause +select p_mfgr,p_name, p_size, +rank() as r, denserank() as dr +from part +distribute by p_mfgr +window w1 as rows between 2 preceding and 2 following +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 1 1 +Manufacturer#1 almond antique chartreuse lavender yellow 34 1 1 +Manufacturer#1 almond antique burnished rose metallic 2 1 1 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 1 1 +Manufacturer#1 almond aquamarine burnished black steel 28 1 1 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 1 1 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 +Manufacturer#2 almond antique violet turquoise frosted 40 1 1 +Manufacturer#2 almond aquamarine midnight light salmon 2 1 1 +Manufacturer#2 almond aquamarine rose maroon antique 25 1 1 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 1 1 +Manufacturer#3 almond antique metallic orange dim 19 1 1 +Manufacturer#3 almond antique chartreuse khaki white 17 1 1 +Manufacturer#3 almond antique forest lavender goldenrod 14 1 1 +Manufacturer#3 almond antique misty red olive 1 1 1 +Manufacturer#3 almond antique olive coral navajo 45 1 1 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 +Manufacturer#4 almond antique violet mint lemon 39 1 1 +Manufacturer#4 almond aquamarine floral ivory bisque 27 1 1 +Manufacturer#4 almond aquamarine yellow dodger mint 7 1 1 +Manufacturer#4 almond azure aquamarine papaya violet 12 1 1 +Manufacturer#5 almond antique blue firebrick mint 31 1 1 +Manufacturer#5 almond antique medium spring khaki 6 1 1 +Manufacturer#5 almond antique sky peru orange 2 1 1 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 1 1 +Manufacturer#5 almond azure blanched chiffon midnight 23 1 1 +PREHOOK: query: -- 31. testExpressions +select p_mfgr,p_name, p_size, +rank() as r, +denserank() as dr, +cumedist() as cud, +percentrank() as pr, +ntile(3) as nt, +count(p_size) as ca, +avg(p_size) as avg, +stddev(p_size) as st, +first_value(p_size % 5) as fv, +last_value(p_size) as lv, +first_value(p_size, true) as fvW1 over (w1) +from part +having p_size > 5 +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 31. testExpressions +select p_mfgr,p_name, p_size, +rank() as r, +denserank() as dr, +cumedist() as cud, +percentrank() as pr, +ntile(3) as nt, +count(p_size) as ca, +avg(p_size) as avg, +stddev(p_size) as st, +first_value(p_size % 5) as fv, +last_value(p_size) as lv, +first_value(p_size, true) as fvW1 over (w1) +from part +having p_size > 5 +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 0.5 0.4 2 6 19.0 16.237815945091466 2 42 2 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 0.6666666666666666 0.6 2 6 19.0 16.237815945091466 2 42 2 +Manufacturer#1 almond aquamarine burnished black steel 28 5 4 0.8333333333333334 0.8 3 6 19.0 16.237815945091466 2 42 34 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 1.0 1.0 3 6 19.0 16.237815945091466 2 42 6 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 0.2 0.0 1 5 19.8 12.560254774486067 4 18 14 +Manufacturer#2 almond antique violet turquoise frosted 40 2 2 0.4 0.25 1 5 19.8 12.560254774486067 4 18 14 +Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 0.8 0.75 2 5 19.8 12.560254774486067 4 18 40 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 1.0 1.0 3 5 19.8 12.560254774486067 4 18 2 +Manufacturer#3 almond antique chartreuse khaki white 17 1 1 0.2 0.0 1 5 19.2 14.344336861632886 2 45 17 +Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 0.4 0.25 1 5 19.2 14.344336861632886 2 45 17 +Manufacturer#3 almond antique metallic orange dim 19 3 3 0.6 0.5 2 5 19.2 14.344336861632886 2 45 17 +Manufacturer#3 almond antique olive coral navajo 45 5 5 1.0 1.0 3 5 19.2 14.344336861632886 2 45 19 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 0.2 0.0 1 5 19.0 12.149074038789951 0 12 10 +Manufacturer#4 almond antique violet mint lemon 39 2 2 0.4 0.25 1 5 19.0 12.149074038789951 0 12 10 +Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 0.6 0.5 2 5 19.0 12.149074038789951 0 12 10 +Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 0.8 0.75 2 5 19.0 12.149074038789951 0 12 39 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 1.0 1.0 3 5 19.0 12.149074038789951 0 12 27 +Manufacturer#5 almond antique blue firebrick mint 31 1 1 0.2 0.0 1 5 21.6 16.206171663906314 1 23 31 +Manufacturer#5 almond antique medium spring khaki 6 2 2 0.4 0.25 1 5 21.6 16.206171663906314 1 23 31 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 0.8 0.75 2 5 21.6 16.206171663906314 1 23 6 +Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 1.0 1.0 3 5 21.6 16.206171663906314 1 23 2 +PREHOOK: query: -- 32. testMultipleWindows +select p_mfgr,p_name, p_size, + rank() as r, denserank() as dr, +cumedist() as cud, +sum(p_size) as s1 over (rows between unbounded preceding and current row), +sum(p_size) as s2 over (range between p_size 5 less and current row), +first_value(p_size, true) as fv1 over (w1) +from part +having p_size > 5 +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 32. testMultipleWindows +select p_mfgr,p_name, p_size, + rank() as r, denserank() as dr, +cumedist() as cud, +sum(p_size) as s1 over (rows between unbounded preceding and current row), +sum(p_size) as s2 over (range between p_size 5 less and current row), +first_value(p_size, true) as fv1 over (w1) +from part +having p_size > 5 +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 0.5 38 34 2 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 0.6666666666666666 44 44 2 +Manufacturer#1 almond aquamarine burnished black steel 28 5 4 0.8333333333333334 72 28 34 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 1.0 114 42 6 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 0.2 14 14 14 +Manufacturer#2 almond antique violet turquoise frosted 40 2 2 0.4 54 40 14 +Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 0.8 81 25 40 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 1.0 99 43 2 +Manufacturer#3 almond antique chartreuse khaki white 17 1 1 0.2 17 17 17 +Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 0.4 31 31 17 +Manufacturer#3 almond antique metallic orange dim 19 3 3 0.6 50 19 17 +Manufacturer#3 almond antique olive coral navajo 45 5 5 1.0 96 45 19 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 0.2 10 10 10 +Manufacturer#4 almond antique violet mint lemon 39 2 2 0.4 49 39 10 +Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 0.6 76 66 10 +Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 0.8 83 83 39 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 1.0 95 12 27 +Manufacturer#5 almond antique blue firebrick mint 31 1 1 0.2 31 31 31 +Manufacturer#5 almond antique medium spring khaki 6 2 2 0.4 37 37 31 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 0.8 85 46 6 +Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 1.0 108 69 2 +PREHOOK: query: -- 33. testFunctionChain +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from noop(noopwithmap(noop(part +distribute by p_mfgr +sort by p_mfgr, p_name +))) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 33. testFunctionChain +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from noop(noopwithmap(noop(part +distribute by p_mfgr +sort by p_mfgr, p_name +))) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15 +Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3 +Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 5702.650000000001 +Manufacturer#1 almond aquamarine burnished black steel 28 5 4 7117.070000000001 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 8749.730000000001 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68 +Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38 +Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001 +Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 7222.02 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 8923.62 +Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68 +Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95 +Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34 +Manufacturer#3 almond antique misty red olive 1 4 4 6195.32 +Manufacturer#3 almond antique olive coral navajo 45 5 5 7532.61 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67 +Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09 +Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35 +Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 6047.27 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 7337.620000000001 +Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69 +Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004 +Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6208.18 +Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 7672.66 +PREHOOK: query: -- 34. testPTFAndWindowingInSubQ +select p_mfgr, p_name, +sub1.cd, sub1.s1 +from (select p_mfgr, p_name, +count(p_size) as cd, +p_retailprice, +sum(p_retailprice) as s1 over (w1) +from noop(part +distribute by p_mfgr +sort by p_name) +window w1 as rows between 2 preceding and 2 following +) sub1 +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 34. testPTFAndWindowingInSubQ +select p_mfgr, p_name, +sub1.cd, sub1.s1 +from (select p_mfgr, p_name, +count(p_size) as cd, +p_retailprice, +sum(p_retailprice) as s1 over (w1) +from noop(part +distribute by p_mfgr +sort by p_name) +window w1 as rows between 2 preceding and 2 following +) sub1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 6 4100.06 +Manufacturer#1 almond antique burnished rose metallic 6 5702.650000000001 +Manufacturer#1 almond antique chartreuse lavender yellow 6 7117.070000000001 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 7576.58 +Manufacturer#1 almond aquamarine burnished black steel 6 6403.43 +Manufacturer#1 almond aquamarine pink moccasin thistle 6 4649.67 +Manufacturer#2 almond antique violet chocolate turquoise 5 5523.360000000001 +Manufacturer#2 almond antique violet turquoise frosted 5 7222.02 +Manufacturer#2 almond aquamarine midnight light salmon 5 8923.62 +Manufacturer#2 almond aquamarine rose maroon antique 5 7232.9400000000005 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 5 5432.24 +Manufacturer#3 almond antique chartreuse khaki white 5 4272.34 +Manufacturer#3 almond antique forest lavender goldenrod 5 6195.32 +Manufacturer#3 almond antique metallic orange dim 5 7532.61 +Manufacturer#3 almond antique misty red olive 5 5860.929999999999 +Manufacturer#3 almond antique olive coral navajo 5 4670.66 +Manufacturer#4 almond antique gainsboro frosted violet 5 4202.35 +Manufacturer#4 almond antique violet mint lemon 5 6047.27 +Manufacturer#4 almond aquamarine floral ivory bisque 5 7337.620000000001 +Manufacturer#4 almond aquamarine yellow dodger mint 5 5716.950000000001 +Manufacturer#4 almond azure aquamarine papaya violet 5 4341.530000000001 +Manufacturer#5 almond antique blue firebrick mint 5 5190.08 +Manufacturer#5 almond antique medium spring khaki 5 6208.18 +Manufacturer#5 almond antique sky peru orange 5 7672.66 +Manufacturer#5 almond aquamarine dodger light gainsboro 5 5882.970000000001 +Manufacturer#5 almond azure blanched chiffon midnight 5 4271.3099999999995 +PREHOOK: query: -- 35. testCountStar +select p_mfgr,p_name, p_size, +count(*) as c, +count(p_size) as ca, +first_value(p_size, true) as fvW1 over (w1) +from part +having p_size > 5 +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 35. testCountStar +select p_mfgr,p_name, p_size, +count(*) as c, +count(p_size) as ca, +first_value(p_size, true) as fvW1 over (w1) +from part +having p_size > 5 +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique chartreuse lavender yellow 34 6 6 2 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 6 6 2 +Manufacturer#1 almond aquamarine burnished black steel 28 6 6 34 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 6 6 +Manufacturer#2 almond antique violet chocolate turquoise 14 5 5 14 +Manufacturer#2 almond antique violet turquoise frosted 40 5 5 14 +Manufacturer#2 almond aquamarine rose maroon antique 25 5 5 40 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 2 +Manufacturer#3 almond antique chartreuse khaki white 17 5 5 17 +Manufacturer#3 almond antique forest lavender goldenrod 14 5 5 17 +Manufacturer#3 almond antique metallic orange dim 19 5 5 17 +Manufacturer#3 almond antique olive coral navajo 45 5 5 19 +Manufacturer#4 almond antique gainsboro frosted violet 10 5 5 10 +Manufacturer#4 almond antique violet mint lemon 39 5 5 10 +Manufacturer#4 almond aquamarine floral ivory bisque 27 5 5 10 +Manufacturer#4 almond aquamarine yellow dodger mint 7 5 5 39 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 27 +Manufacturer#5 almond antique blue firebrick mint 31 5 5 31 +Manufacturer#5 almond antique medium spring khaki 6 5 5 31 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 5 5 6 +Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 2 +PREHOOK: query: -- 36. testJoinWithWindowingWithCount +select abc.p_mfgr, abc.p_name, +rank() as r, +denserank() as dr, +count(abc.p_name) as cd, +abc.p_retailprice, sum(abc.p_retailprice) as s1 over (rows between unbounded preceding and current row), +abc.p_size, abc.p_size - lag(abc.p_size,1) as deltaSz +from noop(part +distribute by p_mfgr +sort by p_name +) abc join part p1 on abc.p_partkey = p1.p_partkey +distribute by abc.p_mfgr +sort by abc.p_name +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 36. testJoinWithWindowingWithCount +select abc.p_mfgr, abc.p_name, +rank() as r, +denserank() as dr, +count(abc.p_name) as cd, +abc.p_retailprice, sum(abc.p_retailprice) as s1 over (rows between unbounded preceding and current row), +abc.p_size, abc.p_size - lag(abc.p_size,1) as deltaSz +from noop(part +distribute by p_mfgr +sort by p_name +) abc join part p1 on abc.p_partkey = p1.p_partkey +distribute by abc.p_mfgr +sort by abc.p_name +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 1 1 8 1173.15 1173.15 2 0 +Manufacturer#1 almond antique burnished rose metallic 1 1 8 1173.15 2346.3 2 0 +Manufacturer#1 almond antique burnished rose metallic 1 1 8 1173.15 3519.4500000000003 2 0 +Manufacturer#1 almond antique burnished rose metallic 1 1 8 1173.15 4692.6 2 0 +Manufacturer#1 almond antique chartreuse lavender yellow 5 2 8 1753.76 6446.360000000001 34 32 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 3 8 1602.59 8048.950000000001 6 -28 +Manufacturer#1 almond aquamarine burnished black steel 7 4 8 1414.42 9463.37 28 22 +Manufacturer#1 almond aquamarine pink moccasin thistle 8 5 8 1632.66 11096.03 42 14 +Manufacturer#2 almond antique violet chocolate turquoise 1 1 5 1690.68 1690.68 14 0 +Manufacturer#2 almond antique violet turquoise frosted 2 2 5 1800.7 3491.38 40 26 +Manufacturer#2 almond aquamarine midnight light salmon 3 3 5 2031.98 5523.360000000001 2 -38 +Manufacturer#2 almond aquamarine rose maroon antique 4 4 5 1698.66 7222.02 25 23 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 5 5 5 1701.6 8923.62 18 -7 +Manufacturer#3 almond antique chartreuse khaki white 1 1 5 1671.68 1671.68 17 0 +Manufacturer#3 almond antique forest lavender goldenrod 2 2 5 1190.27 2861.95 14 -3 +Manufacturer#3 almond antique metallic orange dim 3 3 5 1410.39 4272.34 19 5 +Manufacturer#3 almond antique misty red olive 4 4 5 1922.98 6195.32 1 -18 +Manufacturer#3 almond antique olive coral navajo 5 5 5 1337.29 7532.61 45 44 +Manufacturer#4 almond antique gainsboro frosted violet 1 1 5 1620.67 1620.67 10 0 +Manufacturer#4 almond antique violet mint lemon 2 2 5 1375.42 2996.09 39 29 +Manufacturer#4 almond aquamarine floral ivory bisque 3 3 5 1206.26 4202.35 27 -12 +Manufacturer#4 almond aquamarine yellow dodger mint 4 4 5 1844.92 6047.27 7 -20 +Manufacturer#4 almond azure aquamarine papaya violet 5 5 5 1290.35 7337.620000000001 12 5 +Manufacturer#5 almond antique blue firebrick mint 1 1 5 1789.69 1789.69 31 0 +Manufacturer#5 almond antique medium spring khaki 2 2 5 1611.66 3401.3500000000004 6 -25 +Manufacturer#5 almond antique sky peru orange 3 3 5 1788.73 5190.08 2 -4 +Manufacturer#5 almond aquamarine dodger light gainsboro 4 4 5 1018.1 6208.18 46 44 +Manufacturer#5 almond azure blanched chiffon midnight 5 5 5 1464.48 7672.66 23 -23 +PREHOOK: query: -- 37. testSumDelta +select p_mfgr,p_name, p_size, +sum(p_size - lag(p_size,1)) as deltaSum +from part +distribute by p_mfgr +sort by p_mfgr +window w1 as rows between 2 preceding and 2 following +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 37. testSumDelta +select p_mfgr,p_name, p_size, +sum(p_size - lag(p_size,1)) as deltaSum +from part +distribute by p_mfgr +sort by p_mfgr +window w1 as rows between 2 preceding and 2 following +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 40 +Manufacturer#1 almond antique chartreuse lavender yellow 34 40 +Manufacturer#1 almond antique burnished rose metallic 2 40 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 40 +Manufacturer#1 almond aquamarine burnished black steel 28 40 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 40 +Manufacturer#2 almond antique violet chocolate turquoise 14 4 +Manufacturer#2 almond antique violet turquoise frosted 40 4 +Manufacturer#2 almond aquamarine midnight light salmon 2 4 +Manufacturer#2 almond aquamarine rose maroon antique 25 4 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 4 +Manufacturer#3 almond antique metallic orange dim 19 26 +Manufacturer#3 almond antique chartreuse khaki white 17 26 +Manufacturer#3 almond antique forest lavender goldenrod 14 26 +Manufacturer#3 almond antique misty red olive 1 26 +Manufacturer#3 almond antique olive coral navajo 45 26 +Manufacturer#4 almond antique gainsboro frosted violet 10 2 +Manufacturer#4 almond antique violet mint lemon 39 2 +Manufacturer#4 almond aquamarine floral ivory bisque 27 2 +Manufacturer#4 almond aquamarine yellow dodger mint 7 2 +Manufacturer#4 almond azure aquamarine papaya violet 12 2 +Manufacturer#5 almond antique blue firebrick mint 31 -8 +Manufacturer#5 almond antique medium spring khaki 6 -8 +Manufacturer#5 almond antique sky peru orange 2 -8 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 -8 +Manufacturer#5 almond azure blanched chiffon midnight 23 -8 +PREHOOK: query: -- 38. testUDAFs +select p_mfgr,p_name, p_size, +sum(p_retailprice) as s over (w1), +min(p_retailprice) as mi over (w1), +max(p_retailprice) as ma over (w1), +avg(p_retailprice) as ag over (w1) +from part +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 38. testUDAFs +select p_mfgr,p_name, p_size, +sum(p_retailprice) as s over (w1), +min(p_retailprice) as mi over (w1), +max(p_retailprice) as ma over (w1), +avg(p_retailprice) as ag over (w1) +from part +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 4100.06 1173.15 1753.76 1366.6866666666667 +Manufacturer#1 almond antique burnished rose metallic 2 5702.650000000001 1173.15 1753.76 1425.6625000000001 +Manufacturer#1 almond antique chartreuse lavender yellow 34 7117.070000000001 1173.15 1753.76 1423.4140000000002 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 7576.58 1173.15 1753.76 1515.316 +Manufacturer#1 almond aquamarine burnished black steel 28 6403.43 1414.42 1753.76 1600.8575 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 4649.67 1414.42 1632.66 1549.89 +Manufacturer#2 almond antique violet chocolate turquoise 14 5523.360000000001 1690.68 2031.98 1841.1200000000001 +Manufacturer#2 almond antique violet turquoise frosted 40 7222.02 1690.68 2031.98 1805.505 +Manufacturer#2 almond aquamarine midnight light salmon 2 8923.62 1690.68 2031.98 1784.7240000000002 +Manufacturer#2 almond aquamarine rose maroon antique 25 7232.9400000000005 1698.66 2031.98 1808.2350000000001 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5432.24 1698.66 2031.98 1810.7466666666667 +Manufacturer#3 almond antique chartreuse khaki white 17 4272.34 1190.27 1671.68 1424.1133333333335 +Manufacturer#3 almond antique forest lavender goldenrod 14 6195.32 1190.27 1922.98 1548.83 +Manufacturer#3 almond antique metallic orange dim 19 7532.61 1190.27 1922.98 1506.522 +Manufacturer#3 almond antique misty red olive 1 5860.929999999999 1190.27 1922.98 1465.2324999999998 +Manufacturer#3 almond antique olive coral navajo 45 4670.66 1337.29 1922.98 1556.8866666666665 +Manufacturer#4 almond antique gainsboro frosted violet 10 4202.35 1206.26 1620.67 1400.7833333333335 +Manufacturer#4 almond antique violet mint lemon 39 6047.27 1206.26 1844.92 1511.8175 +Manufacturer#4 almond aquamarine floral ivory bisque 27 7337.620000000001 1206.26 1844.92 1467.5240000000001 +Manufacturer#4 almond aquamarine yellow dodger mint 7 5716.950000000001 1206.26 1844.92 1429.2375000000002 +Manufacturer#4 almond azure aquamarine papaya violet 12 4341.530000000001 1206.26 1844.92 1447.176666666667 +Manufacturer#5 almond antique blue firebrick mint 31 5190.08 1611.66 1789.69 1730.0266666666666 +Manufacturer#5 almond antique medium spring khaki 6 6208.18 1018.1 1789.69 1552.045 +Manufacturer#5 almond antique sky peru orange 2 7672.66 1018.1 1789.69 1534.532 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 5882.970000000001 1018.1 1788.73 1470.7425000000003 +Manufacturer#5 almond azure blanched chiffon midnight 23 4271.3099999999995 1018.1 1788.73 1423.7699999999998 +PREHOOK: query: -- 39. testUDAFsWithPTFWithGBY +select p_mfgr,p_name, p_size, p_retailprice, +sum(p_retailprice) as s over (w1), +min(p_retailprice) as mi , +max(p_retailprice) as ma , +avg(p_retailprice) as ag over (w1) +from part +group by p_mfgr,p_name, p_size, p_retailprice +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 39. testUDAFsWithPTFWithGBY +select p_mfgr,p_name, p_size, p_retailprice, +sum(p_retailprice) as s over (w1), +min(p_retailprice) as mi , +max(p_retailprice) as ma , +avg(p_retailprice) as ag over (w1) +from part +group by p_mfgr,p_name, p_size, p_retailprice +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 1173.15 4529.5 1173.15 1173.15 1509.8333333333333 +Manufacturer#1 almond antique chartreuse lavender yellow 34 1753.76 5943.92 1753.76 1753.76 1485.98 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 1602.59 7576.58 1602.59 1602.59 1515.316 +Manufacturer#1 almond aquamarine burnished black steel 28 1414.42 6403.43 1414.42 1414.42 1600.8575 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 1632.66 4649.67 1632.66 1632.66 1549.89 +Manufacturer#2 almond antique violet chocolate turquoise 14 1690.68 5523.360000000001 1690.68 1690.68 1841.1200000000001 +Manufacturer#2 almond antique violet turquoise frosted 40 1800.7 7222.02 1800.7 1800.7 1805.505 +Manufacturer#2 almond aquamarine midnight light salmon 2 2031.98 8923.62 2031.98 2031.98 1784.7240000000002 +Manufacturer#2 almond aquamarine rose maroon antique 25 1698.66 7232.9400000000005 1698.66 1698.66 1808.2350000000001 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 1701.6 5432.24 1701.6 1701.6 1810.7466666666667 +Manufacturer#3 almond antique chartreuse khaki white 17 1671.68 4272.34 1671.68 1671.68 1424.1133333333335 +Manufacturer#3 almond antique forest lavender goldenrod 14 1190.27 6195.32 1190.27 1190.27 1548.83 +Manufacturer#3 almond antique metallic orange dim 19 1410.39 7532.61 1410.39 1410.39 1506.522 +Manufacturer#3 almond antique misty red olive 1 1922.98 5860.929999999999 1922.98 1922.98 1465.2324999999998 +Manufacturer#3 almond antique olive coral navajo 45 1337.29 4670.66 1337.29 1337.29 1556.8866666666665 +Manufacturer#4 almond antique gainsboro frosted violet 10 1620.67 4202.35 1620.67 1620.67 1400.7833333333335 +Manufacturer#4 almond antique violet mint lemon 39 1375.42 6047.27 1375.42 1375.42 1511.8175 +Manufacturer#4 almond aquamarine floral ivory bisque 27 1206.26 7337.620000000001 1206.26 1206.26 1467.5240000000001 +Manufacturer#4 almond aquamarine yellow dodger mint 7 1844.92 5716.950000000001 1844.92 1844.92 1429.2375000000002 +Manufacturer#4 almond azure aquamarine papaya violet 12 1290.35 4341.530000000001 1290.35 1290.35 1447.176666666667 +Manufacturer#5 almond antique blue firebrick mint 31 1789.69 5190.08 1789.69 1789.69 1730.0266666666666 +Manufacturer#5 almond antique medium spring khaki 6 1611.66 6208.18 1611.66 1611.66 1552.045 +Manufacturer#5 almond antique sky peru orange 2 1788.73 7672.66 1788.73 1788.73 1534.532 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 1018.1 5882.970000000001 1018.1 1018.1 1470.7425000000003 +Manufacturer#5 almond azure blanched chiffon midnight 23 1464.48 4271.3099999999995 1464.48 1464.48 1423.7699999999998 +PREHOOK: query: -- 40. testSTATs +select p_mfgr,p_name, p_size, +stddev(p_retailprice) as sdev over (w1), +stddev_pop(p_retailprice) as sdev_pop over (w1), +collect_set(p_size) as uniq_size over (w1), +variance(p_retailprice) as var over (w1), +corr(p_size, p_retailprice) as cor over (w1), +covar_pop(p_size, p_retailprice) as covarp over (w1) +from part +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 40. testSTATs +select p_mfgr,p_name, p_size, +stddev(p_retailprice) as sdev over (w1), +stddev_pop(p_retailprice) as sdev_pop over (w1), +collect_set(p_size) as uniq_size over (w1), +variance(p_retailprice) as var over (w1), +corr(p_size, p_retailprice) as cor over (w1), +covar_pop(p_size, p_retailprice) as covarp over (w1) +from part +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 273.70217881648074 273.70217881648074 [34,2] 74912.8826888888 1.0 4128.782222222221 +Manufacturer#1 almond antique burnished rose metallic 2 258.10677784349235 258.10677784349235 [34,2,6] 66619.10876874991 0.811328754177887 2801.7074999999995 +Manufacturer#1 almond antique chartreuse lavender yellow 34 230.90151585470358 230.90151585470358 [34,2,6,28] 53315.51002399992 0.695639377397664 2210.7864 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 202.73109328368946 202.73109328368946 [34,2,6,42,28] 41099.896184 0.630785977101214 2009.9536000000007 +Manufacturer#1 almond aquamarine burnished black steel 28 121.6064517973862 121.6064517973862 [34,6,42,28] 14788.129118750014 0.2036684720435979 331.1337500000004 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 96.5751586416853 96.5751586416853 [6,42,28] 9326.761266666683 -1.4442181184933883E-4 -0.20666666666708502 +Manufacturer#2 almond antique violet chocolate turquoise 14 142.2363169751898 142.2363169751898 [2,40,14] 20231.169866666663 -0.49369526554523185 -1113.7466666666658 +Manufacturer#2 almond antique violet turquoise frosted 40 137.76306498840682 137.76306498840682 [2,25,40,14] 18978.662075 -0.5205630897335946 -1004.4812499999995 +Manufacturer#2 almond aquamarine midnight light salmon 2 130.03972279269132 130.03972279269132 [2,18,25,40,14] 16910.329504000005 -0.46908967495720255 -766.1791999999995 +Manufacturer#2 almond aquamarine rose maroon antique 25 135.55100986344584 135.55100986344584 [2,18,25,40] 18374.07627499999 -0.6091405874714462 -1128.1787499999987 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 156.44019460768044 156.44019460768044 [2,18,25] 24473.534488888927 -0.9571686373491608 -1441.4466666666676 +Manufacturer#3 almond antique chartreuse khaki white 17 196.7742266885805 196.7742266885805 [17,19,14] 38720.09628888887 0.5557168646224995 224.6944444444446 +Manufacturer#3 almond antique forest lavender goldenrod 14 275.14144189852607 275.14144189852607 [17,1,19,14] 75702.81305 -0.6720833036576083 -1296.9000000000003 +Manufacturer#3 almond antique metallic orange dim 19 260.23473614412046 260.23473614412046 [17,1,19,14,45] 67722.117896 -0.5703526513979519 -2129.0664 +Manufacturer#3 almond antique misty red olive 1 275.9139962356932 275.9139962356932 [1,19,14,45] 76128.53331875012 -0.577476899644802 -2547.7868749999993 +Manufacturer#3 almond antique olive coral navajo 45 260.5815918713796 260.5815918713796 [1,19,45] 67902.76602222225 -0.8710736366736884 -4099.731111111111 +Manufacturer#4 almond antique gainsboro frosted violet 10 170.13011889596618 170.13011889596618 [39,27,10] 28944.25735555559 -0.6656975320098423 -1347.4777777777779 +Manufacturer#4 almond antique violet mint lemon 39 242.26834609323197 242.26834609323197 [39,7,27,10] 58693.95151875002 -0.8051852719193339 -2537.328125 +Manufacturer#4 almond aquamarine floral ivory bisque 27 234.10001662537326 234.10001662537326 [39,7,27,10,12] 54802.817784000035 -0.6046935574240581 -1719.8079999999995 +Manufacturer#4 almond aquamarine yellow dodger mint 7 247.3342714197732 247.3342714197732 [39,7,27,12] 61174.24181875003 -0.5508665654707869 -1719.0368749999975 +Manufacturer#4 almond azure aquamarine papaya violet 12 283.3344330566893 283.3344330566893 [7,27,12] 80278.40095555557 -0.7755740084632333 -1867.4888888888881 +Manufacturer#5 almond antique blue firebrick mint 31 83.69879024746363 83.69879024746363 [2,6,31] 7005.487488888913 0.39004303087285047 418.9233333333353 +Manufacturer#5 almond antique medium spring khaki 6 316.68049612345885 316.68049612345885 [2,6,46,31] 100286.53662500004 -0.713612911776183 -4090.853749999999 +Manufacturer#5 almond antique sky peru orange 2 285.40506298242155 285.40506298242155 [2,23,6,46,31] 81456.04997600002 -0.712858514567818 -3297.2011999999986 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 285.43749038756283 285.43749038756283 [2,23,6,46] 81474.56091875004 -0.984128787153391 -4871.028125000002 +Manufacturer#5 almond azure blanched chiffon midnight 23 315.9225931564038 315.9225931564038 [2,23,46] 99807.08486666664 -0.9978877469246936 -5664.856666666666 +PREHOOK: query: -- 41. testDISTs +select p_mfgr,p_name, p_size, +histogram_numeric(p_retailprice, 5) as hist over (w1), +percentile(p_partkey, 0.5) as per over (w1), +rownumber() as rn +from part +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 41. testDISTs +select p_mfgr,p_name, p_size, +histogram_numeric(p_retailprice, 5) as hist over (w1), +percentile(p_partkey, 0.5) as per over (w1), +rownumber() as rn +from part +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 [{"x":1173.15,"y":2.0},{"x":1753.76,"y":1.0}] 121152.0 1 +Manufacturer#1 almond antique burnished rose metallic 2 [{"x":1173.15,"y":2.0},{"x":1602.59,"y":1.0},{"x":1753.76,"y":1.0}] 115872.0 2 +Manufacturer#1 almond antique chartreuse lavender yellow 34 [{"x":1173.15,"y":2.0},{"x":1414.42,"y":1.0},{"x":1602.59,"y":1.0},{"x":1753.76,"y":1.0}] 110592.0 3 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 [{"x":1173.15,"y":1.0},{"x":1414.42,"y":1.0},{"x":1602.59,"y":1.0},{"x":1632.66,"y":1.0},{"x":1753.76,"y":1.0}] 86428.0 4 +Manufacturer#1 almond aquamarine burnished black steel 28 [{"x":1414.42,"y":1.0},{"x":1602.59,"y":1.0},{"x":1632.66,"y":1.0},{"x":1753.76,"y":1.0}] 86098.0 5 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 [{"x":1414.42,"y":1.0},{"x":1602.59,"y":1.0},{"x":1632.66,"y":1.0}] 86428.0 6 +Manufacturer#2 almond antique violet chocolate turquoise 14 [{"x":1690.68,"y":1.0},{"x":1800.7,"y":1.0},{"x":2031.98,"y":1.0}] 146985.0 1 +Manufacturer#2 almond antique violet turquoise frosted 40 [{"x":1690.68,"y":1.0},{"x":1698.66,"y":1.0},{"x":1800.7,"y":1.0},{"x":2031.98,"y":1.0}] 139825.5 2 +Manufacturer#2 almond aquamarine midnight light salmon 2 [{"x":1690.68,"y":1.0},{"x":1698.66,"y":1.0},{"x":1701.6,"y":1.0},{"x":1800.7,"y":1.0},{"x":2031.98,"y":1.0}] 146985.0 3 +Manufacturer#2 almond aquamarine rose maroon antique 25 [{"x":1698.66,"y":1.0},{"x":1701.6,"y":1.0},{"x":1800.7,"y":1.0},{"x":2031.98,"y":1.0}] 169347.0 4 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 [{"x":1698.66,"y":1.0},{"x":1701.6,"y":1.0},{"x":2031.98,"y":1.0}] 146985.0 5 +Manufacturer#3 almond antique chartreuse khaki white 17 [{"x":1190.27,"y":1.0},{"x":1410.39,"y":1.0},{"x":1671.68,"y":1.0}] 90681.0 1 +Manufacturer#3 almond antique forest lavender goldenrod 14 [{"x":1190.27,"y":1.0},{"x":1410.39,"y":1.0},{"x":1671.68,"y":1.0},{"x":1922.98,"y":1.0}] 65831.5 2 +Manufacturer#3 almond antique metallic orange dim 19 [{"x":1190.27,"y":1.0},{"x":1337.29,"y":1.0},{"x":1410.39,"y":1.0},{"x":1671.68,"y":1.0},{"x":1922.98,"y":1.0}] 90681.0 3 +Manufacturer#3 almond antique misty red olive 1 [{"x":1190.27,"y":1.0},{"x":1337.29,"y":1.0},{"x":1410.39,"y":1.0},{"x":1922.98,"y":1.0}] 76690.0 4 +Manufacturer#3 almond antique olive coral navajo 45 [{"x":1337.29,"y":1.0},{"x":1410.39,"y":1.0},{"x":1922.98,"y":1.0}] 112398.0 5 +Manufacturer#4 almond antique gainsboro frosted violet 10 [{"x":1206.26,"y":1.0},{"x":1375.42,"y":1.0},{"x":1620.67,"y":1.0}] 48427.0 1 +Manufacturer#4 almond antique violet mint lemon 39 [{"x":1206.26,"y":1.0},{"x":1375.42,"y":1.0},{"x":1620.67,"y":1.0},{"x":1844.92,"y":1.0}] 46844.0 2 +Manufacturer#4 almond aquamarine floral ivory bisque 27 [{"x":1206.26,"y":1.0},{"x":1290.35,"y":1.0},{"x":1375.42,"y":1.0},{"x":1620.67,"y":1.0},{"x":1844.92,"y":1.0}] 45261.0 3 +Manufacturer#4 almond aquamarine yellow dodger mint 7 [{"x":1206.26,"y":1.0},{"x":1290.35,"y":1.0},{"x":1375.42,"y":1.0},{"x":1844.92,"y":1.0}] 39309.0 4 +Manufacturer#4 almond azure aquamarine papaya violet 12 [{"x":1206.26,"y":1.0},{"x":1290.35,"y":1.0},{"x":1844.92,"y":1.0}] 33357.0 5 +Manufacturer#5 almond antique blue firebrick mint 31 [{"x":1611.66,"y":1.0},{"x":1788.73,"y":1.0},{"x":1789.69,"y":1.0}] 155733.0 1 +Manufacturer#5 almond antique medium spring khaki 6 [{"x":1018.1,"y":1.0},{"x":1611.66,"y":1.0},{"x":1788.73,"y":1.0},{"x":1789.69,"y":1.0}] 99201.0 2 +Manufacturer#5 almond antique sky peru orange 2 [{"x":1018.1,"y":1.0},{"x":1464.48,"y":1.0},{"x":1611.66,"y":1.0},{"x":1788.73,"y":1.0},{"x":1789.69,"y":1.0}] 78486.0 3 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 [{"x":1018.1,"y":1.0},{"x":1464.48,"y":1.0},{"x":1611.66,"y":1.0},{"x":1788.73,"y":1.0}] 60577.5 4 +Manufacturer#5 almond azure blanched chiffon midnight 23 [{"x":1018.1,"y":1.0},{"x":1464.48,"y":1.0},{"x":1788.73,"y":1.0}] 78486.0 5 +PREHOOK: query: -- 42. testDistinctInSelectWithPTF +select DISTINCT p_mfgr, p_name, p_size +from noop(part +distribute by p_mfgr +sort by p_name) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 42. testDistinctInSelectWithPTF +select DISTINCT p_mfgr, p_name, p_size +from noop(part +distribute by p_mfgr +sort by p_name) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 +Manufacturer#1 almond antique chartreuse lavender yellow 34 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 +Manufacturer#1 almond aquamarine burnished black steel 28 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 +Manufacturer#2 almond antique violet chocolate turquoise 14 +Manufacturer#2 almond antique violet turquoise frosted 40 +Manufacturer#2 almond aquamarine midnight light salmon 2 +Manufacturer#2 almond aquamarine rose maroon antique 25 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 +Manufacturer#3 almond antique chartreuse khaki white 17 +Manufacturer#3 almond antique forest lavender goldenrod 14 +Manufacturer#3 almond antique metallic orange dim 19 +Manufacturer#3 almond antique misty red olive 1 +Manufacturer#3 almond antique olive coral navajo 45 +Manufacturer#4 almond antique gainsboro frosted violet 10 +Manufacturer#4 almond antique violet mint lemon 39 +Manufacturer#4 almond aquamarine floral ivory bisque 27 +Manufacturer#4 almond aquamarine yellow dodger mint 7 +Manufacturer#4 almond azure aquamarine papaya violet 12 +Manufacturer#5 almond antique blue firebrick mint 31 +Manufacturer#5 almond antique medium spring khaki 6 +Manufacturer#5 almond antique sky peru orange 2 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 +Manufacturer#5 almond azure blanched chiffon midnight 23 +PREHOOK: query: -- 43. testUDAFsNoWindowingNoPTFNoGBY +select p_mfgr,p_name, p_retailprice, +sum(p_retailprice) as s, +min(p_retailprice) as mi, +max(p_retailprice) as ma, +avg(p_retailprice) as av +from part +distribute by p_mfgr +sort by p_mfgr, p_name +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 43. testUDAFsNoWindowingNoPTFNoGBY +select p_mfgr,p_name, p_retailprice, +sum(p_retailprice) as s, +min(p_retailprice) as mi, +max(p_retailprice) as ma, +avg(p_retailprice) as av +from part +distribute by p_mfgr +sort by p_mfgr, p_name +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 1173.15 8749.730000000001 1173.15 1753.76 1458.2883333333336 +Manufacturer#1 almond antique burnished rose metallic 1173.15 8749.730000000001 1173.15 1753.76 1458.2883333333336 +Manufacturer#1 almond antique chartreuse lavender yellow 1753.76 8749.730000000001 1173.15 1753.76 1458.2883333333336 +Manufacturer#1 almond antique salmon chartreuse burlywood 1602.59 8749.730000000001 1173.15 1753.76 1458.2883333333336 +Manufacturer#1 almond aquamarine burnished black steel 1414.42 8749.730000000001 1173.15 1753.76 1458.2883333333336 +Manufacturer#1 almond aquamarine pink moccasin thistle 1632.66 8749.730000000001 1173.15 1753.76 1458.2883333333336 +Manufacturer#2 almond antique violet chocolate turquoise 1690.68 8923.62 1690.68 2031.98 1784.7240000000002 +Manufacturer#2 almond antique violet turquoise frosted 1800.7 8923.62 1690.68 2031.98 1784.7240000000002 +Manufacturer#2 almond aquamarine midnight light salmon 2031.98 8923.62 1690.68 2031.98 1784.7240000000002 +Manufacturer#2 almond aquamarine rose maroon antique 1698.66 8923.62 1690.68 2031.98 1784.7240000000002 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 1701.6 8923.62 1690.68 2031.98 1784.7240000000002 +Manufacturer#3 almond antique chartreuse khaki white 1671.68 7532.61 1190.27 1922.98 1506.522 +Manufacturer#3 almond antique forest lavender goldenrod 1190.27 7532.61 1190.27 1922.98 1506.522 +Manufacturer#3 almond antique metallic orange dim 1410.39 7532.61 1190.27 1922.98 1506.522 +Manufacturer#3 almond antique misty red olive 1922.98 7532.61 1190.27 1922.98 1506.522 +Manufacturer#3 almond antique olive coral navajo 1337.29 7532.61 1190.27 1922.98 1506.522 +Manufacturer#4 almond antique gainsboro frosted violet 1620.67 7337.620000000001 1206.26 1844.92 1467.5240000000001 +Manufacturer#4 almond antique violet mint lemon 1375.42 7337.620000000001 1206.26 1844.92 1467.5240000000001 +Manufacturer#4 almond aquamarine floral ivory bisque 1206.26 7337.620000000001 1206.26 1844.92 1467.5240000000001 +Manufacturer#4 almond aquamarine yellow dodger mint 1844.92 7337.620000000001 1206.26 1844.92 1467.5240000000001 +Manufacturer#4 almond azure aquamarine papaya violet 1290.35 7337.620000000001 1206.26 1844.92 1467.5240000000001 +Manufacturer#5 almond antique blue firebrick mint 1789.69 7672.66 1018.1 1789.69 1534.532 +Manufacturer#5 almond antique medium spring khaki 1611.66 7672.66 1018.1 1789.69 1534.532 +Manufacturer#5 almond antique sky peru orange 1788.73 7672.66 1018.1 1789.69 1534.532 +Manufacturer#5 almond aquamarine dodger light gainsboro 1018.1 7672.66 1018.1 1789.69 1534.532 +Manufacturer#5 almond azure blanched chiffon midnight 1464.48 7672.66 1018.1 1789.69 1534.532 +PREHOOK: query: -- 44. testViewAsTableInputWithWindowing +create view IF NOT EXISTS mfgr_price_view as +select p_mfgr, p_brand, +sum(p_retailprice) as s +from part +group by p_mfgr, p_brand +PREHOOK: type: CREATEVIEW +#### A masked pattern was here #### +POSTHOOK: query: -- 44. testViewAsTableInputWithWindowing +create view IF NOT EXISTS mfgr_price_view as +select p_mfgr, p_brand, +sum(p_retailprice) as s +from part +group by p_mfgr, p_brand +POSTHOOK: type: CREATEVIEW +POSTHOOK: Output: default@mfgr_price_view +#### A masked pattern was here #### +PREHOOK: query: select p_mfgr, p_brand, s, +sum(s) as s1 over (w1) +from mfgr_price_view +distribute by p_mfgr +sort by p_mfgr +window w1 as rows between 2 preceding and current row +PREHOOK: type: QUERY +PREHOOK: Input: default@mfgr_price_view +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: select p_mfgr, p_brand, s, +sum(s) as s1 over (w1) +from mfgr_price_view +distribute by p_mfgr +sort by p_mfgr +window w1 as rows between 2 preceding and current row +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mfgr_price_view +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 Brand#14 2346.3 2346.3 +Manufacturer#1 Brand#12 4800.84 7147.14 +Manufacturer#1 Brand#15 1602.59 8749.73 +Manufacturer#2 Brand#22 3491.38 3491.38 +Manufacturer#2 Brand#23 2031.98 5523.360000000001 +Manufacturer#2 Brand#24 1698.66 7222.02 +Manufacturer#2 Brand#25 1701.6 5432.24 +Manufacturer#3 Brand#32 3333.37 3333.37 +Manufacturer#3 Brand#31 1671.68 5005.05 +Manufacturer#3 Brand#34 1337.29 6342.34 +Manufacturer#3 Brand#35 1190.27 4199.24 +Manufacturer#4 Brand#41 4755.9400000000005 4755.9400000000005 +Manufacturer#4 Brand#42 2581.6800000000003 7337.620000000001 +Manufacturer#5 Brand#51 1611.66 1611.66 +Manufacturer#5 Brand#52 3254.17 4865.83 +Manufacturer#5 Brand#53 2806.83 7672.66 +PREHOOK: query: -- 45. testViewAsTableInputToPTF +select p_mfgr, p_brand, s, +sum(s) as s1 over (w1) +from noop(mfgr_price_view +distribute by p_mfgr +sort by p_mfgr) +window w1 as rows between 2 preceding and current row +PREHOOK: type: QUERY +PREHOOK: Input: default@mfgr_price_view +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 45. testViewAsTableInputToPTF +select p_mfgr, p_brand, s, +sum(s) as s1 over (w1) +from noop(mfgr_price_view +distribute by p_mfgr +sort by p_mfgr) +window w1 as rows between 2 preceding and current row +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mfgr_price_view +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 Brand#14 2346.3 2346.3 +Manufacturer#1 Brand#12 4800.84 7147.14 +Manufacturer#1 Brand#15 1602.59 8749.73 +Manufacturer#2 Brand#22 3491.38 3491.38 +Manufacturer#2 Brand#23 2031.98 5523.360000000001 +Manufacturer#2 Brand#24 1698.66 7222.02 +Manufacturer#2 Brand#25 1701.6 5432.24 +Manufacturer#3 Brand#32 3333.37 3333.37 +Manufacturer#3 Brand#31 1671.68 5005.05 +Manufacturer#3 Brand#34 1337.29 6342.34 +Manufacturer#3 Brand#35 1190.27 4199.24 +Manufacturer#4 Brand#41 4755.9400000000005 4755.9400000000005 +Manufacturer#4 Brand#42 2581.6800000000003 7337.620000000001 +Manufacturer#5 Brand#51 1611.66 1611.66 +Manufacturer#5 Brand#52 3254.17 4865.83 +Manufacturer#5 Brand#53 2806.83 7672.66 +PREHOOK: query: -- 46. testCreateViewWithWindowingQuery +create view IF NOT EXISTS mfgr_brand_price_view as +select p_mfgr, p_brand, +sum(p_retailprice) as s over (w1) +from part +distribute by p_mfgr +sort by p_mfgr +window w1 as rows between 2 preceding and current row +PREHOOK: type: CREATEVIEW +#### A masked pattern was here #### +POSTHOOK: query: -- 46. testCreateViewWithWindowingQuery +create view IF NOT EXISTS mfgr_brand_price_view as +select p_mfgr, p_brand, +sum(p_retailprice) as s over (w1) +from part +distribute by p_mfgr +sort by p_mfgr +window w1 as rows between 2 preceding and current row +POSTHOOK: type: CREATEVIEW +POSTHOOK: Output: default@mfgr_brand_price_view +#### A masked pattern was here #### +PREHOOK: query: select * from mfgr_brand_price_view +PREHOOK: type: QUERY +PREHOOK: Input: default@mfgr_brand_price_view +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: select * from mfgr_brand_price_view +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mfgr_brand_price_view +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 Brand#14 1173.15 +Manufacturer#1 Brand#12 2926.91 +Manufacturer#1 Brand#14 4100.0599999999995 +Manufacturer#1 Brand#15 4529.5 +Manufacturer#1 Brand#12 4190.16 +Manufacturer#1 Brand#12 4649.67 +Manufacturer#2 Brand#22 1690.68 +Manufacturer#2 Brand#22 3491.38 +Manufacturer#2 Brand#23 5523.360000000001 +Manufacturer#2 Brand#24 5531.34 +Manufacturer#2 Brand#25 5432.24 +Manufacturer#3 Brand#32 1410.39 +Manufacturer#3 Brand#31 3082.07 +Manufacturer#3 Brand#35 4272.34 +Manufacturer#3 Brand#32 4784.93 +Manufacturer#3 Brand#34 4450.54 +Manufacturer#4 Brand#41 1620.67 +Manufacturer#4 Brand#42 2996.09 +Manufacturer#4 Brand#42 4202.35 +Manufacturer#4 Brand#41 4426.6 +Manufacturer#4 Brand#41 4341.530000000001 +Manufacturer#5 Brand#52 1789.69 +Manufacturer#5 Brand#51 3401.3500000000004 +Manufacturer#5 Brand#53 5190.08 +Manufacturer#5 Brand#53 4418.490000000001 +Manufacturer#5 Brand#52 4271.3099999999995 +PREHOOK: query: -- 47. testLateralViews +select p_mfgr, p_name, +lv_col, p_size, sum(p_size) as s over (w1) +from (select p_mfgr, p_name, p_size, array(1,2,3) arr from part) p +lateral view explode(arr) part_lv as lv_col +distribute by p_mfgr +sort by p_name +window w1 as rows between 2 preceding and current row +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 47. testLateralViews +select p_mfgr, p_name, +lv_col, p_size, sum(p_size) as s over (w1) +from (select p_mfgr, p_name, p_size, array(1,2,3) arr from part) p +lateral view explode(arr) part_lv as lv_col +distribute by p_mfgr +sort by p_name +window w1 as rows between 2 preceding and current row +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 2 2 +Manufacturer#1 almond antique burnished rose metallic 1 2 4 +Manufacturer#1 almond antique burnished rose metallic 3 2 6 +Manufacturer#1 almond antique burnished rose metallic 1 2 6 +Manufacturer#1 almond antique burnished rose metallic 2 2 6 +Manufacturer#1 almond antique burnished rose metallic 3 2 6 +Manufacturer#1 almond antique chartreuse lavender yellow 1 34 38 +Manufacturer#1 almond antique chartreuse lavender yellow 2 34 70 +Manufacturer#1 almond antique chartreuse lavender yellow 3 34 102 +Manufacturer#1 almond antique salmon chartreuse burlywood 1 6 74 +Manufacturer#1 almond antique salmon chartreuse burlywood 2 6 46 +Manufacturer#1 almond antique salmon chartreuse burlywood 3 6 18 +Manufacturer#1 almond aquamarine burnished black steel 1 28 40 +Manufacturer#1 almond aquamarine burnished black steel 2 28 62 +Manufacturer#1 almond aquamarine burnished black steel 3 28 84 +Manufacturer#1 almond aquamarine pink moccasin thistle 1 42 98 +Manufacturer#1 almond aquamarine pink moccasin thistle 2 42 112 +Manufacturer#1 almond aquamarine pink moccasin thistle 3 42 126 +Manufacturer#2 almond antique violet chocolate turquoise 2 14 14 +Manufacturer#2 almond antique violet chocolate turquoise 1 14 28 +Manufacturer#2 almond antique violet chocolate turquoise 3 14 42 +Manufacturer#2 almond antique violet turquoise frosted 1 40 68 +Manufacturer#2 almond antique violet turquoise frosted 2 40 94 +Manufacturer#2 almond antique violet turquoise frosted 3 40 120 +Manufacturer#2 almond aquamarine midnight light salmon 1 2 82 +Manufacturer#2 almond aquamarine midnight light salmon 2 2 44 +Manufacturer#2 almond aquamarine midnight light salmon 3 2 6 +Manufacturer#2 almond aquamarine rose maroon antique 1 25 29 +Manufacturer#2 almond aquamarine rose maroon antique 2 25 52 +Manufacturer#2 almond aquamarine rose maroon antique 3 25 75 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 1 18 68 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 2 18 61 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 3 18 54 +Manufacturer#3 almond antique chartreuse khaki white 1 17 17 +Manufacturer#3 almond antique chartreuse khaki white 2 17 34 +Manufacturer#3 almond antique chartreuse khaki white 3 17 51 +Manufacturer#3 almond antique forest lavender goldenrod 1 14 48 +Manufacturer#3 almond antique forest lavender goldenrod 2 14 45 +Manufacturer#3 almond antique forest lavender goldenrod 3 14 42 +Manufacturer#3 almond antique metallic orange dim 1 19 47 +Manufacturer#3 almond antique metallic orange dim 2 19 52 +Manufacturer#3 almond antique metallic orange dim 3 19 57 +Manufacturer#3 almond antique misty red olive 1 1 39 +Manufacturer#3 almond antique misty red olive 2 1 21 +Manufacturer#3 almond antique misty red olive 3 1 3 +Manufacturer#3 almond antique olive coral navajo 1 45 47 +Manufacturer#3 almond antique olive coral navajo 2 45 91 +Manufacturer#3 almond antique olive coral navajo 3 45 135 +Manufacturer#4 almond antique gainsboro frosted violet 1 10 10 +Manufacturer#4 almond antique gainsboro frosted violet 2 10 20 +Manufacturer#4 almond antique gainsboro frosted violet 3 10 30 +Manufacturer#4 almond antique violet mint lemon 1 39 59 +Manufacturer#4 almond antique violet mint lemon 2 39 88 +Manufacturer#4 almond antique violet mint lemon 3 39 117 +Manufacturer#4 almond aquamarine floral ivory bisque 1 27 105 +Manufacturer#4 almond aquamarine floral ivory bisque 2 27 93 +Manufacturer#4 almond aquamarine floral ivory bisque 3 27 81 +Manufacturer#4 almond aquamarine yellow dodger mint 1 7 61 +Manufacturer#4 almond aquamarine yellow dodger mint 2 7 41 +Manufacturer#4 almond aquamarine yellow dodger mint 3 7 21 +Manufacturer#4 almond azure aquamarine papaya violet 1 12 26 +Manufacturer#4 almond azure aquamarine papaya violet 2 12 31 +Manufacturer#4 almond azure aquamarine papaya violet 3 12 36 +Manufacturer#5 almond antique blue firebrick mint 2 31 31 +Manufacturer#5 almond antique blue firebrick mint 1 31 62 +Manufacturer#5 almond antique blue firebrick mint 3 31 93 +Manufacturer#5 almond antique medium spring khaki 1 6 68 +Manufacturer#5 almond antique medium spring khaki 2 6 43 +Manufacturer#5 almond antique medium spring khaki 3 6 18 +Manufacturer#5 almond antique sky peru orange 2 2 14 +Manufacturer#5 almond antique sky peru orange 1 2 10 +Manufacturer#5 almond antique sky peru orange 3 2 6 +Manufacturer#5 almond aquamarine dodger light gainsboro 1 46 50 +Manufacturer#5 almond aquamarine dodger light gainsboro 2 46 94 +Manufacturer#5 almond aquamarine dodger light gainsboro 3 46 138 +Manufacturer#5 almond azure blanched chiffon midnight 1 23 115 +Manufacturer#5 almond azure blanched chiffon midnight 2 23 92 +Manufacturer#5 almond azure blanched chiffon midnight 3 23 69 +PREHOOK: query: -- 48. testWindowingPTFWithPartRC +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from noop(part_rc +distribute by p_mfgr +sort by p_name) +PREHOOK: type: QUERY +PREHOOK: Input: default@part_rc +#### A masked pattern was here #### +POSTHOOK: query: -- 48. testWindowingPTFWithPartRC +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from noop(part_rc +distribute by p_mfgr +sort by p_name) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_rc +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15 +Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3 +Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 5702.650000000001 +Manufacturer#1 almond aquamarine burnished black steel 28 5 4 7117.070000000001 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 8749.730000000001 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68 +Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38 +Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001 +Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 7222.02 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 8923.62 +Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68 +Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95 +Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34 +Manufacturer#3 almond antique misty red olive 1 4 4 6195.32 +Manufacturer#3 almond antique olive coral navajo 45 5 5 7532.61 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67 +Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09 +Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35 +Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 6047.27 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 7337.620000000001 +Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69 +Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004 +Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6208.18 +Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 7672.66 +PREHOOK: query: -- 49. testWindowingPTFWithPartSeqFile +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from noop(part_seq +distribute by p_mfgr +sort by p_name) +PREHOOK: type: QUERY +PREHOOK: Input: default@part_seq +#### A masked pattern was here #### +POSTHOOK: query: -- 49. testWindowingPTFWithPartSeqFile +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +from noop(part_seq +distribute by p_mfgr +sort by p_name) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_seq +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15 +Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3 +Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 5702.650000000001 +Manufacturer#1 almond aquamarine burnished black steel 28 5 4 7117.070000000001 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 8749.730000000001 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68 +Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38 +Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001 +Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 7222.02 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 8923.62 +Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68 +Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95 +Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34 +Manufacturer#3 almond antique misty red olive 1 4 4 6195.32 +Manufacturer#3 almond antique olive coral navajo 45 5 5 7532.61 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67 +Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09 +Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35 +Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 6047.27 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 7337.620000000001 +Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69 +Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004 +Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6208.18 +Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 7672.66 +PREHOOK: query: -- 50. testConstExprInSelect +select 'tst1' as key, count(1) as value from part +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 50. testConstExprInSelect +select 'tst1' as key, count(1) as value from part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +tst1 26 +PREHOOK: query: -- 51. testMultipleInserts3SWQs +CREATE TABLE part_1( +p_mfgr STRING, +p_name STRING, +p_size INT, +r INT, +dr INT, +s DOUBLE) +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- 51. testMultipleInserts3SWQs +CREATE TABLE part_1( +p_mfgr STRING, +p_name STRING, +p_size INT, +r INT, +dr INT, +s DOUBLE) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@part_1 +PREHOOK: query: CREATE TABLE part_2( +p_mfgr STRING, +p_name STRING, +p_size INT, +r INT, +dr INT, +cud INT, +s1 DOUBLE, +s2 DOUBLE, +fv1 INT) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE part_2( +p_mfgr STRING, +p_name STRING, +p_size INT, +r INT, +dr INT, +cud INT, +s1 DOUBLE, +s2 DOUBLE, +fv1 INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@part_2 +PREHOOK: query: CREATE TABLE part_3( +p_mfgr STRING, +p_name STRING, +p_size INT, +c INT, +ca INT, +fv INT) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE part_3( +p_mfgr STRING, +p_name STRING, +p_size INT, +c INT, +ca INT, +fv INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@part_3 +PREHOOK: query: from part +INSERT OVERWRITE TABLE part_1 +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s over (rows between unbounded preceding and current row) +distribute by p_mfgr +sort by p_name +INSERT OVERWRITE TABLE part_2 +select p_mfgr,p_name, p_size, +rank() as r, denserank() as dr, +cumedist() as cud, +sum(p_size) as s1 over (rows between unbounded preceding and current row), +sum(p_size) as s2 over (range between p_size 5 less and current row), +first_value(p_size, true) as fv1 over (w1) +having p_size > 5 +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following +INSERT OVERWRITE TABLE part_3 +select p_mfgr,p_name, p_size, +count(*) as c, +count(p_size) as ca, +first_value(p_size, true) as fv over (w1) +having p_size > 5 +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following +PREHOOK: type: QUERY +PREHOOK: Input: default@part +PREHOOK: Output: default@part_1 +PREHOOK: Output: default@part_2 +PREHOOK: Output: default@part_3 +POSTHOOK: query: from part +INSERT OVERWRITE TABLE part_1 +select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s over (rows between unbounded preceding and current row) +distribute by p_mfgr +sort by p_name +INSERT OVERWRITE TABLE part_2 +select p_mfgr,p_name, p_size, +rank() as r, denserank() as dr, +cumedist() as cud, +sum(p_size) as s1 over (rows between unbounded preceding and current row), +sum(p_size) as s2 over (range between p_size 5 less and current row), +first_value(p_size, true) as fv1 over (w1) +having p_size > 5 +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following +INSERT OVERWRITE TABLE part_3 +select p_mfgr,p_name, p_size, +count(*) as c, +count(p_size) as ca, +first_value(p_size, true) as fv over (w1) +having p_size > 5 +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +POSTHOOK: Output: default@part_1 +POSTHOOK: Output: default@part_2 +POSTHOOK: Output: default@part_3 +POSTHOOK: Lineage: part_1.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.c SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.ca SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.fv SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +PREHOOK: query: select * from part_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@part_1 +#### A masked pattern was here #### +POSTHOOK: query: select * from part_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: part_1.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.c SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.ca SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.fv SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15 +Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3 +Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 5702.650000000001 +Manufacturer#1 almond aquamarine burnished black steel 28 5 4 7117.070000000001 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 8749.730000000001 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68 +Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38 +Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001 +Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 7222.02 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 8923.62 +Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68 +Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95 +Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34 +Manufacturer#3 almond antique misty red olive 1 4 4 6195.32 +Manufacturer#3 almond antique olive coral navajo 45 5 5 7532.61 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67 +Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09 +Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35 +Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 6047.27 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 7337.620000000001 +Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69 +Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004 +Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6208.18 +Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 7672.66 +PREHOOK: query: select * from part_2 +PREHOOK: type: QUERY +PREHOOK: Input: default@part_2 +#### A masked pattern was here #### +POSTHOOK: query: select * from part_2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: part_1.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.c SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.ca SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.fv SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 0 38.0 34.0 2 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 0 44.0 44.0 2 +Manufacturer#1 almond aquamarine burnished black steel 28 5 4 0 72.0 28.0 34 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 1 114.0 42.0 6 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 0 14.0 14.0 14 +Manufacturer#2 almond antique violet turquoise frosted 40 2 2 0 54.0 40.0 14 +Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 0 81.0 25.0 40 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 1 99.0 43.0 2 +Manufacturer#3 almond antique chartreuse khaki white 17 1 1 0 17.0 17.0 17 +Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 0 31.0 31.0 17 +Manufacturer#3 almond antique metallic orange dim 19 3 3 0 50.0 19.0 17 +Manufacturer#3 almond antique olive coral navajo 45 5 5 1 96.0 45.0 19 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 0 10.0 10.0 10 +Manufacturer#4 almond antique violet mint lemon 39 2 2 0 49.0 39.0 10 +Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 0 76.0 66.0 10 +Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 0 83.0 83.0 39 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 1 95.0 12.0 27 +Manufacturer#5 almond antique blue firebrick mint 31 1 1 0 31.0 31.0 31 +Manufacturer#5 almond antique medium spring khaki 6 2 2 0 37.0 37.0 31 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 0 85.0 46.0 6 +Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 1 108.0 69.0 2 +PREHOOK: query: select * from part_3 +PREHOOK: type: QUERY +PREHOOK: Input: default@part_3 +#### A masked pattern was here #### +POSTHOOK: query: select * from part_3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_3 +#### A masked pattern was here #### +POSTHOOK: Lineage: part_1.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.c SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.ca SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.fv SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +Manufacturer#1 almond antique chartreuse lavender yellow 34 6 6 2 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 6 6 2 +Manufacturer#1 almond aquamarine burnished black steel 28 6 6 34 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 6 6 +Manufacturer#2 almond antique violet chocolate turquoise 14 5 5 14 +Manufacturer#2 almond antique violet turquoise frosted 40 5 5 14 +Manufacturer#2 almond aquamarine rose maroon antique 25 5 5 40 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 2 +Manufacturer#3 almond antique chartreuse khaki white 17 5 5 17 +Manufacturer#3 almond antique forest lavender goldenrod 14 5 5 17 +Manufacturer#3 almond antique metallic orange dim 19 5 5 17 +Manufacturer#3 almond antique olive coral navajo 45 5 5 19 +Manufacturer#4 almond antique gainsboro frosted violet 10 5 5 10 +Manufacturer#4 almond antique violet mint lemon 39 5 5 10 +Manufacturer#4 almond aquamarine floral ivory bisque 27 5 5 10 +Manufacturer#4 almond aquamarine yellow dodger mint 7 5 5 39 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 27 +Manufacturer#5 almond antique blue firebrick mint 31 5 5 31 +Manufacturer#5 almond antique medium spring khaki 6 5 5 31 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 5 5 6 +Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 2 +PREHOOK: query: -- 52. testGroupByHavingWithSWQAndAlias +select p_mfgr, p_name, p_size, min(p_retailprice) as mi, +rank() as r, +denserank() as dr, +p_size, p_size - lag(p_size,1) as deltaSz +from part +group by p_mfgr, p_name, p_size +having p_size > 0 +distribute by p_mfgr +sort by p_name +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 52. testGroupByHavingWithSWQAndAlias +select p_mfgr, p_name, p_size, min(p_retailprice) as mi, +rank() as r, +denserank() as dr, +p_size, p_size - lag(p_size,1) as deltaSz +from part +group by p_mfgr, p_name, p_size +having p_size > 0 +distribute by p_mfgr +sort by p_name +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: Lineage: part_1.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.c SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.ca SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.fv SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +Manufacturer#1 almond antique burnished rose metallic 2 1173.15 1 1 2 0 +Manufacturer#1 almond antique chartreuse lavender yellow 34 1753.76 2 2 34 32 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 1602.59 3 3 6 -28 +Manufacturer#1 almond aquamarine burnished black steel 28 1414.42 4 4 28 22 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 1632.66 5 5 42 14 +Manufacturer#2 almond antique violet chocolate turquoise 14 1690.68 1 1 14 0 +Manufacturer#2 almond antique violet turquoise frosted 40 1800.7 2 2 40 26 +Manufacturer#2 almond aquamarine midnight light salmon 2 2031.98 3 3 2 -38 +Manufacturer#2 almond aquamarine rose maroon antique 25 1698.66 4 4 25 23 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 1701.6 5 5 18 -7 +Manufacturer#3 almond antique chartreuse khaki white 17 1671.68 1 1 17 0 +Manufacturer#3 almond antique forest lavender goldenrod 14 1190.27 2 2 14 -3 +Manufacturer#3 almond antique metallic orange dim 19 1410.39 3 3 19 5 +Manufacturer#3 almond antique misty red olive 1 1922.98 4 4 1 -18 +Manufacturer#3 almond antique olive coral navajo 45 1337.29 5 5 45 44 +Manufacturer#4 almond antique gainsboro frosted violet 10 1620.67 1 1 10 0 +Manufacturer#4 almond antique violet mint lemon 39 1375.42 2 2 39 29 +Manufacturer#4 almond aquamarine floral ivory bisque 27 1206.26 3 3 27 -12 +Manufacturer#4 almond aquamarine yellow dodger mint 7 1844.92 4 4 7 -20 +Manufacturer#4 almond azure aquamarine papaya violet 12 1290.35 5 5 12 5 +Manufacturer#5 almond antique blue firebrick mint 31 1789.69 1 1 31 0 +Manufacturer#5 almond antique medium spring khaki 6 1611.66 2 2 6 -25 +Manufacturer#5 almond antique sky peru orange 2 1788.73 3 3 2 -4 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 1018.1 4 4 46 44 +Manufacturer#5 almond azure blanched chiffon midnight 23 1464.48 5 5 23 -23 +PREHOOK: query: -- 53. testMultipleRangeWindows +select p_mfgr,p_name, p_size, +sum(p_size) as s2 over (range between p_size 10 less and current row), +sum(p_size) as s1 over (range between current row and p_size 10 more ) +from part +distribute by p_mfgr +sort by p_mfgr, p_size +window w1 as rows between 2 preceding and 2 following +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 53. testMultipleRangeWindows +select p_mfgr,p_name, p_size, +sum(p_size) as s2 over (range between p_size 10 less and current row), +sum(p_size) as s1 over (range between current row and p_size 10 more ) +from part +distribute by p_mfgr +sort by p_mfgr, p_size +window w1 as rows between 2 preceding and 2 following +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: Lineage: part_1.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.c SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.ca SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.fv SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +Manufacturer#1 almond antique burnished rose metallic 2 2 10 +Manufacturer#1 almond antique burnished rose metallic 2 4 8 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 10 6 +Manufacturer#1 almond aquamarine burnished black steel 28 28 62 +Manufacturer#1 almond antique chartreuse lavender yellow 34 62 76 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 76 42 +Manufacturer#2 almond aquamarine midnight light salmon 2 2 2 +Manufacturer#2 almond antique violet chocolate turquoise 14 14 32 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 32 43 +Manufacturer#2 almond aquamarine rose maroon antique 25 43 25 +Manufacturer#2 almond antique violet turquoise frosted 40 40 40 +Manufacturer#3 almond antique misty red olive 1 1 1 +Manufacturer#3 almond antique forest lavender goldenrod 14 14 50 +Manufacturer#3 almond antique chartreuse khaki white 17 31 36 +Manufacturer#3 almond antique metallic orange dim 19 50 19 +Manufacturer#3 almond antique olive coral navajo 45 45 45 +Manufacturer#4 almond aquamarine yellow dodger mint 7 7 29 +Manufacturer#4 almond antique gainsboro frosted violet 10 17 22 +Manufacturer#4 almond azure aquamarine papaya violet 12 29 12 +Manufacturer#4 almond aquamarine floral ivory bisque 27 27 27 +Manufacturer#4 almond antique violet mint lemon 39 39 39 +Manufacturer#5 almond antique sky peru orange 2 2 8 +Manufacturer#5 almond antique medium spring khaki 6 8 6 +Manufacturer#5 almond azure blanched chiffon midnight 23 23 54 +Manufacturer#5 almond antique blue firebrick mint 31 54 31 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 46 46 +PREHOOK: query: -- 54. testMultipleInserts2SWQsWithPTF +CREATE TABLE part_4( +p_mfgr STRING, +p_name STRING, +p_size INT, +r INT, +dr INT, +s DOUBLE) +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- 54. testMultipleInserts2SWQsWithPTF +CREATE TABLE part_4( +p_mfgr STRING, +p_name STRING, +p_size INT, +r INT, +dr INT, +s DOUBLE) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@part_4 +POSTHOOK: Lineage: part_1.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.c SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.ca SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.fv SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +PREHOOK: query: CREATE TABLE part_5( +p_mfgr STRING, +p_name STRING, +p_size INT, +s1 INT, +s2 INT, +r INT, +dr INT, +cud DOUBLE, +fv1 INT) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE part_5( +p_mfgr STRING, +p_name STRING, +p_size INT, +s1 INT, +s2 INT, +r INT, +dr INT, +cud DOUBLE, +fv1 INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@part_5 +POSTHOOK: Lineage: part_1.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.c SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.ca SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.fv SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +PREHOOK: query: from noop(part +distribute by p_mfgr +sort by p_name) +INSERT OVERWRITE TABLE part_4 select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s over (rows between unbounded preceding and current row) +distribute by p_mfgr +sort by p_name +INSERT OVERWRITE TABLE part_5 select p_mfgr,p_name, p_size, +sum(p_size) as s1 over (rows between unbounded preceding and current row), +sum(p_size) as s2 over (range between p_size 5 less and current row), +rank() as r, +denserank() as dr, +cumedist() as cud, +first_value(p_size, true) as fv1 over (w1) +having p_size > 5 +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following +PREHOOK: type: QUERY +PREHOOK: Input: default@part +PREHOOK: Output: default@part_4 +PREHOOK: Output: default@part_5 +POSTHOOK: query: from noop(part +distribute by p_mfgr +sort by p_name) +INSERT OVERWRITE TABLE part_4 select p_mfgr, p_name, p_size, +rank() as r, +denserank() as dr, +sum(p_retailprice) as s over (rows between unbounded preceding and current row) +distribute by p_mfgr +sort by p_name +INSERT OVERWRITE TABLE part_5 select p_mfgr,p_name, p_size, +sum(p_size) as s1 over (rows between unbounded preceding and current row), +sum(p_size) as s2 over (range between p_size 5 less and current row), +rank() as r, +denserank() as dr, +cumedist() as cud, +first_value(p_size, true) as fv1 over (w1) +having p_size > 5 +distribute by p_mfgr +sort by p_mfgr, p_name +window w1 as rows between 2 preceding and 2 following +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +POSTHOOK: Output: default@part_4 +POSTHOOK: Output: default@part_5 +POSTHOOK: Lineage: part_1.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.c SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.ca SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.fv SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +PREHOOK: query: select * from part_4 +PREHOOK: type: QUERY +PREHOOK: Input: default@part_4 +#### A masked pattern was here #### +POSTHOOK: query: select * from part_4 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_4 +#### A masked pattern was here #### +POSTHOOK: Lineage: part_1.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.c SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.ca SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.fv SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15 +Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3 +Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 5702.650000000001 +Manufacturer#1 almond aquamarine burnished black steel 28 5 4 7117.070000000001 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 8749.730000000001 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68 +Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38 +Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001 +Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 7222.02 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 8923.62 +Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68 +Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95 +Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34 +Manufacturer#3 almond antique misty red olive 1 4 4 6195.32 +Manufacturer#3 almond antique olive coral navajo 45 5 5 7532.61 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67 +Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09 +Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35 +Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 6047.27 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 7337.620000000001 +Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69 +Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004 +Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6208.18 +Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 7672.66 +PREHOOK: query: select * from part_5 +PREHOOK: type: QUERY +PREHOOK: Input: default@part_5 +#### A masked pattern was here #### +POSTHOOK: query: select * from part_5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_5 +#### A masked pattern was here #### +POSTHOOK: Lineage: part_1.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.c SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.ca SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.fv SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +Manufacturer#1 almond antique chartreuse lavender yellow 34 38 34 3 2 0.5 2 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 44 44 4 3 0.6666666666666666 2 +Manufacturer#1 almond aquamarine burnished black steel 28 72 28 5 4 0.8333333333333334 34 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 114 42 6 5 1.0 6 +Manufacturer#2 almond antique violet chocolate turquoise 14 14 14 1 1 0.2 14 +Manufacturer#2 almond antique violet turquoise frosted 40 54 40 2 2 0.4 14 +Manufacturer#2 almond aquamarine rose maroon antique 25 81 25 4 4 0.8 40 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 99 43 5 5 1.0 2 +Manufacturer#3 almond antique chartreuse khaki white 17 17 17 1 1 0.2 17 +Manufacturer#3 almond antique forest lavender goldenrod 14 31 31 2 2 0.4 17 +Manufacturer#3 almond antique metallic orange dim 19 50 19 3 3 0.6 17 +Manufacturer#3 almond antique olive coral navajo 45 96 45 5 5 1.0 19 +Manufacturer#4 almond antique gainsboro frosted violet 10 10 10 1 1 0.2 10 +Manufacturer#4 almond antique violet mint lemon 39 49 39 2 2 0.4 10 +Manufacturer#4 almond aquamarine floral ivory bisque 27 76 66 3 3 0.6 10 +Manufacturer#4 almond aquamarine yellow dodger mint 7 83 83 4 4 0.8 39 +Manufacturer#4 almond azure aquamarine papaya violet 12 95 12 5 5 1.0 27 +Manufacturer#5 almond antique blue firebrick mint 31 31 31 1 1 0.2 31 +Manufacturer#5 almond antique medium spring khaki 6 37 37 2 2 0.4 31 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 85 46 4 4 0.8 6 +Manufacturer#5 almond azure blanched chiffon midnight 23 108 69 5 5 1.0 2 +PREHOOK: query: -- 55. testPartOrderInUDAFInvoke +select p_mfgr, p_name, p_size, +sum(p_size) as s over (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following) +from part +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 55. testPartOrderInUDAFInvoke +select p_mfgr, p_name, p_size, +sum(p_size) as s over (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following) +from part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: Lineage: part_1.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.c SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.ca SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.fv SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +Manufacturer#1 almond antique burnished rose metallic 2 38 +Manufacturer#1 almond antique burnished rose metallic 2 44 +Manufacturer#1 almond antique chartreuse lavender yellow 34 72 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 112 +Manufacturer#1 almond aquamarine burnished black steel 28 110 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 76 +Manufacturer#2 almond antique violet chocolate turquoise 14 56 +Manufacturer#2 almond antique violet turquoise frosted 40 81 +Manufacturer#2 almond aquamarine midnight light salmon 2 99 +Manufacturer#2 almond aquamarine rose maroon antique 25 85 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 45 +Manufacturer#3 almond antique chartreuse khaki white 17 50 +Manufacturer#3 almond antique forest lavender goldenrod 14 51 +Manufacturer#3 almond antique metallic orange dim 19 96 +Manufacturer#3 almond antique misty red olive 1 79 +Manufacturer#3 almond antique olive coral navajo 45 65 +Manufacturer#4 almond antique gainsboro frosted violet 10 76 +Manufacturer#4 almond antique violet mint lemon 39 83 +Manufacturer#4 almond aquamarine floral ivory bisque 27 95 +Manufacturer#4 almond aquamarine yellow dodger mint 7 85 +Manufacturer#4 almond azure aquamarine papaya violet 12 46 +Manufacturer#5 almond antique blue firebrick mint 31 39 +Manufacturer#5 almond antique medium spring khaki 6 85 +Manufacturer#5 almond antique sky peru orange 2 108 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 77 +Manufacturer#5 almond azure blanched chiffon midnight 23 71 +PREHOOK: query: -- 56. testPartOrderInWdwDef +select p_mfgr, p_name, p_size, +sum(p_size) as s over (w1) +from part +window w1 as distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 56. testPartOrderInWdwDef +select p_mfgr, p_name, p_size, +sum(p_size) as s over (w1) +from part +window w1 as distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: Lineage: part_1.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.c SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.ca SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.fv SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +Manufacturer#1 almond antique burnished rose metallic 2 38 +Manufacturer#1 almond antique burnished rose metallic 2 44 +Manufacturer#1 almond antique chartreuse lavender yellow 34 72 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 112 +Manufacturer#1 almond aquamarine burnished black steel 28 110 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 76 +Manufacturer#2 almond antique violet chocolate turquoise 14 56 +Manufacturer#2 almond antique violet turquoise frosted 40 81 +Manufacturer#2 almond aquamarine midnight light salmon 2 99 +Manufacturer#2 almond aquamarine rose maroon antique 25 85 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 45 +Manufacturer#3 almond antique chartreuse khaki white 17 50 +Manufacturer#3 almond antique forest lavender goldenrod 14 51 +Manufacturer#3 almond antique metallic orange dim 19 96 +Manufacturer#3 almond antique misty red olive 1 79 +Manufacturer#3 almond antique olive coral navajo 45 65 +Manufacturer#4 almond antique gainsboro frosted violet 10 76 +Manufacturer#4 almond antique violet mint lemon 39 83 +Manufacturer#4 almond aquamarine floral ivory bisque 27 95 +Manufacturer#4 almond aquamarine yellow dodger mint 7 85 +Manufacturer#4 almond azure aquamarine papaya violet 12 46 +Manufacturer#5 almond antique blue firebrick mint 31 39 +Manufacturer#5 almond antique medium spring khaki 6 85 +Manufacturer#5 almond antique sky peru orange 2 108 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 77 +Manufacturer#5 almond azure blanched chiffon midnight 23 71 +PREHOOK: query: -- 57. testDefaultPartitioningSpecRules +select p_mfgr, p_name, p_size, +sum(p_size) as s over (w1), + sum(p_size) as s2 over(w2) +from part +sort by p_name +window w1 as distribute by p_mfgr rows between 2 preceding and 2 following, + w2 as distribute by p_mfgr sort by p_name +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 57. testDefaultPartitioningSpecRules +select p_mfgr, p_name, p_size, +sum(p_size) as s over (w1), + sum(p_size) as s2 over(w2) +from part +sort by p_name +window w1 as distribute by p_mfgr rows between 2 preceding and 2 following, + w2 as distribute by p_mfgr sort by p_name +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: Lineage: part_1.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.c SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.ca SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.fv SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +Manufacturer#1 almond antique burnished rose metallic 2 38 114 +Manufacturer#1 almond antique burnished rose metallic 2 44 114 +Manufacturer#1 almond antique chartreuse lavender yellow 34 72 114 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 112 114 +Manufacturer#1 almond aquamarine burnished black steel 28 110 114 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 76 114 +Manufacturer#2 almond antique violet chocolate turquoise 14 56 99 +Manufacturer#2 almond antique violet turquoise frosted 40 81 99 +Manufacturer#2 almond aquamarine midnight light salmon 2 99 99 +Manufacturer#2 almond aquamarine rose maroon antique 25 85 99 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 45 99 +Manufacturer#3 almond antique chartreuse khaki white 17 50 96 +Manufacturer#3 almond antique forest lavender goldenrod 14 51 96 +Manufacturer#3 almond antique metallic orange dim 19 96 96 +Manufacturer#3 almond antique misty red olive 1 79 96 +Manufacturer#3 almond antique olive coral navajo 45 65 96 +Manufacturer#4 almond antique gainsboro frosted violet 10 76 95 +Manufacturer#4 almond antique violet mint lemon 39 83 95 +Manufacturer#4 almond aquamarine floral ivory bisque 27 95 95 +Manufacturer#4 almond aquamarine yellow dodger mint 7 85 95 +Manufacturer#4 almond azure aquamarine papaya violet 12 46 95 +Manufacturer#5 almond antique blue firebrick mint 31 39 108 +Manufacturer#5 almond antique medium spring khaki 6 85 108 +Manufacturer#5 almond antique sky peru orange 2 108 108 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 77 108 +Manufacturer#5 almond azure blanched chiffon midnight 23 71 108 +PREHOOK: query: -- 58. testWindowCrossReference +select p_mfgr, p_name, p_size, +sum(p_size) as s1 over (w1), +sum(p_size) as s2 over (w2) +from part +window w1 as distribute by p_mfgr sort by p_mfgr rows between 2 preceding and 2 following, + w2 as w1 +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 58. testWindowCrossReference +select p_mfgr, p_name, p_size, +sum(p_size) as s1 over (w1), +sum(p_size) as s2 over (w2) +from part +window w1 as distribute by p_mfgr sort by p_mfgr rows between 2 preceding and 2 following, + w2 as w1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: Lineage: part_1.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.c SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.ca SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.fv SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +Manufacturer#1 almond antique burnished rose metallic 2 38 38 +Manufacturer#1 almond antique chartreuse lavender yellow 34 44 44 +Manufacturer#1 almond antique burnished rose metallic 2 72 72 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 112 112 +Manufacturer#1 almond aquamarine burnished black steel 28 78 78 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 76 76 +Manufacturer#2 almond antique violet chocolate turquoise 14 56 56 +Manufacturer#2 almond antique violet turquoise frosted 40 81 81 +Manufacturer#2 almond aquamarine midnight light salmon 2 99 99 +Manufacturer#2 almond aquamarine rose maroon antique 25 85 85 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 45 45 +Manufacturer#3 almond antique metallic orange dim 19 50 50 +Manufacturer#3 almond antique chartreuse khaki white 17 51 51 +Manufacturer#3 almond antique forest lavender goldenrod 14 96 96 +Manufacturer#3 almond antique misty red olive 1 77 77 +Manufacturer#3 almond antique olive coral navajo 45 60 60 +Manufacturer#4 almond antique gainsboro frosted violet 10 76 76 +Manufacturer#4 almond antique violet mint lemon 39 83 83 +Manufacturer#4 almond aquamarine floral ivory bisque 27 95 95 +Manufacturer#4 almond aquamarine yellow dodger mint 7 85 85 +Manufacturer#4 almond azure aquamarine papaya violet 12 46 46 +Manufacturer#5 almond antique blue firebrick mint 31 39 39 +Manufacturer#5 almond antique medium spring khaki 6 85 85 +Manufacturer#5 almond antique sky peru orange 2 108 108 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 77 77 +Manufacturer#5 almond azure blanched chiffon midnight 23 71 71 +PREHOOK: query: -- 59. testWindowInheritance +select p_mfgr, p_name, p_size, +sum(p_size) as s1 over (w1), +sum(p_size) as s2 over (w2) +from part +window w1 as distribute by p_mfgr sort by p_mfgr rows between 2 preceding and 2 following, + w2 as w1 rows between unbounded preceding and current row +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 59. testWindowInheritance +select p_mfgr, p_name, p_size, +sum(p_size) as s1 over (w1), +sum(p_size) as s2 over (w2) +from part +window w1 as distribute by p_mfgr sort by p_mfgr rows between 2 preceding and 2 following, + w2 as w1 rows between unbounded preceding and current row +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: Lineage: part_1.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.c SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.ca SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.fv SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +Manufacturer#1 almond antique burnished rose metallic 2 38 2 +Manufacturer#1 almond antique chartreuse lavender yellow 34 44 36 +Manufacturer#1 almond antique burnished rose metallic 2 72 38 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 112 44 +Manufacturer#1 almond aquamarine burnished black steel 28 78 72 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 76 114 +Manufacturer#2 almond antique violet chocolate turquoise 14 56 14 +Manufacturer#2 almond antique violet turquoise frosted 40 81 54 +Manufacturer#2 almond aquamarine midnight light salmon 2 99 56 +Manufacturer#2 almond aquamarine rose maroon antique 25 85 81 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 45 99 +Manufacturer#3 almond antique metallic orange dim 19 50 19 +Manufacturer#3 almond antique chartreuse khaki white 17 51 36 +Manufacturer#3 almond antique forest lavender goldenrod 14 96 50 +Manufacturer#3 almond antique misty red olive 1 77 51 +Manufacturer#3 almond antique olive coral navajo 45 60 96 +Manufacturer#4 almond antique gainsboro frosted violet 10 76 10 +Manufacturer#4 almond antique violet mint lemon 39 83 49 +Manufacturer#4 almond aquamarine floral ivory bisque 27 95 76 +Manufacturer#4 almond aquamarine yellow dodger mint 7 85 83 +Manufacturer#4 almond azure aquamarine papaya violet 12 46 95 +Manufacturer#5 almond antique blue firebrick mint 31 39 31 +Manufacturer#5 almond antique medium spring khaki 6 85 37 +Manufacturer#5 almond antique sky peru orange 2 108 39 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 77 85 +Manufacturer#5 almond azure blanched chiffon midnight 23 71 108 +PREHOOK: query: -- 60. testWindowForwardReference +select p_mfgr, p_name, p_size, +sum(p_size) as s1 over (w1), +sum(p_size) as s2 over (w2), +sum(p_size) as s3 over (w3) +from part +distribute by p_mfgr +sort by p_mfgr +window w1 as rows between 2 preceding and 2 following, + w2 as w3, + w3 as rows between unbounded preceding and current row +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 60. testWindowForwardReference +select p_mfgr, p_name, p_size, +sum(p_size) as s1 over (w1), +sum(p_size) as s2 over (w2), +sum(p_size) as s3 over (w3) +from part +distribute by p_mfgr +sort by p_mfgr +window w1 as rows between 2 preceding and 2 following, + w2 as w3, + w3 as rows between unbounded preceding and current row +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: Lineage: part_1.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.c SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.ca SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.fv SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +Manufacturer#1 almond antique burnished rose metallic 2 38 2 2 +Manufacturer#1 almond antique chartreuse lavender yellow 34 44 36 36 +Manufacturer#1 almond antique burnished rose metallic 2 72 38 38 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 112 44 44 +Manufacturer#1 almond aquamarine burnished black steel 28 78 72 72 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 76 114 114 +Manufacturer#2 almond antique violet chocolate turquoise 14 56 14 14 +Manufacturer#2 almond antique violet turquoise frosted 40 81 54 54 +Manufacturer#2 almond aquamarine midnight light salmon 2 99 56 56 +Manufacturer#2 almond aquamarine rose maroon antique 25 85 81 81 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 45 99 99 +Manufacturer#3 almond antique metallic orange dim 19 50 19 19 +Manufacturer#3 almond antique chartreuse khaki white 17 51 36 36 +Manufacturer#3 almond antique forest lavender goldenrod 14 96 50 50 +Manufacturer#3 almond antique misty red olive 1 77 51 51 +Manufacturer#3 almond antique olive coral navajo 45 60 96 96 +Manufacturer#4 almond antique gainsboro frosted violet 10 76 10 10 +Manufacturer#4 almond antique violet mint lemon 39 83 49 49 +Manufacturer#4 almond aquamarine floral ivory bisque 27 95 76 76 +Manufacturer#4 almond aquamarine yellow dodger mint 7 85 83 83 +Manufacturer#4 almond azure aquamarine papaya violet 12 46 95 95 +Manufacturer#5 almond antique blue firebrick mint 31 39 31 31 +Manufacturer#5 almond antique medium spring khaki 6 85 37 37 +Manufacturer#5 almond antique sky peru orange 2 108 39 39 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 77 85 85 +Manufacturer#5 almond azure blanched chiffon midnight 23 71 108 108 +PREHOOK: query: -- 61. testWindowDefinitionPropagation +select p_mfgr, p_name, p_size, +sum(p_size) as s1 over (w1), +sum(p_size) as s2 over (w2), +sum(p_size) as s3 over (w3 rows between 2 preceding and 2 following) +from part +distribute by p_mfgr +sort by p_mfgr +window w1 as rows between 2 preceding and 2 following, + w2 as w3, + w3 as rows between unbounded preceding and current row +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 61. testWindowDefinitionPropagation +select p_mfgr, p_name, p_size, +sum(p_size) as s1 over (w1), +sum(p_size) as s2 over (w2), +sum(p_size) as s3 over (w3 rows between 2 preceding and 2 following) +from part +distribute by p_mfgr +sort by p_mfgr +window w1 as rows between 2 preceding and 2 following, + w2 as w3, + w3 as rows between unbounded preceding and current row +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: Lineage: part_1.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.c SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.ca SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.fv SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +Manufacturer#1 almond antique burnished rose metallic 2 38 2 38 +Manufacturer#1 almond antique chartreuse lavender yellow 34 44 36 44 +Manufacturer#1 almond antique burnished rose metallic 2 72 38 72 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 112 44 112 +Manufacturer#1 almond aquamarine burnished black steel 28 78 72 78 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 76 114 76 +Manufacturer#2 almond antique violet chocolate turquoise 14 56 14 56 +Manufacturer#2 almond antique violet turquoise frosted 40 81 54 81 +Manufacturer#2 almond aquamarine midnight light salmon 2 99 56 99 +Manufacturer#2 almond aquamarine rose maroon antique 25 85 81 85 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 45 99 45 +Manufacturer#3 almond antique metallic orange dim 19 50 19 50 +Manufacturer#3 almond antique chartreuse khaki white 17 51 36 51 +Manufacturer#3 almond antique forest lavender goldenrod 14 96 50 96 +Manufacturer#3 almond antique misty red olive 1 77 51 77 +Manufacturer#3 almond antique olive coral navajo 45 60 96 60 +Manufacturer#4 almond antique gainsboro frosted violet 10 76 10 76 +Manufacturer#4 almond antique violet mint lemon 39 83 49 83 +Manufacturer#4 almond aquamarine floral ivory bisque 27 95 76 95 +Manufacturer#4 almond aquamarine yellow dodger mint 7 85 83 85 +Manufacturer#4 almond azure aquamarine papaya violet 12 46 95 46 +Manufacturer#5 almond antique blue firebrick mint 31 39 31 39 +Manufacturer#5 almond antique medium spring khaki 6 85 37 85 +Manufacturer#5 almond antique sky peru orange 2 108 39 108 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 77 85 77 +Manufacturer#5 almond azure blanched chiffon midnight 23 71 108 71 +PREHOOK: query: -- 62. testDistinctWithWindowing +select DISTINCT p_mfgr, p_name, p_size, +sum(p_size) as s over (w1) +from part +distribute by p_mfgr +sort by p_name +window w1 as rows between 2 preceding and 2 following +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 62. testDistinctWithWindowing +select DISTINCT p_mfgr, p_name, p_size, +sum(p_size) as s over (w1) +from part +distribute by p_mfgr +sort by p_name +window w1 as rows between 2 preceding and 2 following +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: Lineage: part_1.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.c SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.ca SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.fv SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +Manufacturer#1 almond antique burnished rose metallic 2 42 +Manufacturer#1 almond antique chartreuse lavender yellow 34 70 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 112 +Manufacturer#1 almond aquamarine burnished black steel 28 110 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 76 +Manufacturer#2 almond antique violet chocolate turquoise 14 56 +Manufacturer#2 almond antique violet turquoise frosted 40 81 +Manufacturer#2 almond aquamarine midnight light salmon 2 99 +Manufacturer#2 almond aquamarine rose maroon antique 25 85 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 45 +Manufacturer#3 almond antique chartreuse khaki white 17 50 +Manufacturer#3 almond antique forest lavender goldenrod 14 51 +Manufacturer#3 almond antique metallic orange dim 19 96 +Manufacturer#3 almond antique misty red olive 1 79 +Manufacturer#3 almond antique olive coral navajo 45 65 +Manufacturer#4 almond antique gainsboro frosted violet 10 76 +Manufacturer#4 almond antique violet mint lemon 39 83 +Manufacturer#4 almond aquamarine floral ivory bisque 27 95 +Manufacturer#4 almond aquamarine yellow dodger mint 7 85 +Manufacturer#4 almond azure aquamarine papaya violet 12 46 +Manufacturer#5 almond antique blue firebrick mint 31 39 +Manufacturer#5 almond antique medium spring khaki 6 85 +Manufacturer#5 almond antique sky peru orange 2 108 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 77 +Manufacturer#5 almond azure blanched chiffon midnight 23 71 +PREHOOK: query: -- 63. testMulti2OperatorsFunctionChainWithMap +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +p_size, sum(p_size) as s1 over (rows between unbounded preceding and current row) +from noop( + noopwithmap( + noop( + noop(part + distribute by p_mfgr + sort by p_mfgr) + ) + distribute by p_mfgr,p_name + sort by p_mfgr,p_name) + distribute by p_mfgr,p_name + sort by p_mfgr,p_name) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 63. testMulti2OperatorsFunctionChainWithMap +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +p_size, sum(p_size) as s1 over (rows between unbounded preceding and current row) +from noop( + noopwithmap( + noop( + noop(part + distribute by p_mfgr + sort by p_mfgr) + ) + distribute by p_mfgr,p_name + sort by p_mfgr,p_name) + distribute by p_mfgr,p_name + sort by p_mfgr,p_name) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: Lineage: part_1.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.c SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.ca SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.fv SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +Manufacturer#1 almond antique burnished rose metallic 1 1 2 2 +Manufacturer#1 almond antique burnished rose metallic 1 1 2 4 +Manufacturer#1 almond antique chartreuse lavender yellow 1 1 34 34 +Manufacturer#1 almond antique salmon chartreuse burlywood 1 1 6 6 +Manufacturer#1 almond aquamarine burnished black steel 1 1 28 28 +Manufacturer#1 almond aquamarine pink moccasin thistle 1 1 42 42 +Manufacturer#2 almond antique violet chocolate turquoise 1 1 14 14 +Manufacturer#2 almond antique violet turquoise frosted 1 1 40 40 +Manufacturer#2 almond aquamarine midnight light salmon 1 1 2 2 +Manufacturer#2 almond aquamarine rose maroon antique 1 1 25 25 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 1 1 18 18 +Manufacturer#3 almond antique chartreuse khaki white 1 1 17 17 +Manufacturer#3 almond antique forest lavender goldenrod 1 1 14 14 +Manufacturer#3 almond antique metallic orange dim 1 1 19 19 +Manufacturer#3 almond antique misty red olive 1 1 1 1 +Manufacturer#3 almond antique olive coral navajo 1 1 45 45 +Manufacturer#4 almond antique gainsboro frosted violet 1 1 10 10 +Manufacturer#4 almond antique violet mint lemon 1 1 39 39 +Manufacturer#4 almond aquamarine floral ivory bisque 1 1 27 27 +Manufacturer#4 almond aquamarine yellow dodger mint 1 1 7 7 +Manufacturer#4 almond azure aquamarine papaya violet 1 1 12 12 +Manufacturer#5 almond antique blue firebrick mint 1 1 31 31 +Manufacturer#5 almond antique medium spring khaki 1 1 6 6 +Manufacturer#5 almond antique sky peru orange 1 1 2 2 +Manufacturer#5 almond aquamarine dodger light gainsboro 1 1 46 46 +Manufacturer#5 almond azure blanched chiffon midnight 1 1 23 23 +PREHOOK: query: -- 64. testMulti3OperatorsFunctionChain +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +p_size, sum(p_size) as s1 over (rows between unbounded preceding and current row) +from noop( + noop( + noop( + noop(part + distribute by p_mfgr + sort by p_mfgr) + ) + distribute by p_mfgr,p_name + sort by p_mfgr,p_name) + distribute by p_mfgr + sort by p_mfgr ) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 64. testMulti3OperatorsFunctionChain +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +p_size, sum(p_size) as s1 over (rows between unbounded preceding and current row) +from noop( + noop( + noop( + noop(part + distribute by p_mfgr + sort by p_mfgr) + ) + distribute by p_mfgr,p_name + sort by p_mfgr,p_name) + distribute by p_mfgr + sort by p_mfgr ) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: Lineage: part_1.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.c SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.ca SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.fv SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +Manufacturer#1 almond antique burnished rose metallic 1 1 2 2 +Manufacturer#1 almond antique chartreuse lavender yellow 1 1 34 36 +Manufacturer#1 almond antique burnished rose metallic 1 1 2 38 +Manufacturer#1 almond antique salmon chartreuse burlywood 1 1 6 44 +Manufacturer#1 almond aquamarine burnished black steel 1 1 28 72 +Manufacturer#1 almond aquamarine pink moccasin thistle 1 1 42 114 +Manufacturer#2 almond antique violet chocolate turquoise 1 1 14 14 +Manufacturer#2 almond antique violet turquoise frosted 1 1 40 54 +Manufacturer#2 almond aquamarine midnight light salmon 1 1 2 56 +Manufacturer#2 almond aquamarine rose maroon antique 1 1 25 81 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 1 1 18 99 +Manufacturer#3 almond antique metallic orange dim 1 1 19 19 +Manufacturer#3 almond antique chartreuse khaki white 1 1 17 36 +Manufacturer#3 almond antique forest lavender goldenrod 1 1 14 50 +Manufacturer#3 almond antique misty red olive 1 1 1 51 +Manufacturer#3 almond antique olive coral navajo 1 1 45 96 +Manufacturer#4 almond antique gainsboro frosted violet 1 1 10 10 +Manufacturer#4 almond antique violet mint lemon 1 1 39 49 +Manufacturer#4 almond aquamarine floral ivory bisque 1 1 27 76 +Manufacturer#4 almond aquamarine yellow dodger mint 1 1 7 83 +Manufacturer#4 almond azure aquamarine papaya violet 1 1 12 95 +Manufacturer#5 almond antique blue firebrick mint 1 1 31 31 +Manufacturer#5 almond antique medium spring khaki 1 1 6 37 +Manufacturer#5 almond antique sky peru orange 1 1 2 39 +Manufacturer#5 almond aquamarine dodger light gainsboro 1 1 46 85 +Manufacturer#5 almond azure blanched chiffon midnight 1 1 23 108 +PREHOOK: query: -- 65. testMultiOperatorChainWithNoWindowing +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +p_size, sum(p_size) as s1 +from noop( + noop( + noop( + noop(part + distribute by p_mfgr,p_name + sort by p_mfgr,p_name) + ) + distribute by p_mfgr + sort by p_mfgr)) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 65. testMultiOperatorChainWithNoWindowing +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +p_size, sum(p_size) as s1 +from noop( + noop( + noop( + noop(part + distribute by p_mfgr,p_name + sort by p_mfgr,p_name) + ) + distribute by p_mfgr + sort by p_mfgr)) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: Lineage: part_1.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.c SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.ca SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.fv SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +Manufacturer#1 almond antique burnished rose metallic 1 1 2 114 +Manufacturer#1 almond antique chartreuse lavender yellow 1 1 34 114 +Manufacturer#1 almond antique burnished rose metallic 1 1 2 114 +Manufacturer#1 almond antique salmon chartreuse burlywood 1 1 6 114 +Manufacturer#1 almond aquamarine burnished black steel 1 1 28 114 +Manufacturer#1 almond aquamarine pink moccasin thistle 1 1 42 114 +Manufacturer#2 almond antique violet chocolate turquoise 1 1 14 99 +Manufacturer#2 almond antique violet turquoise frosted 1 1 40 99 +Manufacturer#2 almond aquamarine midnight light salmon 1 1 2 99 +Manufacturer#2 almond aquamarine rose maroon antique 1 1 25 99 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 1 1 18 99 +Manufacturer#3 almond antique metallic orange dim 1 1 19 96 +Manufacturer#3 almond antique chartreuse khaki white 1 1 17 96 +Manufacturer#3 almond antique forest lavender goldenrod 1 1 14 96 +Manufacturer#3 almond antique misty red olive 1 1 1 96 +Manufacturer#3 almond antique olive coral navajo 1 1 45 96 +Manufacturer#4 almond antique gainsboro frosted violet 1 1 10 95 +Manufacturer#4 almond antique violet mint lemon 1 1 39 95 +Manufacturer#4 almond aquamarine floral ivory bisque 1 1 27 95 +Manufacturer#4 almond aquamarine yellow dodger mint 1 1 7 95 +Manufacturer#4 almond azure aquamarine papaya violet 1 1 12 95 +Manufacturer#5 almond antique blue firebrick mint 1 1 31 108 +Manufacturer#5 almond antique medium spring khaki 1 1 6 108 +Manufacturer#5 almond antique sky peru orange 1 1 2 108 +Manufacturer#5 almond aquamarine dodger light gainsboro 1 1 46 108 +Manufacturer#5 almond azure blanched chiffon midnight 1 1 23 108 +PREHOOK: query: -- 66. testMultiOperatorChainEndsWithNoopMap +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +p_size, sum(p_size) as s1 over (rows between unbounded preceding and current row) +from noopwithmap( + noop( + noop( + noop(part + distribute by p_mfgr,p_name + sort by p_mfgr,p_name) + ) + distribute by p_mfgr + sort by p_mfgr) + distribute by p_mfgr,p_name + sort by p_mfgr,p_name) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 66. testMultiOperatorChainEndsWithNoopMap +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +p_size, sum(p_size) as s1 over (rows between unbounded preceding and current row) +from noopwithmap( + noop( + noop( + noop(part + distribute by p_mfgr,p_name + sort by p_mfgr,p_name) + ) + distribute by p_mfgr + sort by p_mfgr) + distribute by p_mfgr,p_name + sort by p_mfgr,p_name) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: Lineage: part_1.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.c SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.ca SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.fv SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +Manufacturer#1 almond antique burnished rose metallic 1 1 2 2 +Manufacturer#1 almond antique burnished rose metallic 1 1 2 4 +Manufacturer#1 almond antique chartreuse lavender yellow 1 1 34 34 +Manufacturer#1 almond antique salmon chartreuse burlywood 1 1 6 6 +Manufacturer#1 almond aquamarine burnished black steel 1 1 28 28 +Manufacturer#1 almond aquamarine pink moccasin thistle 1 1 42 42 +Manufacturer#2 almond antique violet chocolate turquoise 1 1 14 14 +Manufacturer#2 almond antique violet turquoise frosted 1 1 40 40 +Manufacturer#2 almond aquamarine midnight light salmon 1 1 2 2 +Manufacturer#2 almond aquamarine rose maroon antique 1 1 25 25 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 1 1 18 18 +Manufacturer#3 almond antique chartreuse khaki white 1 1 17 17 +Manufacturer#3 almond antique forest lavender goldenrod 1 1 14 14 +Manufacturer#3 almond antique metallic orange dim 1 1 19 19 +Manufacturer#3 almond antique misty red olive 1 1 1 1 +Manufacturer#3 almond antique olive coral navajo 1 1 45 45 +Manufacturer#4 almond antique gainsboro frosted violet 1 1 10 10 +Manufacturer#4 almond antique violet mint lemon 1 1 39 39 +Manufacturer#4 almond aquamarine floral ivory bisque 1 1 27 27 +Manufacturer#4 almond aquamarine yellow dodger mint 1 1 7 7 +Manufacturer#4 almond azure aquamarine papaya violet 1 1 12 12 +Manufacturer#5 almond antique blue firebrick mint 1 1 31 31 +Manufacturer#5 almond antique medium spring khaki 1 1 6 6 +Manufacturer#5 almond antique sky peru orange 1 1 2 2 +Manufacturer#5 almond aquamarine dodger light gainsboro 1 1 46 46 +Manufacturer#5 almond azure blanched chiffon midnight 1 1 23 23 +PREHOOK: query: --67. testMultiOperatorChainWithDiffPartitionForWindow1 +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +p_size, +sum(p_size) as s1 over (rows between unbounded preceding and current row), +sum(p_size) as s2 over (distribute by p_mfgr,p_name sort by p_mfgr,p_name rows between unbounded preceding and current row) +from noop( + noopwithmap( + noop(part + distribute by p_mfgr, p_name + sort by p_mfgr, p_name) + distribute by p_mfgr + sort by p_mfgr + )) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: --67. testMultiOperatorChainWithDiffPartitionForWindow1 +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +p_size, +sum(p_size) as s1 over (rows between unbounded preceding and current row), +sum(p_size) as s2 over (distribute by p_mfgr,p_name sort by p_mfgr,p_name rows between unbounded preceding and current row) +from noop( + noopwithmap( + noop(part + distribute by p_mfgr, p_name + sort by p_mfgr, p_name) + distribute by p_mfgr + sort by p_mfgr + )) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: Lineage: part_1.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.c SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.ca SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.fv SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +Manufacturer#1 almond antique burnished rose metallic 1 1 2 2 2 +Manufacturer#1 almond antique burnished rose metallic 1 1 2 4 4 +Manufacturer#1 almond antique chartreuse lavender yellow 1 1 34 34 34 +Manufacturer#1 almond antique salmon chartreuse burlywood 1 1 6 6 6 +Manufacturer#1 almond aquamarine burnished black steel 1 1 28 28 28 +Manufacturer#1 almond aquamarine pink moccasin thistle 1 1 42 42 42 +Manufacturer#2 almond antique violet chocolate turquoise 1 1 14 14 14 +Manufacturer#2 almond antique violet turquoise frosted 1 1 40 40 40 +Manufacturer#2 almond aquamarine midnight light salmon 1 1 2 2 2 +Manufacturer#2 almond aquamarine rose maroon antique 1 1 25 25 25 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 1 1 18 18 18 +Manufacturer#3 almond antique chartreuse khaki white 1 1 17 17 17 +Manufacturer#3 almond antique forest lavender goldenrod 1 1 14 14 14 +Manufacturer#3 almond antique metallic orange dim 1 1 19 19 19 +Manufacturer#3 almond antique misty red olive 1 1 1 1 1 +Manufacturer#3 almond antique olive coral navajo 1 1 45 45 45 +Manufacturer#4 almond antique gainsboro frosted violet 1 1 10 10 10 +Manufacturer#4 almond antique violet mint lemon 1 1 39 39 39 +Manufacturer#4 almond aquamarine floral ivory bisque 1 1 27 27 27 +Manufacturer#4 almond aquamarine yellow dodger mint 1 1 7 7 7 +Manufacturer#4 almond azure aquamarine papaya violet 1 1 12 12 12 +Manufacturer#5 almond antique blue firebrick mint 1 1 31 31 31 +Manufacturer#5 almond antique medium spring khaki 1 1 6 6 6 +Manufacturer#5 almond antique sky peru orange 1 1 2 2 2 +Manufacturer#5 almond aquamarine dodger light gainsboro 1 1 46 46 46 +Manufacturer#5 almond azure blanched chiffon midnight 1 1 23 23 23 +PREHOOK: query: --68. testMultiOperatorChainWithDiffPartitionForWindow2 +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +p_size, +sum(p_size) as s1 over (rows between unbounded preceding and current row), +sum(p_size) as s2 over (distribute by p_mfgr sort by p_mfgr rows between unbounded preceding and current row) +from noopwithmap( + noop( + noop(part + distribute by p_mfgr, p_name + sort by p_mfgr, p_name) + )) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: --68. testMultiOperatorChainWithDiffPartitionForWindow2 +select p_mfgr, p_name, +rank() as r, +denserank() as dr, +p_size, +sum(p_size) as s1 over (rows between unbounded preceding and current row), +sum(p_size) as s2 over (distribute by p_mfgr sort by p_mfgr rows between unbounded preceding and current row) +from noopwithmap( + noop( + noop(part + distribute by p_mfgr, p_name + sort by p_mfgr, p_name) + )) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: Lineage: part_1.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.c SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.ca SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.fv SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +Manufacturer#1 almond antique burnished rose metallic 1 1 2 2 2 +Manufacturer#1 almond antique chartreuse lavender yellow 1 1 34 36 36 +Manufacturer#1 almond antique burnished rose metallic 1 1 2 38 38 +Manufacturer#1 almond antique salmon chartreuse burlywood 1 1 6 44 44 +Manufacturer#1 almond aquamarine burnished black steel 1 1 28 72 72 +Manufacturer#1 almond aquamarine pink moccasin thistle 1 1 42 114 114 +Manufacturer#2 almond antique violet chocolate turquoise 1 1 14 14 14 +Manufacturer#2 almond antique violet turquoise frosted 1 1 40 54 54 +Manufacturer#2 almond aquamarine midnight light salmon 1 1 2 56 56 +Manufacturer#2 almond aquamarine rose maroon antique 1 1 25 81 81 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 1 1 18 99 99 +Manufacturer#3 almond antique metallic orange dim 1 1 19 19 19 +Manufacturer#3 almond antique chartreuse khaki white 1 1 17 36 36 +Manufacturer#3 almond antique forest lavender goldenrod 1 1 14 50 50 +Manufacturer#3 almond antique misty red olive 1 1 1 51 51 +Manufacturer#3 almond antique olive coral navajo 1 1 45 96 96 +Manufacturer#4 almond antique gainsboro frosted violet 1 1 10 10 10 +Manufacturer#4 almond antique violet mint lemon 1 1 39 49 49 +Manufacturer#4 almond aquamarine floral ivory bisque 1 1 27 76 76 +Manufacturer#4 almond aquamarine yellow dodger mint 1 1 7 83 83 +Manufacturer#4 almond azure aquamarine papaya violet 1 1 12 95 95 +Manufacturer#5 almond antique blue firebrick mint 1 1 31 31 31 +Manufacturer#5 almond antique medium spring khaki 1 1 6 37 37 +Manufacturer#5 almond antique sky peru orange 1 1 2 39 39 +Manufacturer#5 almond aquamarine dodger light gainsboro 1 1 46 85 85 +Manufacturer#5 almond azure blanched chiffon midnight 1 1 23 108 108 +PREHOOK: query: -- 69. basic Npath test +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from npath( + 'LATE.LATE+', + 'LATE', arr_delay > 15, + 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath as tpath' + on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + ) +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: -- 69. basic Npath test +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from npath( + 'LATE.LATE+', + 'LATE', arr_delay > 15, + 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath as tpath' + on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + ) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: Lineage: part_1.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.c SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.ca SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.fv SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +Baltimore 1142 2010 10 20 6 [{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":20,"arr_delay":23.0,"fl_num":"1142"},{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":21,"arr_delay":105.0,"fl_num":"1142"},{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":22,"arr_delay":54.0,"fl_num":"1142"},{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":25,"arr_delay":92.0,"fl_num":"1142"},{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":26,"arr_delay":123.0,"fl_num":"1142"},{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":27,"arr_delay":49.0,"fl_num":"1142"}] +Baltimore 1142 2010 10 21 5 [{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":21,"arr_delay":105.0,"fl_num":"1142"},{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":22,"arr_delay":54.0,"fl_num":"1142"},{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":25,"arr_delay":92.0,"fl_num":"1142"},{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":26,"arr_delay":123.0,"fl_num":"1142"},{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":27,"arr_delay":49.0,"fl_num":"1142"}] +Baltimore 1142 2010 10 22 4 [{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":22,"arr_delay":54.0,"fl_num":"1142"},{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":25,"arr_delay":92.0,"fl_num":"1142"},{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":26,"arr_delay":123.0,"fl_num":"1142"},{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":27,"arr_delay":49.0,"fl_num":"1142"}] +Baltimore 1142 2010 10 25 3 [{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":25,"arr_delay":92.0,"fl_num":"1142"},{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":26,"arr_delay":123.0,"fl_num":"1142"},{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":27,"arr_delay":49.0,"fl_num":"1142"}] +Baltimore 1142 2010 10 26 2 [{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":26,"arr_delay":123.0,"fl_num":"1142"},{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":27,"arr_delay":49.0,"fl_num":"1142"}] +Chicago 1531 2010 10 21 2 [{"origin_city_name":"Chicago","dest_city_name":"New York","year":2010,"month":10,"day_of_month":21,"arr_delay":53.0,"fl_num":"1531"},{"origin_city_name":"Chicago","dest_city_name":"New York","year":2010,"month":10,"day_of_month":22,"arr_delay":16.0,"fl_num":"1531"}] +Chicago 1531 2010 10 25 3 [{"origin_city_name":"Chicago","dest_city_name":"New York","year":2010,"month":10,"day_of_month":25,"arr_delay":43.0,"fl_num":"1531"},{"origin_city_name":"Chicago","dest_city_name":"New York","year":2010,"month":10,"day_of_month":26,"arr_delay":29.0,"fl_num":"1531"},{"origin_city_name":"Chicago","dest_city_name":"New York","year":2010,"month":10,"day_of_month":27,"arr_delay":70.0,"fl_num":"1531"}] +Chicago 1531 2010 10 26 2 [{"origin_city_name":"Chicago","dest_city_name":"New York","year":2010,"month":10,"day_of_month":26,"arr_delay":29.0,"fl_num":"1531"},{"origin_city_name":"Chicago","dest_city_name":"New York","year":2010,"month":10,"day_of_month":27,"arr_delay":70.0,"fl_num":"1531"}] +Baltimore 1599 2010 10 21 2 [{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":21,"arr_delay":28.0,"fl_num":"1599"},{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":22,"arr_delay":18.0,"fl_num":"1599"}] +Baltimore 1599 2010 10 25 3 [{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":25,"arr_delay":106.0,"fl_num":"1599"},{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":26,"arr_delay":90.0,"fl_num":"1599"},{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":27,"arr_delay":92.0,"fl_num":"1599"}] +Baltimore 1599 2010 10 26 2 [{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":26,"arr_delay":90.0,"fl_num":"1599"},{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":27,"arr_delay":92.0,"fl_num":"1599"}] +Chicago 361 2010 10 20 2 [{"origin_city_name":"Chicago","dest_city_name":"New York","year":2010,"month":10,"day_of_month":20,"arr_delay":42.0,"fl_num":"361"},{"origin_city_name":"Chicago","dest_city_name":"New York","year":2010,"month":10,"day_of_month":21,"arr_delay":142.0,"fl_num":"361"}] +Washington 7291 2010 10 27 2 [{"origin_city_name":"Washington","dest_city_name":"New York","year":2010,"month":10,"day_of_month":27,"arr_delay":26.0,"fl_num":"7291"},{"origin_city_name":"Washington","dest_city_name":"New York","year":2010,"month":10,"day_of_month":28,"arr_delay":45.0,"fl_num":"7291"}] +Chicago 897 2010 10 20 4 [{"origin_city_name":"Chicago","dest_city_name":"New York","year":2010,"month":10,"day_of_month":20,"arr_delay":24.0,"fl_num":"897"},{"origin_city_name":"Chicago","dest_city_name":"New York","year":2010,"month":10,"day_of_month":21,"arr_delay":77.0,"fl_num":"897"},{"origin_city_name":"Chicago","dest_city_name":"New York","year":2010,"month":10,"day_of_month":22,"arr_delay":24.0,"fl_num":"897"},{"origin_city_name":"Chicago","dest_city_name":"New York","year":2010,"month":10,"day_of_month":24,"arr_delay":113.0,"fl_num":"897"}] +Chicago 897 2010 10 21 3 [{"origin_city_name":"Chicago","dest_city_name":"New York","year":2010,"month":10,"day_of_month":21,"arr_delay":77.0,"fl_num":"897"},{"origin_city_name":"Chicago","dest_city_name":"New York","year":2010,"month":10,"day_of_month":22,"arr_delay":24.0,"fl_num":"897"},{"origin_city_name":"Chicago","dest_city_name":"New York","year":2010,"month":10,"day_of_month":24,"arr_delay":113.0,"fl_num":"897"}] +Chicago 897 2010 10 22 2 [{"origin_city_name":"Chicago","dest_city_name":"New York","year":2010,"month":10,"day_of_month":22,"arr_delay":24.0,"fl_num":"897"},{"origin_city_name":"Chicago","dest_city_name":"New York","year":2010,"month":10,"day_of_month":24,"arr_delay":113.0,"fl_num":"897"}] +PREHOOK: query: -- 70. testRankWithPartitioning +select p_mfgr, p_name, p_size, +rank() as r over (distribute by p_mfgr sort by p_name ) +from part +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 70. testRankWithPartitioning +select p_mfgr, p_name, p_size, +rank() as r over (distribute by p_mfgr sort by p_name ) +from part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: Lineage: part_1.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_1.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_2.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.c SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.ca SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.fv SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_3.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_4.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_mfgr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_name SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.p_size SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +POSTHOOK: Lineage: part_5.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ] +Manufacturer#1 almond antique burnished rose metallic 2 1 +Manufacturer#1 almond antique burnished rose metallic 2 1 +Manufacturer#1 almond antique chartreuse lavender yellow 34 3 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 +Manufacturer#1 almond aquamarine burnished black steel 28 5 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 +Manufacturer#2 almond antique violet turquoise frosted 40 2 +Manufacturer#2 almond aquamarine midnight light salmon 2 3 +Manufacturer#2 almond aquamarine rose maroon antique 25 4 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 +Manufacturer#3 almond antique chartreuse khaki white 17 1 +Manufacturer#3 almond antique forest lavender goldenrod 14 2 +Manufacturer#3 almond antique metallic orange dim 19 3 +Manufacturer#3 almond antique misty red olive 1 4 +Manufacturer#3 almond antique olive coral navajo 45 5 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 +Manufacturer#4 almond antique violet mint lemon 39 2 +Manufacturer#4 almond aquamarine floral ivory bisque 27 3 +Manufacturer#4 almond aquamarine yellow dodger mint 7 4 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 +Manufacturer#5 almond antique blue firebrick mint 31 1 +Manufacturer#5 almond antique medium spring khaki 6 2 +Manufacturer#5 almond antique sky peru orange 2 3 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 +Manufacturer#5 almond azure blanched chiffon midnight 23 5