diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index fcedcfe..27239a4 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -312,6 +312,12 @@ minimr.query.negative.files=cluster_tasklog_retrieval.q,\ spark.query.files=spark_test.q \ avro_compression_enabled_native.q \ avro_decimal_native.q \ + ptf_decimal.q \ + ptf_general_queries.q \ + ptf_matchpath.q \ + ptf_rcfile.q \ + ptf_register_tblfn.q \ + ptf_seqfile.q \ timestamp_1.q \ timestamp_2.q \ timestamp_3.q \ diff --git ql/src/test/results/clientpositive/spark/ptf_decimal.q.out ql/src/test/results/clientpositive/spark/ptf_decimal.q.out new file mode 100644 index 0000000..de984c4 --- /dev/null +++ ql/src/test/results/clientpositive/spark/ptf_decimal.q.out @@ -0,0 +1,226 @@ +PREHOOK: query: DROP TABLE IF EXISTS part +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS part +POSTHOOK: type: DROPTABLE +PREHOOK: query: -- data setup +CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DECIMAL(6,2), + p_comment STRING +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part +POSTHOOK: query: -- data setup +CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DECIMAL(6,2), + p_comment STRING +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/part_tiny.txt' overwrite into table part +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@part +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/part_tiny.txt' overwrite into table part +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@part +PREHOOK: query: -- 1. aggregate functions with decimal type + +select p_mfgr, p_retailprice, +lead(p_retailprice) over (partition by p_mfgr ORDER BY p_name) as c1, +lag(p_retailprice) over (partition by p_mfgr ORDER BY p_name) as c2, +first_value(p_retailprice) over (partition by p_mfgr ORDER BY p_name) as c3, +last_value(p_retailprice) over (partition by p_mfgr ORDER BY p_name) as c4 +from part +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 1. aggregate functions with decimal type + +select p_mfgr, p_retailprice, +lead(p_retailprice) over (partition by p_mfgr ORDER BY p_name) as c1, +lag(p_retailprice) over (partition by p_mfgr ORDER BY p_name) as c2, +first_value(p_retailprice) over (partition by p_mfgr ORDER BY p_name) as c3, +last_value(p_retailprice) over (partition by p_mfgr ORDER BY p_name) as c4 +from part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 1173.15 1173.15 NULL 1173.15 1173.15 +Manufacturer#1 1173.15 1753.76 1173.15 1173.15 1173.15 +Manufacturer#1 1753.76 1602.59 1173.15 1173.15 1753.76 +Manufacturer#1 1602.59 1414.42 1753.76 1173.15 1602.59 +Manufacturer#1 1414.42 1632.66 1602.59 1173.15 1414.42 +Manufacturer#1 1632.66 NULL 1414.42 1173.15 1632.66 +Manufacturer#2 1690.68 1800.7 NULL 1690.68 1690.68 +Manufacturer#2 1800.7 2031.98 1690.68 1690.68 1800.7 +Manufacturer#2 2031.98 1698.66 1800.7 1690.68 2031.98 +Manufacturer#2 1698.66 1701.6 2031.98 1690.68 1698.66 +Manufacturer#2 1701.6 NULL 1698.66 1690.68 1701.6 +Manufacturer#3 1671.68 1190.27 NULL 1671.68 1671.68 +Manufacturer#3 1190.27 1410.39 1671.68 1671.68 1190.27 +Manufacturer#3 1410.39 1922.98 1190.27 1671.68 1410.39 +Manufacturer#3 1922.98 1337.29 1410.39 1671.68 1922.98 +Manufacturer#3 1337.29 NULL 1922.98 1671.68 1337.29 +Manufacturer#4 1620.67 1375.42 NULL 1620.67 1620.67 +Manufacturer#4 1375.42 1206.26 1620.67 1620.67 1375.42 +Manufacturer#4 1206.26 1844.92 1375.42 1620.67 1206.26 +Manufacturer#4 1844.92 1290.35 1206.26 1620.67 1844.92 +Manufacturer#4 1290.35 NULL 1844.92 1620.67 1290.35 +Manufacturer#5 1789.69 1611.66 NULL 1789.69 1789.69 +Manufacturer#5 1611.66 1788.73 1789.69 1789.69 1611.66 +Manufacturer#5 1788.73 1018.1 1611.66 1789.69 1788.73 +Manufacturer#5 1018.1 1464.48 1788.73 1789.69 1018.1 +Manufacturer#5 1464.48 NULL 1018.1 1789.69 1464.48 +PREHOOK: query: -- 2. ranking functions with decimal type + +select p_mfgr, p_retailprice, +row_number() over (PARTITION BY p_mfgr ORDER BY p_retailprice) as c1, +rank() over (PARTITION BY p_mfgr ORDER BY p_retailprice) as c2, +dense_rank() over (PARTITION BY p_mfgr ORDER BY p_retailprice) as c3, +percent_rank() over (PARTITION BY p_mfgr ORDER BY p_retailprice) as c4, +cume_dist() over (PARTITION BY p_mfgr ORDER BY p_retailprice) as c5, +ntile(5) over (PARTITION BY p_mfgr ORDER BY p_retailprice) as c6 +from part +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 2. ranking functions with decimal type + +select p_mfgr, p_retailprice, +row_number() over (PARTITION BY p_mfgr ORDER BY p_retailprice) as c1, +rank() over (PARTITION BY p_mfgr ORDER BY p_retailprice) as c2, +dense_rank() over (PARTITION BY p_mfgr ORDER BY p_retailprice) as c3, +percent_rank() over (PARTITION BY p_mfgr ORDER BY p_retailprice) as c4, +cume_dist() over (PARTITION BY p_mfgr ORDER BY p_retailprice) as c5, +ntile(5) over (PARTITION BY p_mfgr ORDER BY p_retailprice) as c6 +from part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 1173.15 1 1 1 0.0 0.3333333333333333 1 +Manufacturer#1 1173.15 2 1 1 0.0 0.3333333333333333 1 +Manufacturer#1 1414.42 3 3 2 0.4 0.5 2 +Manufacturer#1 1602.59 4 4 3 0.6 0.6666666666666666 3 +Manufacturer#1 1632.66 5 5 4 0.8 0.8333333333333334 4 +Manufacturer#1 1753.76 6 6 5 1.0 1.0 5 +Manufacturer#2 1690.68 1 1 1 0.0 0.2 1 +Manufacturer#2 1698.66 2 2 2 0.25 0.4 2 +Manufacturer#2 1701.6 3 3 3 0.5 0.6 3 +Manufacturer#2 1800.7 4 4 4 0.75 0.8 4 +Manufacturer#2 2031.98 5 5 5 1.0 1.0 5 +Manufacturer#3 1190.27 1 1 1 0.0 0.2 1 +Manufacturer#3 1337.29 2 2 2 0.25 0.4 2 +Manufacturer#3 1410.39 3 3 3 0.5 0.6 3 +Manufacturer#3 1671.68 4 4 4 0.75 0.8 4 +Manufacturer#3 1922.98 5 5 5 1.0 1.0 5 +Manufacturer#4 1206.26 1 1 1 0.0 0.2 1 +Manufacturer#4 1290.35 2 2 2 0.25 0.4 2 +Manufacturer#4 1375.42 3 3 3 0.5 0.6 3 +Manufacturer#4 1620.67 4 4 4 0.75 0.8 4 +Manufacturer#4 1844.92 5 5 5 1.0 1.0 5 +Manufacturer#5 1018.1 1 1 1 0.0 0.2 1 +Manufacturer#5 1464.48 2 2 2 0.25 0.4 2 +Manufacturer#5 1611.66 3 3 3 0.5 0.6 3 +Manufacturer#5 1788.73 4 4 4 0.75 0.8 4 +Manufacturer#5 1789.69 5 5 5 1.0 1.0 5 +PREHOOK: query: -- 3. order by decimal + +select p_mfgr, p_retailprice, +lag(p_retailprice) over (partition by p_mfgr ORDER BY p_retailprice desc) as c1 +from part +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 3. order by decimal + +select p_mfgr, p_retailprice, +lag(p_retailprice) over (partition by p_mfgr ORDER BY p_retailprice desc) as c1 +from part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 1753.76 NULL +Manufacturer#1 1632.66 1753.76 +Manufacturer#1 1602.59 1632.66 +Manufacturer#1 1414.42 1602.59 +Manufacturer#1 1173.15 1414.42 +Manufacturer#1 1173.15 1173.15 +Manufacturer#2 2031.98 NULL +Manufacturer#2 1800.7 2031.98 +Manufacturer#2 1701.6 1800.7 +Manufacturer#2 1698.66 1701.6 +Manufacturer#2 1690.68 1698.66 +Manufacturer#3 1922.98 NULL +Manufacturer#3 1671.68 1922.98 +Manufacturer#3 1410.39 1671.68 +Manufacturer#3 1337.29 1410.39 +Manufacturer#3 1190.27 1337.29 +Manufacturer#4 1844.92 NULL +Manufacturer#4 1620.67 1844.92 +Manufacturer#4 1375.42 1620.67 +Manufacturer#4 1290.35 1375.42 +Manufacturer#4 1206.26 1290.35 +Manufacturer#5 1789.69 NULL +Manufacturer#5 1788.73 1789.69 +Manufacturer#5 1611.66 1788.73 +Manufacturer#5 1464.48 1611.66 +Manufacturer#5 1018.1 1464.48 +PREHOOK: query: -- 4. partition by decimal + +select p_mfgr, p_retailprice, +lag(p_retailprice) over (partition by p_retailprice) as c1 +from part +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 4. partition by decimal + +select p_mfgr, p_retailprice, +lag(p_retailprice) over (partition by p_retailprice) as c1 +from part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#5 1018.1 NULL +Manufacturer#1 1173.15 NULL +Manufacturer#1 1173.15 1173.15 +Manufacturer#3 1190.27 NULL +Manufacturer#4 1206.26 NULL +Manufacturer#4 1290.35 NULL +Manufacturer#3 1337.29 NULL +Manufacturer#4 1375.42 NULL +Manufacturer#3 1410.39 NULL +Manufacturer#1 1414.42 NULL +Manufacturer#5 1464.48 NULL +Manufacturer#1 1602.59 NULL +Manufacturer#5 1611.66 NULL +Manufacturer#4 1620.67 NULL +Manufacturer#1 1632.66 NULL +Manufacturer#3 1671.68 NULL +Manufacturer#2 1690.68 NULL +Manufacturer#2 1698.66 NULL +Manufacturer#2 1701.6 NULL +Manufacturer#1 1753.76 NULL +Manufacturer#5 1788.73 NULL +Manufacturer#5 1789.69 NULL +Manufacturer#2 1800.7 NULL +Manufacturer#4 1844.92 NULL +Manufacturer#3 1922.98 NULL +Manufacturer#2 2031.98 NULL diff --git ql/src/test/results/clientpositive/spark/ptf_general_queries.q.out ql/src/test/results/clientpositive/spark/ptf_general_queries.q.out new file mode 100644 index 0000000..df42af5 --- /dev/null +++ ql/src/test/results/clientpositive/spark/ptf_general_queries.q.out @@ -0,0 +1,141 @@ +PREHOOK: query: DROP TABLE part +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE part +POSTHOOK: type: DROPTABLE +PREHOOK: query: -- data setup +CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part +POSTHOOK: query: -- data setup +CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/part_tiny.txt' overwrite into table part +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@part +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/part_tiny.txt' overwrite into table part +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@part +PREHOOK: query: -- 1. testNoPTFNoWindowing +select p_mfgr, p_name, p_size +from part +distribute by p_mfgr +sort by p_name +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 1. testNoPTFNoWindowing +select p_mfgr, p_name, p_size +from part +distribute by p_mfgr +sort by p_name +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#5 almond antique blue firebrick mint 31 +Manufacturer#1 almond antique burnished rose metallic 2 +Manufacturer#1 almond antique burnished rose metallic 2 +Manufacturer#3 almond antique chartreuse khaki white 17 +Manufacturer#1 almond antique chartreuse lavender yellow 34 +Manufacturer#3 almond antique forest lavender goldenrod 14 +Manufacturer#4 almond antique gainsboro frosted violet 10 +Manufacturer#5 almond antique medium spring khaki 6 +Manufacturer#3 almond antique metallic orange dim 19 +Manufacturer#3 almond antique misty red olive 1 +Manufacturer#3 almond antique olive coral navajo 45 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 +Manufacturer#5 almond antique sky peru orange 2 +Manufacturer#2 almond antique violet chocolate turquoise 14 +Manufacturer#4 almond antique violet mint lemon 39 +Manufacturer#2 almond antique violet turquoise frosted 40 +Manufacturer#1 almond aquamarine burnished black steel 28 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 +Manufacturer#4 almond aquamarine floral ivory bisque 27 +Manufacturer#2 almond aquamarine midnight light salmon 2 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 +Manufacturer#2 almond aquamarine rose maroon antique 25 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 +Manufacturer#4 almond aquamarine yellow dodger mint 7 +Manufacturer#4 almond azure aquamarine papaya violet 12 +Manufacturer#5 almond azure blanched chiffon midnight 23 +PREHOOK: query: -- 2. testUDAFsNoWindowingNoPTFNoGBY +select p_mfgr,p_name, p_retailprice, +sum(p_retailprice) over(partition by p_mfgr order by p_name) as s, +min(p_retailprice) over(partition by p_mfgr order by p_name) as mi, +max(p_retailprice) over(partition by p_mfgr order by p_name) as ma, +avg(p_retailprice) over(partition by p_mfgr order by p_name) as av +from part +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 2. testUDAFsNoWindowingNoPTFNoGBY +select p_mfgr,p_name, p_retailprice, +sum(p_retailprice) over(partition by p_mfgr order by p_name) as s, +min(p_retailprice) over(partition by p_mfgr order by p_name) as mi, +max(p_retailprice) over(partition by p_mfgr order by p_name) as ma, +avg(p_retailprice) over(partition by p_mfgr order by p_name) as av +from part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 1173.15 2346.3 1173.15 1173.15 1173.15 +Manufacturer#1 almond antique burnished rose metallic 1173.15 2346.3 1173.15 1173.15 1173.15 +Manufacturer#1 almond antique chartreuse lavender yellow 1753.76 4100.06 1173.15 1753.76 1366.6866666666667 +Manufacturer#1 almond antique salmon chartreuse burlywood 1602.59 5702.650000000001 1173.15 1753.76 1425.6625000000001 +Manufacturer#1 almond aquamarine burnished black steel 1414.42 7117.070000000001 1173.15 1753.76 1423.4140000000002 +Manufacturer#1 almond aquamarine pink moccasin thistle 1632.66 8749.730000000001 1173.15 1753.76 1458.2883333333336 +Manufacturer#2 almond antique violet chocolate turquoise 1690.68 1690.68 1690.68 1690.68 1690.68 +Manufacturer#2 almond antique violet turquoise frosted 1800.7 3491.38 1690.68 1800.7 1745.69 +Manufacturer#2 almond aquamarine midnight light salmon 2031.98 5523.360000000001 1690.68 2031.98 1841.1200000000001 +Manufacturer#2 almond aquamarine rose maroon antique 1698.66 7222.02 1690.68 2031.98 1805.505 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 1701.6 8923.62 1690.68 2031.98 1784.7240000000002 +Manufacturer#3 almond antique chartreuse khaki white 1671.68 1671.68 1671.68 1671.68 1671.68 +Manufacturer#3 almond antique forest lavender goldenrod 1190.27 2861.95 1190.27 1671.68 1430.975 +Manufacturer#3 almond antique metallic orange dim 1410.39 4272.34 1190.27 1671.68 1424.1133333333335 +Manufacturer#3 almond antique misty red olive 1922.98 6195.32 1190.27 1922.98 1548.83 +Manufacturer#3 almond antique olive coral navajo 1337.29 7532.61 1190.27 1922.98 1506.522 +Manufacturer#4 almond antique gainsboro frosted violet 1620.67 1620.67 1620.67 1620.67 1620.67 +Manufacturer#4 almond antique violet mint lemon 1375.42 2996.09 1375.42 1620.67 1498.045 +Manufacturer#4 almond aquamarine floral ivory bisque 1206.26 4202.35 1206.26 1620.67 1400.7833333333335 +Manufacturer#4 almond aquamarine yellow dodger mint 1844.92 6047.27 1206.26 1844.92 1511.8175 +Manufacturer#4 almond azure aquamarine papaya violet 1290.35 7337.620000000001 1206.26 1844.92 1467.5240000000001 +Manufacturer#5 almond antique blue firebrick mint 1789.69 1789.69 1789.69 1789.69 1789.69 +Manufacturer#5 almond antique medium spring khaki 1611.66 3401.3500000000004 1611.66 1789.69 1700.6750000000002 +Manufacturer#5 almond antique sky peru orange 1788.73 5190.08 1611.66 1789.69 1730.0266666666666 +Manufacturer#5 almond aquamarine dodger light gainsboro 1018.1 6208.18 1018.1 1789.69 1552.045 +Manufacturer#5 almond azure blanched chiffon midnight 1464.48 7672.66 1018.1 1789.69 1534.532 +PREHOOK: query: -- 3. testConstExprInSelect +select 'tst1' as key, count(1) as value from part +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 3. testConstExprInSelect +select 'tst1' as key, count(1) as value from part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +tst1 26 diff --git ql/src/test/results/clientpositive/spark/ptf_matchpath.q.out ql/src/test/results/clientpositive/spark/ptf_matchpath.q.out new file mode 100644 index 0000000..f04c998 --- /dev/null +++ ql/src/test/results/clientpositive/spark/ptf_matchpath.q.out @@ -0,0 +1,109 @@ +PREHOOK: query: DROP TABLE flights_tiny +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE flights_tiny +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table flights_tiny ( +ORIGIN_CITY_NAME string, +DEST_CITY_NAME string, +YEAR int, +MONTH int, +DAY_OF_MONTH int, +ARR_DELAY float, +FL_NUM string +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@flights_tiny +POSTHOOK: query: create table flights_tiny ( +ORIGIN_CITY_NAME string, +DEST_CITY_NAME string, +YEAR int, +MONTH int, +DAY_OF_MONTH int, +ARR_DELAY float, +FL_NUM string +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@flights_tiny +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/flights_tiny.txt' OVERWRITE INTO TABLE flights_tiny +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@flights_tiny +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/flights_tiny.txt' OVERWRITE INTO TABLE flights_tiny +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@flights_tiny +PREHOOK: query: -- 1. basic Matchpath test +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: -- 1. basic Matchpath test +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +Baltimore 1142 2010 10 20 6 20 +Baltimore 1142 2010 10 21 5 21 +Baltimore 1142 2010 10 22 4 22 +Baltimore 1142 2010 10 25 3 25 +Baltimore 1142 2010 10 26 2 26 +Chicago 1531 2010 10 21 2 21 +Chicago 1531 2010 10 25 3 25 +Chicago 1531 2010 10 26 2 26 +Baltimore 1599 2010 10 21 2 21 +Baltimore 1599 2010 10 25 3 25 +Baltimore 1599 2010 10 26 2 26 +Chicago 361 2010 10 20 2 20 +Washington 7291 2010 10 27 2 27 +Chicago 897 2010 10 20 4 20 +Chicago 897 2010 10 21 3 21 +Chicago 897 2010 10 22 2 22 +PREHOOK: query: -- 2. Matchpath on 1 partition +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +where fl_num = 1142 +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: -- 2. Matchpath on 1 partition +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +where fl_num = 1142 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +Baltimore 1142 2010 10 20 6 20 +Baltimore 1142 2010 10 21 5 21 +Baltimore 1142 2010 10 22 4 22 +Baltimore 1142 2010 10 25 3 25 +Baltimore 1142 2010 10 26 2 26 diff --git ql/src/test/results/clientpositive/spark/ptf_rcfile.q.out ql/src/test/results/clientpositive/spark/ptf_rcfile.q.out new file mode 100644 index 0000000..7cc0933 --- /dev/null +++ ql/src/test/results/clientpositive/spark/ptf_rcfile.q.out @@ -0,0 +1,88 @@ +PREHOOK: query: DROP TABLE part_rc +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE part_rc +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE part_rc( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) STORED AS RCFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_rc +POSTHOOK: query: CREATE TABLE part_rc( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) STORED AS RCFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_rc +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/part.rc' overwrite into table part_rc +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@part_rc +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/part.rc' overwrite into table part_rc +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@part_rc +PREHOOK: query: -- testWindowingPTFWithPartRC +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noop(on part_rc +partition by p_mfgr +order by p_name) +PREHOOK: type: QUERY +PREHOOK: Input: default@part_rc +#### A masked pattern was here #### +POSTHOOK: query: -- testWindowingPTFWithPartRC +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noop(on part_rc +partition by p_mfgr +order by p_name) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_rc +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15 +Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3 +Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 5702.650000000001 +Manufacturer#1 almond aquamarine burnished black steel 28 5 4 7117.070000000001 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 8749.730000000001 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68 +Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38 +Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001 +Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 7222.02 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 8923.62 +Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68 +Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95 +Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34 +Manufacturer#3 almond antique misty red olive 1 4 4 6195.32 +Manufacturer#3 almond antique olive coral navajo 45 5 5 7532.61 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67 +Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09 +Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35 +Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 6047.27 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 7337.620000000001 +Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69 +Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004 +Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6208.18 +Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 7672.66 diff --git ql/src/test/results/clientpositive/spark/ptf_register_tblfn.q.out ql/src/test/results/clientpositive/spark/ptf_register_tblfn.q.out new file mode 100644 index 0000000..5a02350 --- /dev/null +++ ql/src/test/results/clientpositive/spark/ptf_register_tblfn.q.out @@ -0,0 +1,90 @@ +PREHOOK: query: DROP TABLE flights_tiny +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE flights_tiny +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table flights_tiny ( +ORIGIN_CITY_NAME string, +DEST_CITY_NAME string, +YEAR int, +MONTH int, +DAY_OF_MONTH int, +ARR_DELAY float, +FL_NUM string +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@flights_tiny +POSTHOOK: query: create table flights_tiny ( +ORIGIN_CITY_NAME string, +DEST_CITY_NAME string, +YEAR int, +MONTH int, +DAY_OF_MONTH int, +ARR_DELAY float, +FL_NUM string +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@flights_tiny +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/flights_tiny.txt' OVERWRITE INTO TABLE flights_tiny +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@flights_tiny +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/flights_tiny.txt' OVERWRITE INTO TABLE flights_tiny +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@flights_tiny +PREHOOK: query: create temporary function matchpathtest as 'org.apache.hadoop.hive.ql.udf.ptf.MatchPath$MatchPathResolver' +PREHOOK: type: CREATEFUNCTION +PREHOOK: Output: matchpathtest +POSTHOOK: query: create temporary function matchpathtest as 'org.apache.hadoop.hive.ql.udf.ptf.MatchPath$MatchPathResolver' +POSTHOOK: type: CREATEFUNCTION +POSTHOOK: Output: matchpathtest +PREHOOK: query: -- 1. basic Matchpath test +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpathtest(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: -- 1. basic Matchpath test +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpathtest(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +Baltimore 1142 2010 10 20 6 20 +Baltimore 1142 2010 10 21 5 21 +Baltimore 1142 2010 10 22 4 22 +Baltimore 1142 2010 10 25 3 25 +Baltimore 1142 2010 10 26 2 26 +Chicago 1531 2010 10 21 2 21 +Chicago 1531 2010 10 25 3 25 +Chicago 1531 2010 10 26 2 26 +Baltimore 1599 2010 10 21 2 21 +Baltimore 1599 2010 10 25 3 25 +Baltimore 1599 2010 10 26 2 26 +Chicago 361 2010 10 20 2 20 +Washington 7291 2010 10 27 2 27 +Chicago 897 2010 10 20 4 20 +Chicago 897 2010 10 21 3 21 +Chicago 897 2010 10 22 2 22 +PREHOOK: query: drop temporary function matchpathtest +PREHOOK: type: DROPFUNCTION +PREHOOK: Output: matchpathtest +POSTHOOK: query: drop temporary function matchpathtest +POSTHOOK: type: DROPFUNCTION +POSTHOOK: Output: matchpathtest diff --git ql/src/test/results/clientpositive/spark/ptf_seqfile.q.out ql/src/test/results/clientpositive/spark/ptf_seqfile.q.out new file mode 100644 index 0000000..eb0f070 --- /dev/null +++ ql/src/test/results/clientpositive/spark/ptf_seqfile.q.out @@ -0,0 +1,88 @@ +PREHOOK: query: DROP TABLE part_seq +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE part_seq +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE part_seq( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) STORED AS SEQUENCEFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_seq +POSTHOOK: query: CREATE TABLE part_seq( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) STORED AS SEQUENCEFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_seq +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/part.seq' overwrite into table part_seq +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@part_seq +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/part.seq' overwrite into table part_seq +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@part_seq +PREHOOK: query: -- testWindowingPTFWithPartSeqFile +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noop(on part_seq +partition by p_mfgr +order by p_name) +PREHOOK: type: QUERY +PREHOOK: Input: default@part_seq +#### A masked pattern was here #### +POSTHOOK: query: -- testWindowingPTFWithPartSeqFile +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noop(on part_seq +partition by p_mfgr +order by p_name) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_seq +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15 +Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3 +Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 5702.650000000001 +Manufacturer#1 almond aquamarine burnished black steel 28 5 4 7117.070000000001 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 8749.730000000001 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68 +Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38 +Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001 +Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 7222.02 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 8923.62 +Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68 +Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95 +Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34 +Manufacturer#3 almond antique misty red olive 1 4 4 6195.32 +Manufacturer#3 almond antique olive coral navajo 45 5 5 7532.61 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67 +Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09 +Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35 +Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 6047.27 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 7337.620000000001 +Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69 +Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004 +Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6208.18 +Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 7672.66