diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java index ccfc4cd..b459ad1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java @@ -613,6 +613,7 @@ private static void validateValueBoundaryExprType(ObjectInspector OI) case SHORT: case DECIMAL: case TIMESTAMP: + case TIMESTAMPLOCALTZ: case DATE: case STRING: case VARCHAR: diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/ValueBoundaryScanner.java ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/ValueBoundaryScanner.java index 88837dc..994d22c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/ValueBoundaryScanner.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/ValueBoundaryScanner.java @@ -18,9 +18,11 @@ package org.apache.hadoop.hive.ql.udf.ptf; +import java.sql.Timestamp; import java.util.Date; import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.common.type.TimestampTZ; import org.apache.hadoop.hive.ql.exec.PTFPartition; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.Order; @@ -428,8 +430,11 @@ public static SingleValueBoundaryScanner getScanner(BoundaryDef start, BoundaryD case INT: case LONG: case SHORT: - case TIMESTAMP: return new LongValueBoundaryScanner(start, end, exprDef); + case TIMESTAMP: + return new TimestampValueBoundaryScanner(start, end, exprDef); + case TIMESTAMPLOCALTZ: + return new TimestampLocalTZValueBoundaryScanner(start, end, exprDef); case DOUBLE: case FLOAT: return new DoubleValueBoundaryScanner(start, end, exprDef); @@ -569,6 +574,66 @@ public boolean isEqual(Object v1, Object v2) { } } +class TimestampValueBoundaryScanner extends SingleValueBoundaryScanner { + public TimestampValueBoundaryScanner(BoundaryDef start, BoundaryDef end, OrderExpressionDef expressionDef) { + super(start, end,expressionDef); + } + + @Override + public boolean isDistanceGreater(Object v1, Object v2, int amt) { + if (v1 != null && v2 != null) { + long l1 = PrimitiveObjectInspectorUtils.getTimestamp(v1, + (PrimitiveObjectInspector) expressionDef.getOI()).getTime(); + long l2 = PrimitiveObjectInspectorUtils.getTimestamp(v2, + (PrimitiveObjectInspector) expressionDef.getOI()).getTime(); + return (double)(l1-l2)/1000 > amt; // TODO: lossy conversion, distance is considered in seconds + } + return v1 != null || v2 != null; // True if only one value is null + } + + @Override + public boolean isEqual(Object v1, Object v2) { + if (v1 != null && v2 != null) { + Timestamp l1 = PrimitiveObjectInspectorUtils.getTimestamp(v1, + (PrimitiveObjectInspector) expressionDef.getOI()); + Timestamp l2 = PrimitiveObjectInspectorUtils.getTimestamp(v2, + (PrimitiveObjectInspector) expressionDef.getOI()); + return l1.equals(l2); + } + return v1 == null && v2 == null; // True if both are null + } +} + +class TimestampLocalTZValueBoundaryScanner extends SingleValueBoundaryScanner { + public TimestampLocalTZValueBoundaryScanner(BoundaryDef start, BoundaryDef end, OrderExpressionDef expressionDef) { + super(start, end,expressionDef); + } + + @Override + public boolean isDistanceGreater(Object v1, Object v2, int amt) { + if (v1 != null && v2 != null) { + long l1 = PrimitiveObjectInspectorUtils.getTimestampLocalTZ(v1, + (PrimitiveObjectInspector) expressionDef.getOI(), null).getEpochSecond(); + long l2 = PrimitiveObjectInspectorUtils.getTimestampLocalTZ(v2, + (PrimitiveObjectInspector) expressionDef.getOI(), null).getEpochSecond(); + return (l1 -l2) > amt; // TODO: lossy conversion, distance is considered seconds similar to timestamp + } + return v1 != null || v2 != null; // True if only one value is null + } + + @Override + public boolean isEqual(Object v1, Object v2) { + if (v1 != null && v2 != null) { + TimestampTZ l1 = PrimitiveObjectInspectorUtils.getTimestampLocalTZ(v1, + (PrimitiveObjectInspector) expressionDef.getOI(), null); + TimestampTZ l2 = PrimitiveObjectInspectorUtils.getTimestampLocalTZ(v2, + (PrimitiveObjectInspector) expressionDef.getOI(), null); + return l1.equals(l2); + } + return v1 == null && v2 == null; // True if both are null + } +} + class StringValueBoundaryScanner extends SingleValueBoundaryScanner { public StringValueBoundaryScanner(BoundaryDef start, BoundaryDef end, OrderExpressionDef expressionDef) { super(start, end,expressionDef); diff --git ql/src/test/queries/clientpositive/windowing_windowspec.q ql/src/test/queries/clientpositive/windowing_windowspec.q index c37aed3..7c5cc2c 100644 --- ql/src/test/queries/clientpositive/windowing_windowspec.q +++ ql/src/test/queries/clientpositive/windowing_windowspec.q @@ -9,8 +9,8 @@ create table over10k( d double, bo boolean, s string, - ts timestamp, - `dec` decimal, + ts timestamp, + `dec` decimal, bin binary) row format delimited fields terminated by '|'; @@ -40,3 +40,20 @@ select s, i, round((avg(d) over w1 + 10.0) - (avg(d) over w1 - 10.0),2) from ov set hive.cbo.enable=false; -- HIVE-9228 select s, i from ( select s, i, round((avg(d) over w1 + 10.0) - (avg(d) over w1 - 10.0),2) from over10k window w1 as (partition by s order by i)) X limit 7; + +create table over10k_2 +as +select t, si, i, b, f, d, bo, s, ts, cast(ts as timestamp with local time zone) as tstz, `dec`, bin +from over10k; + +select ts, i, sum(f) over (partition by i order by ts) +from over10k_2 +where i = 65536; + +select tstz, i, sum(f) over (partition by i order by tstz) +from over10k_2 +where i = 65536; + +select b, i, sum(f) over (partition by i order by b) +from over10k_2 +where i = 65536; diff --git ql/src/test/results/clientpositive/windowing_multipartitioning.q.out ql/src/test/results/clientpositive/windowing_multipartitioning.q.out index 8e27ff1..c2a2033 100644 --- ql/src/test/results/clientpositive/windowing_multipartitioning.q.out +++ ql/src/test/results/clientpositive/windowing_multipartitioning.q.out @@ -168,35 +168,35 @@ where s = 'tom allen' or s = 'bob steinbeck' POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k #### A masked pattern was here #### -bob steinbeck 11 47244642041 -bob steinbeck 1 47244642041 -bob steinbeck 2 47244642041 -bob steinbeck 7 47244642041 -bob steinbeck 8 47244642041 -bob steinbeck 9 47244642041 -bob steinbeck 6 47244642041 -bob steinbeck 10 47244642041 -bob steinbeck 3 47244642041 -bob steinbeck 4 47244642041 +bob steinbeck 11 4294967344 +bob steinbeck 1 8589934849 +bob steinbeck 2 12884902321 +bob steinbeck 7 17179869870 +bob steinbeck 8 21474837212 +bob steinbeck 9 25769804712 +bob steinbeck 6 30064772008 +bob steinbeck 10 34359739552 +bob steinbeck 3 38654707094 +bob steinbeck 4 42949674515 bob steinbeck 5 47244642041 -tom allen 9 81604381169 -tom allen 3 81604381169 -tom allen 7 81604381169 -tom allen 16 81604381169 -tom allen 8 81604381169 -tom allen 10 81604381169 -tom allen 15 81604381169 -tom allen 2 81604381169 -tom allen 6 81604381169 -tom allen 18 81604381169 -tom allen 1 81604381169 -tom allen 5 81604381169 -tom allen 19 81604381169 -tom allen 17 81604381169 -tom allen 11 81604381169 -tom allen 4 81604381169 -tom allen 12 81604381169 -tom allen 13 81604381169 +tom allen 9 4294967478 +tom allen 3 8589934816 +tom allen 7 12884902321 +tom allen 16 17179869673 +tom allen 8 21474837072 +tom allen 10 25769804454 +tom allen 15 30064771969 +tom allen 2 34359739365 +tom allen 6 38654706862 +tom allen 18 42949674383 +tom allen 1 47244641842 +tom allen 5 51539609307 +tom allen 19 55834576824 +tom allen 17 60129544192 +tom allen 11 64424511531 +tom allen 4 68719478972 +tom allen 12 73014446496 +tom allen 13 77309413835 tom allen 14 81604381169 PREHOOK: query: select s, sum(i) over (partition by s), sum(f) over (partition by si) from over10k where s = 'tom allen' or s = 'bob steinbeck' PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/windowing_windowspec.q.out ql/src/test/results/clientpositive/windowing_windowspec.q.out index 9da6183..96125ae 100644 --- ql/src/test/results/clientpositive/windowing_windowspec.q.out +++ ql/src/test/results/clientpositive/windowing_windowspec.q.out @@ -11,8 +11,8 @@ PREHOOK: query: create table over10k( d double, bo boolean, s string, - ts timestamp, - `dec` decimal, + ts timestamp, + `dec` decimal, bin binary) row format delimited fields terminated by '|' @@ -28,8 +28,8 @@ POSTHOOK: query: create table over10k( d double, bo boolean, s string, - ts timestamp, - `dec` decimal, + ts timestamp, + `dec` decimal, bin binary) row format delimited fields terminated by '|' @@ -953,3 +953,202 @@ alice allen 65609 alice allen 65662 alice allen 65670 alice allen 65720 +PREHOOK: query: create table over10k_2 +as +select t, si, i, b, f, d, bo, s, ts, cast(ts as timestamp with local time zone) as tstz, `dec`, bin +from over10k +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@over10k +PREHOOK: Output: database:default +PREHOOK: Output: default@over10k_2 +POSTHOOK: query: create table over10k_2 +as +select t, si, i, b, f, d, bo, s, ts, cast(ts as timestamp with local time zone) as tstz, `dec`, bin +from over10k +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@over10k +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over10k_2 +POSTHOOK: Lineage: over10k_2.b SIMPLE [(over10k)over10k.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over10k_2.bin SIMPLE [(over10k)over10k.FieldSchema(name:bin, type:binary, comment:null), ] +POSTHOOK: Lineage: over10k_2.bo SIMPLE [(over10k)over10k.FieldSchema(name:bo, type:boolean, comment:null), ] +POSTHOOK: Lineage: over10k_2.d SIMPLE [(over10k)over10k.FieldSchema(name:d, type:double, comment:null), ] +POSTHOOK: Lineage: over10k_2.dec SIMPLE [(over10k)over10k.FieldSchema(name:dec, type:decimal(10,0), comment:null), ] +POSTHOOK: Lineage: over10k_2.f SIMPLE [(over10k)over10k.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over10k_2.i SIMPLE [(over10k)over10k.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over10k_2.s SIMPLE [(over10k)over10k.FieldSchema(name:s, type:string, comment:null), ] +POSTHOOK: Lineage: over10k_2.si SIMPLE [(over10k)over10k.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over10k_2.t SIMPLE [(over10k)over10k.FieldSchema(name:t, type:tinyint, comment:null), ] +POSTHOOK: Lineage: over10k_2.ts SIMPLE [(over10k)over10k.FieldSchema(name:ts, type:timestamp, comment:null), ] +POSTHOOK: Lineage: over10k_2.tstz EXPRESSION [(over10k)over10k.FieldSchema(name:ts, type:timestamp, comment:null), ] +PREHOOK: query: select ts, i, sum(f) over (partition by i order by ts) +from over10k_2 +where i = 65536 +PREHOOK: type: QUERY +PREHOOK: Input: default@over10k_2 +#### A masked pattern was here #### +POSTHOOK: query: select ts, i, sum(f) over (partition by i order by ts) +from over10k_2 +where i = 65536 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over10k_2 +#### A masked pattern was here #### +2013-03-01 09:11:58.703071 65536 8.859999656677246 +2013-03-01 09:11:58.703073 65536 27.119999885559082 +2013-03-01 09:11:58.703077 65536 72.77999973297119 +2013-03-01 09:11:58.703079 65536 95.17999935150146 +2013-03-01 09:11:58.703086 65536 128.09999752044678 +2013-03-01 09:11:58.703094 65536 182.42999935150146 +2013-03-01 09:11:58.703108 65536 253.98000240325928 +2013-03-01 09:11:58.703116 65536 300.9500036239624 +2013-03-01 09:11:58.70312 65536 302.5000035762787 +2013-03-01 09:11:58.703131 65536 335.2500035762787 +2013-03-01 09:11:58.703132 65536 385.53000235557556 +2013-03-01 09:11:58.703134 65536 428.53000235557556 +2013-03-01 09:11:58.703135 65536 514.1800038814545 +2013-03-01 09:11:58.703141 65536 523.5100038051605 +2013-03-01 09:11:58.703142 65536 565.7100045681 +2013-03-01 09:11:58.703166 65536 647.9500024318695 +2013-03-01 09:11:58.703168 65536 724.6400048732758 +2013-03-01 09:11:58.703172 65536 762.2400033473969 +2013-03-01 09:11:58.703174 65536 777.9700028896332 +2013-03-01 09:11:58.703178 65536 848.7600038051605 +2013-03-01 09:11:58.703179 65536 881.3800027370453 +2013-03-01 09:11:58.7032 65536 897.3000028133392 +2013-03-01 09:11:58.703203 65536 923.6200025081635 +2013-03-01 09:11:58.70321 65536 989.1600034236908 +2013-03-01 09:11:58.703214 65536 1082.7200009822845 +2013-03-01 09:11:58.70322 65536 1100.440000295639 +2013-03-01 09:11:58.703231 65536 1159.7700021266937 +2013-03-01 09:11:58.703233 65536 1175.590001821518 +2013-03-01 09:11:58.703234 65536 1246.5500009059906 +2013-03-01 09:11:58.703239 65536 1262.5700013637543 +2013-03-01 09:11:58.703241 65536 1350.5800034999847 +2013-03-01 09:11:58.703244 65536 1424.2200028896332 +2013-03-01 09:11:58.703245 65536 1467.7900025844574 +2013-03-01 09:11:58.703247 65536 1571.1000039577484 +2013-03-01 09:11:58.703247 65536 1571.1000039577484 +2013-03-01 09:11:58.703271 65536 1661.1500070095062 +2013-03-01 09:11:58.703276 65536 1730.8700082302094 +2013-03-01 09:11:58.703277 65536 1766.5600068569183 +2013-03-01 09:11:58.703279 65536 1800.2000062465668 +2013-03-01 09:11:58.703284 65536 1820.1400067806244 +2013-03-01 09:11:58.703285 65536 1870.4600064754486 +2013-03-01 09:11:58.70331 65536 1969.2900083065033 +2013-03-01 09:11:58.70331 65536 1969.2900083065033 +2013-03-01 09:11:58.703319 65536 2066.030006170273 +2013-03-01 09:11:58.703319 65536 2066.030006170273 +PREHOOK: query: select tstz, i, sum(f) over (partition by i order by tstz) +from over10k_2 +where i = 65536 +PREHOOK: type: QUERY +PREHOOK: Input: default@over10k_2 +#### A masked pattern was here #### +POSTHOOK: query: select tstz, i, sum(f) over (partition by i order by tstz) +from over10k_2 +where i = 65536 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over10k_2 +#### A masked pattern was here #### +2013-03-01 09:11:58.703071 US/Pacific 65536 8.859999656677246 +2013-03-01 09:11:58.703073 US/Pacific 65536 27.119999885559082 +2013-03-01 09:11:58.703077 US/Pacific 65536 72.77999973297119 +2013-03-01 09:11:58.703079 US/Pacific 65536 95.17999935150146 +2013-03-01 09:11:58.703086 US/Pacific 65536 128.09999752044678 +2013-03-01 09:11:58.703094 US/Pacific 65536 182.42999935150146 +2013-03-01 09:11:58.703108 US/Pacific 65536 253.98000240325928 +2013-03-01 09:11:58.703116 US/Pacific 65536 300.9500036239624 +2013-03-01 09:11:58.70312 US/Pacific 65536 302.5000035762787 +2013-03-01 09:11:58.703131 US/Pacific 65536 335.2500035762787 +2013-03-01 09:11:58.703132 US/Pacific 65536 385.53000235557556 +2013-03-01 09:11:58.703134 US/Pacific 65536 428.53000235557556 +2013-03-01 09:11:58.703135 US/Pacific 65536 514.1800038814545 +2013-03-01 09:11:58.703141 US/Pacific 65536 523.5100038051605 +2013-03-01 09:11:58.703142 US/Pacific 65536 565.7100045681 +2013-03-01 09:11:58.703166 US/Pacific 65536 647.9500024318695 +2013-03-01 09:11:58.703168 US/Pacific 65536 724.6400048732758 +2013-03-01 09:11:58.703172 US/Pacific 65536 762.2400033473969 +2013-03-01 09:11:58.703174 US/Pacific 65536 777.9700028896332 +2013-03-01 09:11:58.703178 US/Pacific 65536 848.7600038051605 +2013-03-01 09:11:58.703179 US/Pacific 65536 881.3800027370453 +2013-03-01 09:11:58.7032 US/Pacific 65536 897.3000028133392 +2013-03-01 09:11:58.703203 US/Pacific 65536 923.6200025081635 +2013-03-01 09:11:58.70321 US/Pacific 65536 989.1600034236908 +2013-03-01 09:11:58.703214 US/Pacific 65536 1082.7200009822845 +2013-03-01 09:11:58.70322 US/Pacific 65536 1100.440000295639 +2013-03-01 09:11:58.703231 US/Pacific 65536 1159.7700021266937 +2013-03-01 09:11:58.703233 US/Pacific 65536 1175.590001821518 +2013-03-01 09:11:58.703234 US/Pacific 65536 1246.5500009059906 +2013-03-01 09:11:58.703239 US/Pacific 65536 1262.5700013637543 +2013-03-01 09:11:58.703241 US/Pacific 65536 1350.5800034999847 +2013-03-01 09:11:58.703244 US/Pacific 65536 1424.2200028896332 +2013-03-01 09:11:58.703245 US/Pacific 65536 1467.7900025844574 +2013-03-01 09:11:58.703247 US/Pacific 65536 1571.1000039577484 +2013-03-01 09:11:58.703247 US/Pacific 65536 1571.1000039577484 +2013-03-01 09:11:58.703271 US/Pacific 65536 1661.1500070095062 +2013-03-01 09:11:58.703276 US/Pacific 65536 1730.8700082302094 +2013-03-01 09:11:58.703277 US/Pacific 65536 1766.5600068569183 +2013-03-01 09:11:58.703279 US/Pacific 65536 1800.2000062465668 +2013-03-01 09:11:58.703284 US/Pacific 65536 1820.1400067806244 +2013-03-01 09:11:58.703285 US/Pacific 65536 1870.4600064754486 +2013-03-01 09:11:58.70331 US/Pacific 65536 1969.2900083065033 +2013-03-01 09:11:58.70331 US/Pacific 65536 1969.2900083065033 +2013-03-01 09:11:58.703319 US/Pacific 65536 2066.030006170273 +2013-03-01 09:11:58.703319 US/Pacific 65536 2066.030006170273 +PREHOOK: query: select b, i, sum(f) over (partition by i order by b) +from over10k_2 +where i = 65536 +PREHOOK: type: QUERY +PREHOOK: Input: default@over10k_2 +#### A masked pattern was here #### +POSTHOOK: query: select b, i, sum(f) over (partition by i order by b) +from over10k_2 +where i = 65536 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over10k_2 +#### A masked pattern was here #### +4294967299 65536 42.20000076293945 +4294967307 65536 118.8900032043457 +4294967308 65536 165.02000427246094 +4294967312 65536 208.02000427246094 +4294967317 65536 278.8100051879883 +4294967320 65536 311.7300033569336 +4294967326 65536 327.75000381469727 +4294967334 65536 421.310001373291 +4294967336 65536 454.060001373291 +4294967338 65536 472.32000160217285 +4294967341 65536 557.9700031280518 +4294967348 65536 573.7000026702881 +4294967349 65536 611.3000011444092 +4294967357 65536 701.350004196167 +4294967365 65536 717.2700042724609 +4294967375 65536 762.930004119873 +4294967381 65536 813.2500038146973 +4294967384 65536 882.9700050354004 +4294967394 65536 891.8300046920776 +4294967404 65536 925.3500051498413 +4294967412 65536 986.6300039291382 +4294967428 65536 1074.6400060653687 +4294967429 65536 1092.3600053787231 +4294967431 65536 1227.4200086593628 +4294967431 65536 1227.4200086593628 +4294967431 65536 1227.4200086593628 +4294967441 65536 1236.7500085830688 +4294967449 65536 1252.570008277893 +4294967451 65536 1288.0300073623657 +4294967460 65536 1373.35000705719 +4294967460 65536 1373.35000705719 +4294967463 65536 1427.6800088882446 +4294967476 65536 1487.0100107192993 +4294967478 65536 1556.8000116348267 +4294967484 65536 1558.350011587143 +4294967489 65536 1584.6700112819672 +4294967490 65536 1665.2800118923187 +4294967490 65536 1665.2800118923187 +4294967491 65536 1738.9200112819672 +4294967497 65536 1809.8800103664398 +4294967508 65536 1892.1200082302094 +4294967509 65536 1957.6600091457367 +4294967510 65536 1993.3500077724457 +4294967513 65536 2015.750007390976 +4294967530 65536 2066.030006170273