diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java index 67c19ad..bdc2806 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java @@ -47,6 +47,7 @@ import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.TimestampUtils; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr; import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf; @@ -1074,41 +1075,20 @@ Object nextVector(Object previousVector, long batchSize) throws IOException { result = (LongColumnVector) previousVector; } - // Read present/isNull stream - super.nextVector(result, batchSize); - - data.nextVector(result, batchSize); - nanoVector.isNull = result.isNull; - nanos.nextVector(nanoVector, batchSize); - - if(result.isRepeating && nanoVector.isRepeating) { - batchSize = 1; - } - - // Non repeating values preset in the vector. Iterate thru the vector and populate the time + result.reset(); + Object obj = null; for (int i = 0; i < batchSize; i++) { - if (!result.isNull[i]) { - long ms = (result.vector[result.isRepeating ? 0 : i] + WriterImpl.BASE_TIMESTAMP) - * WriterImpl.MILLIS_PER_SECOND; - long ns = parseNanos(nanoVector.vector[nanoVector.isRepeating ? 0 : i]); - // the rounding error exists because java always rounds up when dividing integers - // -42001/1000 = -42; and -42001 % 1000 = -1 (+ 1000) - // to get the correct value we need - // (-42 - 1)*1000 + 999 = -42001 - // (42)*1000 + 1 = 42001 - if(ms < 0 && ns != 0) { - ms -= 1000; - } - // Convert millis into nanos and add the nano vector value to it - result.vector[i] = (ms * 1000000) + ns; + obj = next(obj); + if (obj == null) { + result.noNulls = false; + result.isNull[i] = true; + } else { + TimestampWritable writable = (TimestampWritable) obj; + Timestamp timestamp = writable.getTimestamp(); + result.vector[i] = TimestampUtils.getTimeNanoSec(timestamp); } } - if(!(result.isRepeating && nanoVector.isRepeating)) { - // both have to repeat for the result to be repeating - result.isRepeating = false; - } - return result; } diff --git ql/src/test/queries/clientpositive/vector_data_types.q ql/src/test/queries/clientpositive/vector_data_types.q index 4c38968..61356fa 100644 --- ql/src/test/queries/clientpositive/vector_data_types.q +++ ql/src/test/queries/clientpositive/vector_data_types.q @@ -40,8 +40,14 @@ EXPLAIN SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i LIMIT 20; +SELECT SUM(HASH(*)) +FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q; + SET hive.vectorized.execution.enabled=true; EXPLAIN select t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i LIMIT 20; -SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i LIMIT 20; \ No newline at end of file +SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i LIMIT 20; + +SELECT SUM(HASH(*)) +FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q; \ No newline at end of file diff --git ql/src/test/results/clientpositive/tez/vector_data_types.q.out ql/src/test/results/clientpositive/tez/vector_data_types.q.out index a3bf59d..4fb10f1 100644 --- ql/src/test/results/clientpositive/tez/vector_data_types.q.out +++ ql/src/test/results/clientpositive/tez/vector_data_types.q.out @@ -177,6 +177,17 @@ NULL 409 65536 4294967490 46.97 25.92 false fred miller 2013-03-01 09:11:58.7031 120 331 65539 4294967324 88.02 40.94 true holly nixon 2013-03-01 09:11:58.703262 96.64 yard duty 71 298 65540 4294967510 91.63 9.6 false gabriella zipper 2013-03-01 09:11:58.70314 68.14 geology 114 385 65541 4294967458 73.48 34.97 true oscar quirinius 2013-03-01 09:11:58.703143 72.33 xylophone band +PREHOOK: query: SELECT SUM(HASH(*)) +FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q +PREHOOK: type: QUERY +PREHOOK: Input: default@over1korc +#### A masked pattern was here #### +POSTHOOK: query: SELECT SUM(HASH(*)) +FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1korc +#### A masked pattern was here #### +-17045922556 PREHOOK: query: EXPLAIN select t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i LIMIT 20 PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN select t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i LIMIT 20 @@ -239,23 +250,34 @@ POSTHOOK: query: SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc OR POSTHOOK: type: QUERY POSTHOOK: Input: default@over1korc #### A masked pattern was here #### -108 301 65536 4294967357 90.05 17.59 true ethan johnson 1860-11-12 20:05:55.011470936 75.70 undecided -118 497 65536 4294967381 50.32 12.72 false david nixon 1860-11-12 20:05:55.011484936 83.48 values clariffication -18 280 65536 4294967320 32.92 45.94 false holly white 1860-11-12 20:05:55.011285936 58.86 topologyariffication -69 489 65536 4294967404 33.52 17.99 false oscar ichabod 1860-11-12 20:05:55.011446936 32.68 topologyariffication -27 405 65536 4294967508 82.24 29.41 true oscar ovid 1860-11-12 20:05:55.011365936 16.85 biologyariffication -42 495 65536 4294967431 43.57 46.81 false tom johnson 1860-11-12 20:05:55.011444936 62.25 american historytion -NULL 409 65536 4294967490 46.97 25.92 false fred miller 1860-11-12 20:05:55.011315936 33.45 history historytion -59 431 65537 4294967326 11.34 2.9 true oscar robinson 1860-11-12 20:05:55.011360936 81.04 zync studiesorytion -63 458 65537 4294967463 21.94 49.71 true fred van buren 1860-11-12 20:05:55.011477936 99.34 wind surfingorytion -121 355 65537 4294967437 85.9 10.99 true calvin brown 1860-11-12 20:05:55.011453936 4.49 yard dutyngorytion -72 267 65537 4294967460 55.2 42.89 false oscar carson 1860-11-12 20:05:55.011352936 51.91 topologyngorytion -22 264 65537 4294967419 8.07 10.71 false david xylophone 1860-11-12 20:05:55.011335936 11.87 undecidedngorytion -101 283 65538 4294967527 84.03 27.29 true yuri brown 1860-11-12 20:05:55.011505936 44.24 biologydngorytion -109 376 65538 4294967453 59.61 35.62 true priscilla xylophone 1860-11-12 20:05:55.011485936 9.53 study skillsorytion -83 353 65538 4294967339 0.08 23.91 true holly thompson 1860-11-12 20:05:55.011356936 30.27 quiet hoursorytion -29 331 65539 4294967420 73.18 28.96 true ethan brown 1860-11-12 20:05:55.011293936 58.85 zync studiesorytion -70 430 65539 4294967534 18.89 43.84 true tom carson 1860-11-12 20:05:55.011381936 21.93 joggyingiesorytion -120 331 65539 4294967324 88.02 40.94 true holly nixon 1860-11-12 20:05:55.011461936 96.64 yard dutyesorytion -71 298 65540 4294967510 91.63 9.6 false gabriella zipper 1860-11-12 20:05:55.011339936 68.14 geologyyesorytion -114 385 65541 4294967458 73.48 34.97 true oscar quirinius 1860-11-12 20:05:55.011342936 72.33 xylophone bandytion +108 301 65536 4294967357 90.05 17.59 true ethan johnson 2013-03-01 09:11:58.703271 75.70 undecided +118 497 65536 4294967381 50.32 12.72 false david nixon 2013-03-01 09:11:58.703285 83.48 values clariffication +18 280 65536 4294967320 32.92 45.94 false holly white 2013-03-01 09:11:58.703086 58.86 topologyariffication +69 489 65536 4294967404 33.52 17.99 false oscar ichabod 2013-03-01 09:11:58.703247 32.68 topologyariffication +27 405 65536 4294967508 82.24 29.41 true oscar ovid 2013-03-01 09:11:58.703166 16.85 biologyariffication +42 495 65536 4294967431 43.57 46.81 false tom johnson 2013-03-01 09:11:58.703245 62.25 american historytion +NULL 409 65536 4294967490 46.97 25.92 false fred miller 2013-03-01 09:11:58.703116 33.45 history historytion +59 431 65537 4294967326 11.34 2.9 true oscar robinson 2013-03-01 09:11:58.703161 81.04 zync studiesorytion +63 458 65537 4294967463 21.94 49.71 true fred van buren 2013-03-01 09:11:58.703278 99.34 wind surfingorytion +121 355 65537 4294967437 85.9 10.99 true calvin brown 2013-03-01 09:11:58.703254 4.49 yard dutyngorytion +72 267 65537 4294967460 55.2 42.89 false oscar carson 2013-03-01 09:11:58.703153 51.91 topologyngorytion +22 264 65537 4294967419 8.07 10.71 false david xylophone 2013-03-01 09:11:58.703136 11.87 undecidedngorytion +101 283 65538 4294967527 84.03 27.29 true yuri brown 2013-03-01 09:11:58.703306 44.24 biologydngorytion +109 376 65538 4294967453 59.61 35.62 true priscilla xylophone 2013-03-01 09:11:58.703286 9.53 study skillsorytion +83 353 65538 4294967339 0.08 23.91 true holly thompson 2013-03-01 09:11:58.703157 30.27 quiet hoursorytion +29 331 65539 4294967420 73.18 28.96 true ethan brown 2013-03-01 09:11:58.703094 58.85 zync studiesorytion +70 430 65539 4294967534 18.89 43.84 true tom carson 2013-03-01 09:11:58.703182 21.93 joggyingiesorytion +120 331 65539 4294967324 88.02 40.94 true holly nixon 2013-03-01 09:11:58.703262 96.64 yard dutyesorytion +71 298 65540 4294967510 91.63 9.6 false gabriella zipper 2013-03-01 09:11:58.70314 68.14 geologyyesorytion +114 385 65541 4294967458 73.48 34.97 true oscar quirinius 2013-03-01 09:11:58.703143 72.33 xylophone bandytion +PREHOOK: query: SELECT SUM(HASH(*)) +FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q +PREHOOK: type: QUERY +PREHOOK: Input: default@over1korc +#### A masked pattern was here #### +POSTHOOK: query: SELECT SUM(HASH(*)) +FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1korc +#### A masked pattern was here #### +-17045922556 diff --git ql/src/test/results/clientpositive/vector_data_types.q.out ql/src/test/results/clientpositive/vector_data_types.q.out index 6b0598c..07d58ed 100644 --- ql/src/test/results/clientpositive/vector_data_types.q.out +++ ql/src/test/results/clientpositive/vector_data_types.q.out @@ -171,6 +171,17 @@ NULL 409 65536 4294967490 46.97 25.92 false fred miller 2013-03-01 09:11:58.7031 120 331 65539 4294967324 88.02 40.94 true holly nixon 2013-03-01 09:11:58.703262 96.64 yard duty 71 298 65540 4294967510 91.63 9.6 false gabriella zipper 2013-03-01 09:11:58.70314 68.14 geology 114 385 65541 4294967458 73.48 34.97 true oscar quirinius 2013-03-01 09:11:58.703143 72.33 xylophone band +PREHOOK: query: SELECT SUM(HASH(*)) +FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q +PREHOOK: type: QUERY +PREHOOK: Input: default@over1korc +#### A masked pattern was here #### +POSTHOOK: query: SELECT SUM(HASH(*)) +FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1korc +#### A masked pattern was here #### +-17045922556 PREHOOK: query: EXPLAIN select t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i LIMIT 20 PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN select t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i LIMIT 20 @@ -226,23 +237,34 @@ POSTHOOK: query: SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc OR POSTHOOK: type: QUERY POSTHOOK: Input: default@over1korc #### A masked pattern was here #### -108 301 65536 4294967357 90.05 17.59 true ethan johnson 1860-11-12 20:05:55.011470936 75.70 undecided -118 497 65536 4294967381 50.32 12.72 false david nixon 1860-11-12 20:05:55.011484936 83.48 values clariffication -18 280 65536 4294967320 32.92 45.94 false holly white 1860-11-12 20:05:55.011285936 58.86 topology -69 489 65536 4294967404 33.52 17.99 false oscar ichabod 1860-11-12 20:05:55.011446936 32.68 topology -27 405 65536 4294967508 82.24 29.41 true oscar ovid 1860-11-12 20:05:55.011365936 16.85 biology -42 495 65536 4294967431 43.57 46.81 false tom johnson 1860-11-12 20:05:55.011444936 62.25 american history -NULL 409 65536 4294967490 46.97 25.92 false fred miller 1860-11-12 20:05:55.011315936 33.45 history -59 431 65537 4294967326 11.34 2.9 true oscar robinson 1860-11-12 20:05:55.011360936 81.04 zync studies -63 458 65537 4294967463 21.94 49.71 true fred van buren 1860-11-12 20:05:55.011477936 99.34 wind surfing -121 355 65537 4294967437 85.9 10.99 true calvin brown 1860-11-12 20:05:55.011453936 4.49 yard duty -72 267 65537 4294967460 55.2 42.89 false oscar carson 1860-11-12 20:05:55.011352936 51.91 topology -22 264 65537 4294967419 8.07 10.71 false david xylophone 1860-11-12 20:05:55.011335936 11.87 undecided -101 283 65538 4294967527 84.03 27.29 true yuri brown 1860-11-12 20:05:55.011505936 44.24 biology -109 376 65538 4294967453 59.61 35.62 true priscilla xylophone 1860-11-12 20:05:55.011485936 9.53 study skills -83 353 65538 4294967339 0.08 23.91 true holly thompson 1860-11-12 20:05:55.011356936 30.27 quiet hour -29 331 65539 4294967420 73.18 28.96 true ethan brown 1860-11-12 20:05:55.011293936 58.85 zync studies -70 430 65539 4294967534 18.89 43.84 true tom carson 1860-11-12 20:05:55.011381936 21.93 joggying -120 331 65539 4294967324 88.02 40.94 true holly nixon 1860-11-12 20:05:55.011461936 96.64 yard duty -71 298 65540 4294967510 91.63 9.6 false gabriella zipper 1860-11-12 20:05:55.011339936 68.14 geology -114 385 65541 4294967458 73.48 34.97 true oscar quirinius 1860-11-12 20:05:55.011342936 72.33 xylophone band +108 301 65536 4294967357 90.05 17.59 true ethan johnson 2013-03-01 09:11:58.703271 75.70 undecided +118 497 65536 4294967381 50.32 12.72 false david nixon 2013-03-01 09:11:58.703285 83.48 values clariffication +18 280 65536 4294967320 32.92 45.94 false holly white 2013-03-01 09:11:58.703086 58.86 topology +69 489 65536 4294967404 33.52 17.99 false oscar ichabod 2013-03-01 09:11:58.703247 32.68 topology +27 405 65536 4294967508 82.24 29.41 true oscar ovid 2013-03-01 09:11:58.703166 16.85 biology +42 495 65536 4294967431 43.57 46.81 false tom johnson 2013-03-01 09:11:58.703245 62.25 american history +NULL 409 65536 4294967490 46.97 25.92 false fred miller 2013-03-01 09:11:58.703116 33.45 history +59 431 65537 4294967326 11.34 2.9 true oscar robinson 2013-03-01 09:11:58.703161 81.04 zync studies +63 458 65537 4294967463 21.94 49.71 true fred van buren 2013-03-01 09:11:58.703278 99.34 wind surfing +121 355 65537 4294967437 85.9 10.99 true calvin brown 2013-03-01 09:11:58.703254 4.49 yard duty +72 267 65537 4294967460 55.2 42.89 false oscar carson 2013-03-01 09:11:58.703153 51.91 topology +22 264 65537 4294967419 8.07 10.71 false david xylophone 2013-03-01 09:11:58.703136 11.87 undecided +101 283 65538 4294967527 84.03 27.29 true yuri brown 2013-03-01 09:11:58.703306 44.24 biology +109 376 65538 4294967453 59.61 35.62 true priscilla xylophone 2013-03-01 09:11:58.703286 9.53 study skills +83 353 65538 4294967339 0.08 23.91 true holly thompson 2013-03-01 09:11:58.703157 30.27 quiet hour +29 331 65539 4294967420 73.18 28.96 true ethan brown 2013-03-01 09:11:58.703094 58.85 zync studies +70 430 65539 4294967534 18.89 43.84 true tom carson 2013-03-01 09:11:58.703182 21.93 joggying +120 331 65539 4294967324 88.02 40.94 true holly nixon 2013-03-01 09:11:58.703262 96.64 yard duty +71 298 65540 4294967510 91.63 9.6 false gabriella zipper 2013-03-01 09:11:58.70314 68.14 geology +114 385 65541 4294967458 73.48 34.97 true oscar quirinius 2013-03-01 09:11:58.703143 72.33 xylophone band +PREHOOK: query: SELECT SUM(HASH(*)) +FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q +PREHOOK: type: QUERY +PREHOOK: Input: default@over1korc +#### A masked pattern was here #### +POSTHOOK: query: SELECT SUM(HASH(*)) +FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1korc +#### A masked pattern was here #### +-17045922556