diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldLong.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldLong.java index f840f09..afd95bb 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldLong.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldLong.java @@ -18,12 +18,19 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; +import java.nio.charset.CharacterCodingException; import java.sql.Timestamp; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.Calendar; +import java.util.Date; +import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.io.Text; /** * Abstract class to return various fields from a Timestamp. @@ -37,6 +44,9 @@ protected int field; protected transient final Calendar calendar = Calendar.getInstance(); protected transient final Timestamp ts = new Timestamp(0); + protected final SimpleDateFormat dateFormatYMDHMS = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + protected final SimpleDateFormat dateFormatYMD = new SimpleDateFormat("yyyy-MM-dd"); + protected final SimpleDateFormat dateFormatHMS = new SimpleDateFormat("HH:mm:ss"); public VectorUDFTimestampFieldLong(int field, int colNum, int outputColumn) { this(); @@ -59,7 +69,7 @@ protected final Timestamp getTimestamp(long nanos) { */ long ms = (nanos / (1000 * 1000 * 1000)) * 1000; /* the milliseconds should be kept in nanos */ - long ns = nanos % (1000*1000*1000); + long ns = nanos % (1000 * 1000 * 1000); if (ns < 0) { /* * Due to the way java.sql.Timestamp stores sub-second values, it throws an exception @@ -75,6 +85,30 @@ protected final Timestamp getTimestamp(long nanos) { return ts; } + protected Long getField(byte[] timeBytes) { + String timeString = null; + try { + timeString = Text.decode(timeBytes); + } catch (CharacterCodingException e) { + return null; + } + Date date = null; + try { + date = dateFormatYMDHMS.parse(timeString); + } catch (ParseException e) { + try { + date = dateFormatYMD.parse(timeString); + } catch (ParseException e1) { + try { + date = dateFormatHMS.parse(timeString); + } catch (ParseException e2) { + return null; + } + } + } + return getField(date.getTime() * 1000 * 1000); + } + protected long getField(long time) { calendar.setTime(getTimestamp(time)); return calendar.get(field); @@ -82,6 +116,90 @@ protected long getField(long time) { @Override public void evaluate(VectorizedRowBatch batch) { + ColumnVector inputCol = batch.cols[this.colNum]; + if (inputCol instanceof LongColumnVector) { + evaluateLong(batch); + } + if (inputCol instanceof BytesColumnVector) { + evaluateBytes(batch); + } + } + + private void evaluateBytes(VectorizedRowBatch batch) { + LongColumnVector outV = (LongColumnVector) batch.cols[outputColumn]; + BytesColumnVector inputCol = (BytesColumnVector)batch.cols[this.colNum]; + /* every line below this is identical for evaluateLong & evaluateString */ + final int n = inputCol.isRepeating ? 1 : batch.size; + int[] sel = batch.selected; + + if(batch.size == 0) { + /* n != batch.size when isRepeating */ + return; + } + + /* true for all algebraic UDFs with no state */ + outV.isRepeating = inputCol.isRepeating; + + if (inputCol.noNulls) { + outV.noNulls = true; + if (batch.selectedInUse) { + for (int j = 0; j < n; j++) { + int i = sel[j]; + Long fieldValue = getField(inputCol.vector[i]); + if (fieldValue == null) { + outV.isNull[i] = true; + outV.noNulls = false; + } else { + outV.isNull[i] = false; + outV.vector[i] = fieldValue; + } + } + } else { + for (int i = 0; i < n; i++) { + Long fieldValue = getField(inputCol.vector[i]); + if (fieldValue == null) { + outV.isNull[i] = true; + outV.noNulls = false; + } else { + outV.isNull[i] = false; + outV.vector[i] = fieldValue; + } + } + } + } else { + // Handle case with nulls. Don't do function if the value is null, to save time, + // because calling the function can be expensive. + outV.noNulls = false; + if (batch.selectedInUse) { + for (int j = 0; j < n; j++) { + int i = sel[j]; + outV.isNull[i] = inputCol.isNull[i]; + if (!inputCol.isNull[i]) { + Long fieldValue = getField(inputCol.vector[i]); + if (fieldValue == null) { + outV.isNull[i] = true; + } else { + outV.vector[i] = fieldValue; + } + } + } + } else { + for (int i = 0; i < n; i++) { + outV.isNull[i] = inputCol.isNull[i]; + if (!inputCol.isNull[i]) { + Long fieldValue = getField(inputCol.vector[i]); + if (fieldValue == null) { + outV.isNull[i] = true; + } else { + outV.vector[i] = fieldValue; + } + } + } + } + } + } + + private void evaluateLong(VectorizedRowBatch batch) { LongColumnVector outV = (LongColumnVector) batch.cols[outputColumn]; LongColumnVector inputCol = (LongColumnVector)batch.cols[this.colNum]; /* every line below this is identical for evaluateLong & evaluateString */ @@ -167,7 +285,8 @@ public void setOutputColumn(int outputColumn) { b.setMode(VectorExpressionDescriptor.Mode.PROJECTION) .setNumArguments(1) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.LONG) + VectorExpressionDescriptor.ArgumentType.LONG, + VectorExpressionDescriptor.ArgumentType.STRING) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN); return b.build(); diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTimestampExpressions.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTimestampExpressions.java index 33a4fe5..b5107c0 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTimestampExpressions.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTimestampExpressions.java @@ -18,15 +18,21 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; +import java.nio.charset.CharacterCodingException; import java.sql.Timestamp; +import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Arrays; import java.util.Calendar; +import java.util.Date; import java.util.List; import java.util.Random; import junit.framework.Assert; import org.apache.commons.lang.ArrayUtils; +import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.TestVectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; @@ -40,12 +46,14 @@ import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; import org.junit.Test; /** * Unit tests for timestamp expressions. */ public class TestVectorTimestampExpressions { + private SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); /* copied over from VectorUDFTimestampFieldLong */ private TimestampWritable toTimestampWritable(long nanos) { @@ -97,6 +105,31 @@ private VectorizedRowBatch getVectorizedRandomRowBatchLong2(int seed, int size) return batch; } + private VectorizedRowBatch getVectorizedRandomRowBatchStringLong(int seed, int size) { + VectorizedRowBatch batch = new VectorizedRowBatch(2, size); + BytesColumnVector bcv = new BytesColumnVector(size); + Random rand = new Random(seed); + for (int i = 0; i < size; i++) { + /* all 32 bit numbers qualify & multiply up to get nano-seconds */ + bcv.vector[i] = encodeTime(1000 * 1000 * 1000 * rand.nextInt()); + } + batch.cols[0] = bcv; + batch.cols[1] = new LongColumnVector(size); + batch.size = size; + return batch; + } + + private VectorizedRowBatch getVectorizedRandomRowBatch(int seed, int size, TestType testType) { + switch (testType) { + case LONG2: + return getVectorizedRandomRowBatchLong2(seed, size); + case STRING_LONG: + return getVectorizedRandomRowBatchStringLong(seed, size); + default: + throw new IllegalArgumentException(); + } + } + /* * Input array is used to fill the entire size of the vector row batch */ @@ -112,7 +145,65 @@ private VectorizedRowBatch getVectorizedRowBatchLong2(long[] inputs, int size) { return batch; } - /*begin-macro*/ + /* + * Input array is used to fill the entire size of the vector row batch + */ + private VectorizedRowBatch getVectorizedRowBatchStringLong(long[] inputs, int size) { + VectorizedRowBatch batch = new VectorizedRowBatch(2, size); + BytesColumnVector bcv = new BytesColumnVector(size); + for (int i = 0; i < size; i++) { + bcv.vector[i] = encodeTime(inputs[i % inputs.length]); + } + batch.cols[0] = bcv; + batch.cols[1] = new LongColumnVector(size); + batch.size = size; + return batch; + } + + private VectorizedRowBatch getVectorizedRowBatch(long[] inputs, int size, TestType testType) { + switch (testType) { + case LONG2: + return getVectorizedRowBatchLong2(inputs, size); + case STRING_LONG: + return getVectorizedRowBatchStringLong(inputs, size); + default: + throw new IllegalArgumentException(); + } + } + + private byte[] encodeTime(long time) { + byte[] encoded; + try { + encoded = Text.encode(dateFormat.format(new Date(time / (1000 * 1000)))).array(); + } catch (CharacterCodingException e) { + throw new RuntimeException(e); + } + return Arrays.copyOf(encoded, encoded.length); + } + + private long decodeTime(byte[] time) { + try { + return dateFormat.parse(Text.decode(time)).getTime() * 1000 * 1000; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private long readVectorElementAt(ColumnVector col, int i) { + if (col instanceof LongColumnVector) { + return ((LongColumnVector) col).vector[i]; + } + if (col instanceof BytesColumnVector) { + byte[] timeBytes = ((BytesColumnVector) col).vector[i]; + return decodeTime(timeBytes); + } + throw new IllegalArgumentException(); + } + + private enum TestType { + LONG2, STRING_LONG + } + private void compareToUDFYearLong(long t, int y) { UDFYear udf = new UDFYear(); TimestampWritable tsw = toTimestampWritable(t); @@ -133,7 +224,7 @@ private void verifyUDFYearLong(VectorizedRowBatch batch) { if (!batch.cols[in].noNulls) { Assert.assertEquals(batch.cols[out].isNull[i], batch.cols[in].isNull[i]); } - long t = ((LongColumnVector) batch.cols[in]).vector[i]; + long t = readVectorElementAt(batch.cols[in], i); long y = ((LongColumnVector) batch.cols[out]).vector[i]; compareToUDFYearLong(t, (int) y); } else { @@ -144,38 +235,38 @@ private void verifyUDFYearLong(VectorizedRowBatch batch) { @Test public void testVectorUDFYearLong() { - VectorizedRowBatch batch = getVectorizedRowBatchLong2(new long[] {0}, - VectorizedRowBatch.DEFAULT_SIZE); - Assert.assertTrue(((LongColumnVector) batch.cols[1]).noNulls); - Assert.assertFalse(((LongColumnVector) batch.cols[1]).isRepeating); - verifyUDFYearLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); - verifyUDFYearLong(batch); - - long[] boundaries = getAllBoundaries(); - batch = getVectorizedRowBatchLong2(boundaries, boundaries.length); - verifyUDFYearLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); - verifyUDFYearLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); - verifyUDFYearLong(batch); - - batch = getVectorizedRowBatchLong2(new long[] {0}, 1); - batch.cols[0].isRepeating = true; - verifyUDFYearLong(batch); - batch.cols[0].noNulls = false; - batch.cols[0].isNull[0] = true; - verifyUDFYearLong(batch); - - batch = getVectorizedRandomRowBatchLong2(200, VectorizedRowBatch.DEFAULT_SIZE); - verifyUDFYearLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); - verifyUDFYearLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); - verifyUDFYearLong(batch); - } - /*end-macro*/ - + for (TestType testType : TestType.values()) { + VectorizedRowBatch batch = getVectorizedRowBatch(new long[] {0}, + VectorizedRowBatch.DEFAULT_SIZE, testType); + Assert.assertTrue(((LongColumnVector) batch.cols[1]).noNulls); + Assert.assertFalse(((LongColumnVector) batch.cols[1]).isRepeating); + verifyUDFYearLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); + verifyUDFYearLong(batch); + + long[] boundaries = getAllBoundaries(); + batch = getVectorizedRowBatch(boundaries, boundaries.length, testType); + verifyUDFYearLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); + verifyUDFYearLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); + verifyUDFYearLong(batch); + + batch = getVectorizedRowBatch(new long[] {0}, 1, testType); + batch.cols[0].isRepeating = true; + verifyUDFYearLong(batch); + batch.cols[0].noNulls = false; + batch.cols[0].isNull[0] = true; + verifyUDFYearLong(batch); + + batch = getVectorizedRandomRowBatch(200, VectorizedRowBatch.DEFAULT_SIZE, testType); + verifyUDFYearLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); + verifyUDFYearLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); + verifyUDFYearLong(batch); + } + } private void compareToUDFDayOfMonthLong(long t, int y) { UDFDayOfMonth udf = new UDFDayOfMonth(); @@ -197,7 +288,7 @@ private void verifyUDFDayOfMonthLong(VectorizedRowBatch batch) { if (!batch.cols[in].noNulls) { Assert.assertEquals(batch.cols[out].isNull[i], batch.cols[in].isNull[i]); } - long t = ((LongColumnVector) batch.cols[in]).vector[i]; + long t = readVectorElementAt(batch.cols[in], i); long y = ((LongColumnVector) batch.cols[out]).vector[i]; compareToUDFDayOfMonthLong(t, (int) y); } else { @@ -208,35 +299,37 @@ private void verifyUDFDayOfMonthLong(VectorizedRowBatch batch) { @Test public void testVectorUDFDayOfMonthLong() { - VectorizedRowBatch batch = getVectorizedRowBatchLong2(new long[] {0}, - VectorizedRowBatch.DEFAULT_SIZE); - Assert.assertTrue(((LongColumnVector) batch.cols[1]).noNulls); - Assert.assertFalse(((LongColumnVector) batch.cols[1]).isRepeating); - verifyUDFDayOfMonthLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); - verifyUDFDayOfMonthLong(batch); - - long[] boundaries = getAllBoundaries(); - batch = getVectorizedRowBatchLong2(boundaries, boundaries.length); - verifyUDFDayOfMonthLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); - verifyUDFDayOfMonthLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); - verifyUDFDayOfMonthLong(batch); - - batch = getVectorizedRowBatchLong2(new long[] {0}, 1); - batch.cols[0].isRepeating = true; - verifyUDFDayOfMonthLong(batch); - batch.cols[0].noNulls = false; - batch.cols[0].isNull[0] = true; - verifyUDFDayOfMonthLong(batch); - - batch = getVectorizedRandomRowBatchLong2(200, VectorizedRowBatch.DEFAULT_SIZE); - verifyUDFDayOfMonthLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); - verifyUDFDayOfMonthLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); - verifyUDFDayOfMonthLong(batch); + for (TestType testType : TestType.values()) { + VectorizedRowBatch batch = getVectorizedRowBatch(new long[] {0}, + VectorizedRowBatch.DEFAULT_SIZE, testType); + Assert.assertTrue(((LongColumnVector) batch.cols[1]).noNulls); + Assert.assertFalse(((LongColumnVector) batch.cols[1]).isRepeating); + verifyUDFDayOfMonthLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); + verifyUDFDayOfMonthLong(batch); + + long[] boundaries = getAllBoundaries(); + batch = getVectorizedRowBatch(boundaries, boundaries.length, testType); + verifyUDFDayOfMonthLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); + verifyUDFDayOfMonthLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); + verifyUDFDayOfMonthLong(batch); + + batch = getVectorizedRowBatch(new long[] {0}, 1, testType); + batch.cols[0].isRepeating = true; + verifyUDFDayOfMonthLong(batch); + batch.cols[0].noNulls = false; + batch.cols[0].isNull[0] = true; + verifyUDFDayOfMonthLong(batch); + + batch = getVectorizedRandomRowBatch(200, VectorizedRowBatch.DEFAULT_SIZE, testType); + verifyUDFDayOfMonthLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); + verifyUDFDayOfMonthLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); + verifyUDFDayOfMonthLong(batch); + } } private void compareToUDFHourLong(long t, int y) { @@ -259,7 +352,7 @@ private void verifyUDFHourLong(VectorizedRowBatch batch) { if (!batch.cols[in].noNulls) { Assert.assertEquals(batch.cols[out].isNull[i], batch.cols[in].isNull[i]); } - long t = ((LongColumnVector) batch.cols[in]).vector[i]; + long t = readVectorElementAt(batch.cols[in], i); long y = ((LongColumnVector) batch.cols[out]).vector[i]; compareToUDFHourLong(t, (int) y); } else { @@ -270,35 +363,37 @@ private void verifyUDFHourLong(VectorizedRowBatch batch) { @Test public void testVectorUDFHourLong() { - VectorizedRowBatch batch = getVectorizedRowBatchLong2(new long[] {0}, - VectorizedRowBatch.DEFAULT_SIZE); - Assert.assertTrue(((LongColumnVector) batch.cols[1]).noNulls); - Assert.assertFalse(((LongColumnVector) batch.cols[1]).isRepeating); - verifyUDFHourLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); - verifyUDFHourLong(batch); - - long[] boundaries = getAllBoundaries(); - batch = getVectorizedRowBatchLong2(boundaries, boundaries.length); - verifyUDFHourLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); - verifyUDFHourLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); - verifyUDFHourLong(batch); - - batch = getVectorizedRowBatchLong2(new long[] {0}, 1); - batch.cols[0].isRepeating = true; - verifyUDFHourLong(batch); - batch.cols[0].noNulls = false; - batch.cols[0].isNull[0] = true; - verifyUDFHourLong(batch); - - batch = getVectorizedRandomRowBatchLong2(200, VectorizedRowBatch.DEFAULT_SIZE); - verifyUDFHourLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); - verifyUDFHourLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); - verifyUDFHourLong(batch); + for (TestType testType : TestType.values()) { + VectorizedRowBatch batch = getVectorizedRowBatch(new long[] {0}, + VectorizedRowBatch.DEFAULT_SIZE, testType); + Assert.assertTrue(((LongColumnVector) batch.cols[1]).noNulls); + Assert.assertFalse(((LongColumnVector) batch.cols[1]).isRepeating); + verifyUDFHourLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); + verifyUDFHourLong(batch); + + long[] boundaries = getAllBoundaries(); + batch = getVectorizedRowBatch(boundaries, boundaries.length, testType); + verifyUDFHourLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); + verifyUDFHourLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); + verifyUDFHourLong(batch); + + batch = getVectorizedRowBatch(new long[] {0}, 1, testType); + batch.cols[0].isRepeating = true; + verifyUDFHourLong(batch); + batch.cols[0].noNulls = false; + batch.cols[0].isNull[0] = true; + verifyUDFHourLong(batch); + + batch = getVectorizedRandomRowBatch(200, VectorizedRowBatch.DEFAULT_SIZE, testType); + verifyUDFHourLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); + verifyUDFHourLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); + verifyUDFHourLong(batch); + } } private void compareToUDFMinuteLong(long t, int y) { @@ -321,7 +416,7 @@ private void verifyUDFMinuteLong(VectorizedRowBatch batch) { if (!batch.cols[in].noNulls) { Assert.assertEquals(batch.cols[out].isNull[i], batch.cols[in].isNull[i]); } - long t = ((LongColumnVector) batch.cols[in]).vector[i]; + long t = readVectorElementAt(batch.cols[in], i); long y = ((LongColumnVector) batch.cols[out]).vector[i]; compareToUDFMinuteLong(t, (int) y); } else { @@ -332,35 +427,37 @@ private void verifyUDFMinuteLong(VectorizedRowBatch batch) { @Test public void testVectorUDFMinuteLong() { - VectorizedRowBatch batch = getVectorizedRowBatchLong2(new long[] {0}, - VectorizedRowBatch.DEFAULT_SIZE); - Assert.assertTrue(((LongColumnVector) batch.cols[1]).noNulls); - Assert.assertFalse(((LongColumnVector) batch.cols[1]).isRepeating); - verifyUDFMinuteLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); - verifyUDFMinuteLong(batch); - - long[] boundaries = getAllBoundaries(); - batch = getVectorizedRowBatchLong2(boundaries, boundaries.length); - verifyUDFMinuteLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); - verifyUDFMinuteLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); - verifyUDFMinuteLong(batch); - - batch = getVectorizedRowBatchLong2(new long[] {0}, 1); - batch.cols[0].isRepeating = true; - verifyUDFMinuteLong(batch); - batch.cols[0].noNulls = false; - batch.cols[0].isNull[0] = true; - verifyUDFMinuteLong(batch); - - batch = getVectorizedRandomRowBatchLong2(200, VectorizedRowBatch.DEFAULT_SIZE); - verifyUDFMinuteLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); - verifyUDFMinuteLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); - verifyUDFMinuteLong(batch); + for (TestType testType : TestType.values()) { + VectorizedRowBatch batch = getVectorizedRowBatch(new long[] {0}, + VectorizedRowBatch.DEFAULT_SIZE, testType); + Assert.assertTrue(((LongColumnVector) batch.cols[1]).noNulls); + Assert.assertFalse(((LongColumnVector) batch.cols[1]).isRepeating); + verifyUDFMinuteLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); + verifyUDFMinuteLong(batch); + + long[] boundaries = getAllBoundaries(); + batch = getVectorizedRowBatch(boundaries, boundaries.length, testType); + verifyUDFMinuteLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); + verifyUDFMinuteLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); + verifyUDFMinuteLong(batch); + + batch = getVectorizedRowBatch(new long[] {0}, 1, testType); + batch.cols[0].isRepeating = true; + verifyUDFMinuteLong(batch); + batch.cols[0].noNulls = false; + batch.cols[0].isNull[0] = true; + verifyUDFMinuteLong(batch); + + batch = getVectorizedRandomRowBatch(200, VectorizedRowBatch.DEFAULT_SIZE, testType); + verifyUDFMinuteLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); + verifyUDFMinuteLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); + verifyUDFMinuteLong(batch); + } } private void compareToUDFMonthLong(long t, int y) { @@ -383,7 +480,7 @@ private void verifyUDFMonthLong(VectorizedRowBatch batch) { if (!batch.cols[in].noNulls) { Assert.assertEquals(batch.cols[out].isNull[i], batch.cols[in].isNull[i]); } - long t = ((LongColumnVector) batch.cols[in]).vector[i]; + long t = readVectorElementAt(batch.cols[in], i); long y = ((LongColumnVector) batch.cols[out]).vector[i]; compareToUDFMonthLong(t, (int) y); } else { @@ -394,35 +491,37 @@ private void verifyUDFMonthLong(VectorizedRowBatch batch) { @Test public void testVectorUDFMonthLong() { - VectorizedRowBatch batch = getVectorizedRowBatchLong2(new long[] {0}, - VectorizedRowBatch.DEFAULT_SIZE); - Assert.assertTrue(((LongColumnVector) batch.cols[1]).noNulls); - Assert.assertFalse(((LongColumnVector) batch.cols[1]).isRepeating); - verifyUDFMonthLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); - verifyUDFMonthLong(batch); - - long[] boundaries = getAllBoundaries(); - batch = getVectorizedRowBatchLong2(boundaries, boundaries.length); - verifyUDFMonthLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); - verifyUDFMonthLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); - verifyUDFMonthLong(batch); - - batch = getVectorizedRowBatchLong2(new long[] {0}, 1); - batch.cols[0].isRepeating = true; - verifyUDFMonthLong(batch); - batch.cols[0].noNulls = false; - batch.cols[0].isNull[0] = true; - verifyUDFMonthLong(batch); - - batch = getVectorizedRandomRowBatchLong2(200, VectorizedRowBatch.DEFAULT_SIZE); - verifyUDFMonthLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); - verifyUDFMonthLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); - verifyUDFMonthLong(batch); + for (TestType testType : TestType.values()) { + VectorizedRowBatch batch = getVectorizedRowBatch(new long[] {0}, + VectorizedRowBatch.DEFAULT_SIZE, testType); + Assert.assertTrue(((LongColumnVector) batch.cols[1]).noNulls); + Assert.assertFalse(((LongColumnVector) batch.cols[1]).isRepeating); + verifyUDFMonthLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); + verifyUDFMonthLong(batch); + + long[] boundaries = getAllBoundaries(); + batch = getVectorizedRowBatch(boundaries, boundaries.length, testType); + verifyUDFMonthLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); + verifyUDFMonthLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); + verifyUDFMonthLong(batch); + + batch = getVectorizedRowBatch(new long[] {0}, 1, testType); + batch.cols[0].isRepeating = true; + verifyUDFMonthLong(batch); + batch.cols[0].noNulls = false; + batch.cols[0].isNull[0] = true; + verifyUDFMonthLong(batch); + + batch = getVectorizedRandomRowBatch(200, VectorizedRowBatch.DEFAULT_SIZE, testType); + verifyUDFMonthLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); + verifyUDFMonthLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); + verifyUDFMonthLong(batch); + } } private void compareToUDFSecondLong(long t, int y) { @@ -445,7 +544,7 @@ private void verifyUDFSecondLong(VectorizedRowBatch batch) { if (!batch.cols[in].noNulls) { Assert.assertEquals(batch.cols[out].isNull[i], batch.cols[in].isNull[i]); } - long t = ((LongColumnVector) batch.cols[in]).vector[i]; + long t = readVectorElementAt(batch.cols[in], i); long y = ((LongColumnVector) batch.cols[out]).vector[i]; compareToUDFSecondLong(t, (int) y); } else { @@ -456,35 +555,38 @@ private void verifyUDFSecondLong(VectorizedRowBatch batch) { @Test public void testVectorUDFSecondLong() { - VectorizedRowBatch batch = getVectorizedRowBatchLong2(new long[] {0}, - VectorizedRowBatch.DEFAULT_SIZE); - Assert.assertTrue(((LongColumnVector) batch.cols[1]).noNulls); - Assert.assertFalse(((LongColumnVector) batch.cols[1]).isRepeating); - verifyUDFSecondLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); - verifyUDFSecondLong(batch); - - long[] boundaries = getAllBoundaries(); - batch = getVectorizedRowBatchLong2(boundaries, boundaries.length); - verifyUDFSecondLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); - verifyUDFSecondLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); - verifyUDFSecondLong(batch); - - batch = getVectorizedRowBatchLong2(new long[] {0}, 1); - batch.cols[0].isRepeating = true; - verifyUDFSecondLong(batch); - batch.cols[0].noNulls = false; - batch.cols[0].isNull[0] = true; - verifyUDFSecondLong(batch); - - batch = getVectorizedRandomRowBatchLong2(200, VectorizedRowBatch.DEFAULT_SIZE); - verifyUDFSecondLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); - verifyUDFSecondLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); - verifyUDFSecondLong(batch); + for (TestType testType : TestType.values()) { + + VectorizedRowBatch batch = getVectorizedRowBatch(new long[] {0}, + VectorizedRowBatch.DEFAULT_SIZE, testType); + Assert.assertTrue(((LongColumnVector) batch.cols[1]).noNulls); + Assert.assertFalse(((LongColumnVector) batch.cols[1]).isRepeating); + verifyUDFSecondLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); + verifyUDFSecondLong(batch); + + long[] boundaries = getAllBoundaries(); + batch = getVectorizedRowBatch(boundaries, boundaries.length, testType); + verifyUDFSecondLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); + verifyUDFSecondLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); + verifyUDFSecondLong(batch); + + batch = getVectorizedRowBatch(new long[] {0}, 1, testType); + batch.cols[0].isRepeating = true; + verifyUDFSecondLong(batch); + batch.cols[0].noNulls = false; + batch.cols[0].isNull[0] = true; + verifyUDFSecondLong(batch); + + batch = getVectorizedRandomRowBatch(200, VectorizedRowBatch.DEFAULT_SIZE, testType); + verifyUDFSecondLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); + verifyUDFSecondLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); + verifyUDFSecondLong(batch); + } } private LongWritable getLongWritable(TimestampWritable i) { @@ -521,7 +623,7 @@ private void verifyUDFUnixTimeStampLong(VectorizedRowBatch batch) { if (!batch.cols[in].noNulls) { Assert.assertEquals(batch.cols[out].isNull[i], batch.cols[in].isNull[i]); } - long t = ((LongColumnVector) batch.cols[in]).vector[i]; + long t = readVectorElementAt(batch.cols[in], i); long y = ((LongColumnVector) batch.cols[out]).vector[i]; compareToUDFUnixTimeStampLong(t, y); } else { @@ -532,35 +634,37 @@ private void verifyUDFUnixTimeStampLong(VectorizedRowBatch batch) { @Test public void testVectorUDFUnixTimeStampLong() { - VectorizedRowBatch batch = getVectorizedRowBatchLong2(new long[] {0}, - VectorizedRowBatch.DEFAULT_SIZE); - Assert.assertTrue(((LongColumnVector) batch.cols[1]).noNulls); - Assert.assertFalse(((LongColumnVector) batch.cols[1]).isRepeating); - verifyUDFUnixTimeStampLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); - verifyUDFUnixTimeStampLong(batch); - - long[] boundaries = getAllBoundaries(); - batch = getVectorizedRowBatchLong2(boundaries, boundaries.length); - verifyUDFUnixTimeStampLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); - verifyUDFUnixTimeStampLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); - verifyUDFUnixTimeStampLong(batch); - - batch = getVectorizedRowBatchLong2(new long[] {0}, 1); - batch.cols[0].isRepeating = true; - verifyUDFUnixTimeStampLong(batch); - batch.cols[0].noNulls = false; - batch.cols[0].isNull[0] = true; - verifyUDFUnixTimeStampLong(batch); - - batch = getVectorizedRandomRowBatchLong2(200, VectorizedRowBatch.DEFAULT_SIZE); - verifyUDFUnixTimeStampLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); - verifyUDFUnixTimeStampLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); - verifyUDFUnixTimeStampLong(batch); + for (TestType testType : TestType.values()) { + VectorizedRowBatch batch = getVectorizedRowBatch(new long[] {0}, + VectorizedRowBatch.DEFAULT_SIZE, testType); + Assert.assertTrue(((LongColumnVector) batch.cols[1]).noNulls); + Assert.assertFalse(((LongColumnVector) batch.cols[1]).isRepeating); + verifyUDFUnixTimeStampLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); + verifyUDFUnixTimeStampLong(batch); + + long[] boundaries = getAllBoundaries(); + batch = getVectorizedRowBatch(boundaries, boundaries.length, testType); + verifyUDFUnixTimeStampLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); + verifyUDFUnixTimeStampLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); + verifyUDFUnixTimeStampLong(batch); + + batch = getVectorizedRowBatch(new long[] {0}, 1, testType); + batch.cols[0].isRepeating = true; + verifyUDFUnixTimeStampLong(batch); + batch.cols[0].noNulls = false; + batch.cols[0].isNull[0] = true; + verifyUDFUnixTimeStampLong(batch); + + batch = getVectorizedRandomRowBatch(200, VectorizedRowBatch.DEFAULT_SIZE, testType); + verifyUDFUnixTimeStampLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); + verifyUDFUnixTimeStampLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); + verifyUDFUnixTimeStampLong(batch); + } } private void compareToUDFWeekOfYearLong(long t, int y) { @@ -583,7 +687,7 @@ private void verifyUDFWeekOfYearLong(VectorizedRowBatch batch) { if (!batch.cols[in].noNulls) { Assert.assertEquals(batch.cols[out].isNull[i], batch.cols[in].isNull[i]); } - long t = ((LongColumnVector) batch.cols[in]).vector[i]; + long t = readVectorElementAt(batch.cols[in], i); long y = ((LongColumnVector) batch.cols[out]).vector[i]; compareToUDFWeekOfYearLong(t, (int) y); } else { @@ -594,35 +698,37 @@ private void verifyUDFWeekOfYearLong(VectorizedRowBatch batch) { @Test public void testVectorUDFWeekOfYearLong() { - VectorizedRowBatch batch = getVectorizedRowBatchLong2(new long[] {0}, - VectorizedRowBatch.DEFAULT_SIZE); - Assert.assertTrue(((LongColumnVector) batch.cols[1]).noNulls); - Assert.assertFalse(((LongColumnVector) batch.cols[1]).isRepeating); - verifyUDFWeekOfYearLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); - verifyUDFWeekOfYearLong(batch); - - long[] boundaries = getAllBoundaries(); - batch = getVectorizedRowBatchLong2(boundaries, boundaries.length); - verifyUDFWeekOfYearLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); - verifyUDFWeekOfYearLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); - verifyUDFWeekOfYearLong(batch); - - batch = getVectorizedRowBatchLong2(new long[] {0}, 1); - batch.cols[0].isRepeating = true; - verifyUDFWeekOfYearLong(batch); - batch.cols[0].noNulls = false; - batch.cols[0].isNull[0] = true; - verifyUDFWeekOfYearLong(batch); - - batch = getVectorizedRandomRowBatchLong2(200, VectorizedRowBatch.DEFAULT_SIZE); - verifyUDFWeekOfYearLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); - verifyUDFWeekOfYearLong(batch); - TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); - verifyUDFWeekOfYearLong(batch); + for (TestType testType : TestType.values()) { + VectorizedRowBatch batch = getVectorizedRowBatch(new long[] {0}, + VectorizedRowBatch.DEFAULT_SIZE, testType); + Assert.assertTrue(((LongColumnVector) batch.cols[1]).noNulls); + Assert.assertFalse(((LongColumnVector) batch.cols[1]).isRepeating); + verifyUDFWeekOfYearLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); + verifyUDFWeekOfYearLong(batch); + + long[] boundaries = getAllBoundaries(); + batch = getVectorizedRowBatch(boundaries, boundaries.length, testType); + verifyUDFWeekOfYearLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); + verifyUDFWeekOfYearLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); + verifyUDFWeekOfYearLong(batch); + + batch = getVectorizedRowBatch(new long[] {0}, 1, testType); + batch.cols[0].isRepeating = true; + verifyUDFWeekOfYearLong(batch); + batch.cols[0].noNulls = false; + batch.cols[0].isNull[0] = true; + verifyUDFWeekOfYearLong(batch); + + batch = getVectorizedRandomRowBatch(200, VectorizedRowBatch.DEFAULT_SIZE, testType); + verifyUDFWeekOfYearLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[0]); + verifyUDFWeekOfYearLong(batch); + TestVectorizedRowBatch.addRandomNulls(batch.cols[1]); + verifyUDFWeekOfYearLong(batch); + } } public static void main(String[] args) {