diff --git a/storage-api/pom.xml b/storage-api/pom.xml index 80fa22ce63..d768f3f9bd 100644 --- a/storage-api/pom.xml +++ b/storage-api/pom.xml @@ -158,8 +158,8 @@ maven-compiler-plugin 3.1 - 1.7 - 1.7 + 1.8 + 1.8 diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java index 01bd2f6e88..b22fb4d3df 100644 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java +++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java @@ -242,12 +242,23 @@ public void ensureSize(int size, boolean preserveData) { /** * Print the value for this column into the given string builder. + * Considers timestamp values in current system time zone. * @param buffer the buffer to print into * @param row the id of the row to print */ public abstract void stringifyValue(StringBuilder buffer, int row); + /** + * Print the value for this column into the given string builder. + * Considers timestamp values in UTC. + * @param buffer the buffer to print into + * @param row the id of the row to print + */ + public void stringifyValue2(StringBuilder buffer, int row) { + stringifyValue(buffer, row); + } + /** * Shallow copy of the contents of this vector to the other vector; * replaces other vector's values. diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ListColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ListColumnVector.java index 8cbcc029a5..8073f9bf76 100644 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ListColumnVector.java +++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ListColumnVector.java @@ -135,6 +135,28 @@ public void stringifyValue(StringBuilder buffer, int row) { } } + @Override + public void stringifyValue2(StringBuilder buffer, int row) { + if (isRepeating) { + row = 0; + } + if (noNulls || !isNull[row]) { + buffer.append('['); + boolean isFirst = true; + for(long i=offsets[row]; i < offsets[row] + lengths[row]; ++i) { + if (isFirst) { + isFirst = false; + } else { + buffer.append(", "); + } + child.stringifyValue2(buffer, (int) i); + } + buffer.append(']'); + } else { + buffer.append("null"); + } + } + @Override public void init() { super.init(); diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/MapColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/MapColumnVector.java index 3143a44ec8..28b2a57376 100644 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/MapColumnVector.java +++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/MapColumnVector.java @@ -147,6 +147,32 @@ public void stringifyValue(StringBuilder buffer, int row) { } } + @Override + public void stringifyValue2(StringBuilder buffer, int row) { + if (isRepeating) { + row = 0; + } + if (noNulls || !isNull[row]) { + buffer.append('['); + boolean isFirst = true; + for(long i=offsets[row]; i < offsets[row] + lengths[row]; ++i) { + if (isFirst) { + isFirst = false; + } else { + buffer.append(", "); + } + buffer.append("{\"key\": "); + keys.stringifyValue2(buffer, (int) i); + buffer.append(", \"value\": "); + values.stringifyValue2(buffer, (int) i); + buffer.append('}'); + } + buffer.append(']'); + } else { + buffer.append("null"); + } + } + @Override public void init() { super.init(); diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/StructColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/StructColumnVector.java index 70d6ab4b99..4806e19fc8 100644 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/StructColumnVector.java +++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/StructColumnVector.java @@ -129,6 +129,25 @@ public void stringifyValue(StringBuilder buffer, int row) { } } + @Override + public void stringifyValue2(StringBuilder buffer, int row) { + if (isRepeating) { + row = 0; + } + if (noNulls || !isNull[row]) { + buffer.append('['); + for(int i=0; i < fields.length; ++i) { + if (i != 0) { + buffer.append(", "); + } + fields[i].stringifyValue2(buffer, row); + } + buffer.append(']'); + } else { + buffer.append("null"); + } + } + @Override public void ensureSize(int size, boolean preserveData) { super.ensureSize(size, preserveData); diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java index a6f536933e..269ac210d5 100644 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java +++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java @@ -18,6 +18,11 @@ package org.apache.hadoop.hive.ql.exec.vector; import java.sql.Timestamp; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.chrono.IsoChronology; import java.util.Arrays; import org.apache.hadoop.io.Writable; @@ -491,6 +496,22 @@ public void stringifyValue(StringBuilder buffer, int row) { } } + @Override + public void stringifyValue2(StringBuilder buffer, int row) { + if (isRepeating) { + row = 0; + } + if (noNulls || !isNull[row]) { + scratchTimestamp.setTime(time[row]); + scratchTimestamp.setNanos(nanos[row]); + LocalDateTime ts = + LocalDateTime.ofInstant(Instant.ofEpochMilli(time[row]), ZoneOffset.UTC).withNano(nanos[row]); + buffer.append(ts.toLocalDate().toString() + ' ' + ts.toLocalTime().toString()); + } else { + buffer.append("null"); + } + } + @Override public void ensureSize(int size, boolean preserveData) { super.ensureSize(size, preserveData); diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/UnionColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/UnionColumnVector.java index 09d519dd48..ed62c9aa05 100644 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/UnionColumnVector.java +++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/UnionColumnVector.java @@ -129,6 +129,22 @@ public void stringifyValue(StringBuilder buffer, int row) { } } + @Override + public void stringifyValue2(StringBuilder buffer, int row) { + if (isRepeating) { + row = 0; + } + if (noNulls || !isNull[row]) { + buffer.append("{\"tag\": "); + buffer.append(tags[row]); + buffer.append(", \"value\": "); + fields[tags[row]].stringifyValue2(buffer, row); + buffer.append('}'); + } else { + buffer.append("null"); + } + } + @Override public void ensureSize(int size, boolean preserveData) { super.ensureSize(size, preserveData); diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java index bebf7691f7..8dd9945fdf 100644 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java +++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java @@ -180,6 +180,50 @@ public String stringifyColumn(int columnNum) { return b.toString(); } + public String stringifyColumn2(int columnNum) { + if (size == 0) { + return ""; + } + StringBuilder b = new StringBuilder(); + b.append("columnNum "); + b.append(columnNum); + b.append(", size "); + b.append(size); + b.append(", selectedInUse "); + b.append(selectedInUse); + ColumnVector colVector = cols[columnNum]; + b.append(", noNulls "); + b.append(colVector.noNulls); + b.append(", isRepeating "); + b.append(colVector.isRepeating); + b.append('\n'); + + final boolean noNulls = colVector.noNulls; + final boolean[] isNull = colVector.isNull; + if (colVector.isRepeating) { + final boolean hasRepeatedValue = (noNulls || !isNull[0]); + for (int i = 0; i < size; i++) { + if (hasRepeatedValue) { + colVector.stringifyValue2(b, 0); + } else { + b.append("NULL"); + } + b.append('\n'); + } + } else { + for (int i = 0; i < size; i++) { + final int batchIndex = (selectedInUse ? selected[i] : i); + if (noNulls || !isNull[batchIndex]) { + colVector.stringifyValue2(b, batchIndex); + } else { + b.append("NULL"); + } + b.append('\n'); + } + } + return b.toString(); + } + public String stringify(String prefix) { if (size == 0) { return ""; @@ -274,6 +318,100 @@ public String stringify(String prefix) { return b.toString(); } + public String stringify2(String prefix) { + if (size == 0) { + return ""; + } + StringBuilder b = new StringBuilder(); + b.append(prefix); + b.append("Column vector types: "); + for (int k = 0; k < projectionSize; k++) { + int projIndex = projectedColumns[k]; + ColumnVector cv = cols[projIndex]; + if (k > 0) { + b.append(", "); + } + b.append(projIndex); + b.append(":"); + String colVectorType = null; + if (cv instanceof LongColumnVector) { + colVectorType = "LONG"; + } else if (cv instanceof DoubleColumnVector) { + colVectorType = "DOUBLE"; + } else if (cv instanceof BytesColumnVector) { + colVectorType = "BYTES"; + } else if (cv instanceof DecimalColumnVector) { + colVectorType = "DECIMAL"; + } else if (cv instanceof TimestampColumnVector) { + colVectorType = "TIMESTAMP"; + } else if (cv instanceof IntervalDayTimeColumnVector) { + colVectorType = "INTERVAL_DAY_TIME"; + } else if (cv instanceof ListColumnVector) { + colVectorType = "LIST"; + } else if (cv instanceof MapColumnVector) { + colVectorType = "MAP"; + } else if (cv instanceof StructColumnVector) { + colVectorType = "STRUCT"; + } else if (cv instanceof UnionColumnVector) { + colVectorType = "UNION"; + } else { + colVectorType = "Unknown"; + } + b.append(colVectorType); + } + b.append('\n'); + + if (this.selectedInUse) { + for (int j = 0; j < size; j++) { + int i = selected[j]; + b.append('['); + for (int k = 0; k < projectionSize; k++) { + int projIndex = projectedColumns[k]; + ColumnVector cv = cols[projIndex]; + if (k > 0) { + b.append(", "); + } + if (cv != null) { + try { + cv.stringifyValue2(b, i); + } catch (Exception ex) { + b.append(""); + } + } + } + b.append(']'); + if (j < size - 1) { + b.append('\n'); + b.append(prefix); + } + } + } else { + for (int i = 0; i < size; i++) { + b.append('['); + for (int k = 0; k < projectionSize; k++) { + int projIndex = projectedColumns[k]; + ColumnVector cv = cols[projIndex]; + if (k > 0) { + b.append(", "); + } + if (cv != null) { + try { + cv.stringifyValue2(b, i); + } catch (Exception ex) { + b.append(""); + } + } + } + b.append(']'); + if (i < size - 1) { + b.append('\n'); + b.append(prefix); + } + } + } + return b.toString(); + } + @Override public String toString() { return stringify(""); diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/util/TimestampUtils.java b/storage-api/src/java/org/apache/hadoop/hive/ql/util/TimestampUtils.java index a087a4d493..367b932613 100644 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/util/TimestampUtils.java +++ b/storage-api/src/java/org/apache/hadoop/hive/ql/util/TimestampUtils.java @@ -26,7 +26,7 @@ import java.sql.Timestamp; /** - * Utitilities for Timestamps and the relevant conversions. + * Utilities for Timestamps and the relevant conversions. */ public class TimestampUtils { public static final BigDecimal BILLION_BIG_DECIMAL = BigDecimal.valueOf(1000000000); diff --git a/storage-api/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java b/storage-api/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java index a22a10bfd4..0c0e785ed5 100644 --- a/storage-api/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java +++ b/storage-api/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java @@ -30,7 +30,6 @@ import org.apache.hadoop.hive.serde2.io.HiveDecimalWritableV1; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; -import org.apache.hadoop.hive.common.type.RandomTypeUtil; import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr; import org.apache.hadoop.hive.ql.util.TimestampUtils; diff --git a/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java b/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java index 6ffd6d1d81..615f21e085 100644 --- a/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java +++ b/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java @@ -132,4 +132,37 @@ public void testStringify() throws IOException { "[[27, 2000-01-01 00:00:10.0], \"value 9\"]"); assertEquals(EXPECTED, batch.toString()); } + + @Test + public void testStringify2() throws IOException { + VectorizedRowBatch batch = new VectorizedRowBatch(2); + LongColumnVector x1 = new LongColumnVector(); + TimestampColumnVector x2 = new TimestampColumnVector(); + StructColumnVector x = new StructColumnVector(1024, x1, x2); + BytesColumnVector y = new BytesColumnVector(); + batch.cols[0] = x; + batch.cols[1] = y; + batch.reset(); + Timestamp ts = new Timestamp(946684800000L); + for(int r=0; r < 10; ++r) { + batch.size += 1; + x1.vector[r] = 3 * r; + ts.setTime(ts.getTime() + 1000); + x2.set(r, ts); + byte[] buffer = ("value " + r).getBytes(StandardCharsets.UTF_8); + y.setRef(r, buffer, 0, buffer.length); + } + final String EXPECTED = ("Column vector types: 0:STRUCT, 1:BYTES\n" + + "[[0, 2000-01-01 00:00:01], \"value 0\"]\n" + + "[[3, 2000-01-01 00:00:02], \"value 1\"]\n" + + "[[6, 2000-01-01 00:00:03], \"value 2\"]\n" + + "[[9, 2000-01-01 00:00:04], \"value 3\"]\n" + + "[[12, 2000-01-01 00:00:05], \"value 4\"]\n" + + "[[15, 2000-01-01 00:00:06], \"value 5\"]\n" + + "[[18, 2000-01-01 00:00:07], \"value 6\"]\n" + + "[[21, 2000-01-01 00:00:08], \"value 7\"]\n" + + "[[24, 2000-01-01 00:00:09], \"value 8\"]\n" + + "[[27, 2000-01-01 00:00:10], \"value 9\"]"); + assertEquals(EXPECTED, batch.stringify2("")); + } }