diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java index 3bfe4f9..7758ac4 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java @@ -21,7 +21,9 @@ import java.util.Arrays; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.ColumnVector.Type; import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; /** * Class to keep information on a set of typed vector columns. Used by @@ -75,6 +77,7 @@ // Given the keyIndex these arrays return: // The ColumnVector.Type, // The type specific index into longIndices, doubleIndices, etc... + protected TypeInfo[] typeInfos; protected ColumnVector.Type[] columnVectorTypes; protected int[] columnTypeSpecificIndices; @@ -96,13 +99,15 @@ protected VectorColumnSetInfo(int keyCount) { intervalDayTimeIndices = new int[this.keyCount]; addIntervalDayTimeIndex = 0; + typeInfos = new TypeInfo[this.keyCount]; columnVectorTypes = new ColumnVector.Type[this.keyCount]; columnTypeSpecificIndices = new int[this.keyCount]; } - protected void addKey(ColumnVector.Type columnVectorType) throws HiveException { + protected void addKey(TypeInfo typeInfo) throws HiveException { + Type columnVectorType = VectorizationContext.getColumnVectorTypeFromTypeInfo(typeInfo); switch (columnVectorType) { case LONG: case DECIMAL_64: @@ -133,6 +138,7 @@ protected void addKey(ColumnVector.Type columnVectorType) throws HiveException { throw new HiveException("Unexpected column vector type " + columnVectorType); } + typeInfos[addKeyIndex] = typeInfo; columnVectorTypes[addKeyIndex] = columnVectorType; addKeyIndex++; } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java index 6ae6727..82dc4a7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java @@ -49,8 +49,7 @@ void init(VectorExpression[] keyExpressions) throws HiveException { VectorExpression keyExpression = keyExpressions[i]; TypeInfo typeInfo = keyExpression.getOutputTypeInfo(); - Type columnVectorType = VectorizationContext.getColumnVectorTypeFromTypeInfo(typeInfo); - addKey(columnVectorType); + addKey(typeInfo); // The output of the key expression is the input column. final int inputColumnNum = keyExpression.getOutputColumnNum(); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java index eb870a7..1f46f2c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java @@ -20,6 +20,7 @@ import org.apache.hive.common.util.Murmur3; +import java.sql.Date; import java.sql.Timestamp; import java.util.Arrays; @@ -29,8 +30,11 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.util.JavaDataModel; +import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import com.google.common.base.Preconditions; @@ -350,7 +354,10 @@ public void assignNullDecimal(int keyIndex, int index) { } public void assignTimestamp(int index, Timestamp value) { - timestampValues[index] = value; + // Do not assign the input value object to the timestampValues array element. + // Always copy value using set* methods. + timestampValues[index].setTime(value.getTime()); + timestampValues[index].setNanos(value.getNanos()); } public void assignTimestamp(int index, TimestampColumnVector colVector, int elementNum) { @@ -359,7 +366,9 @@ public void assignTimestamp(int index, TimestampColumnVector colVector, int elem public void assignNullTimestamp(int keyIndex, int index) { isNull[keyIndex] = true; - timestampValues[index] = ZERO_TIMESTAMP; // assign 0 to simplify hashcode + // assign 0 to simplify hashcode + timestampValues[index].setTime(ZERO_TIMESTAMP.getTime()); + timestampValues[index].setNanos(ZERO_TIMESTAMP.getNanos()); } public void assignIntervalDayTime(int index, HiveIntervalDayTime value) { @@ -372,7 +381,162 @@ public void assignIntervalDayTime(int index, IntervalDayTimeColumnVector colVect public void assignNullIntervalDayTime(int keyIndex, int index) { isNull[keyIndex] = true; - intervalDayTimeValues[index] = ZERO_INTERVALDAYTIME; // assign 0 to simplify hashcode + intervalDayTimeValues[index].set(ZERO_INTERVALDAYTIME); // assign 0 to simplify hashcode + } + + /* + * This method is mainly intended for debug display purposes. + */ + public String stringifyKeys(VectorColumnSetInfo columnSetInfo) + { + StringBuilder sb = new StringBuilder(); + boolean isFirstKey = true; + + if (longValues.length > 0) { + isFirstKey = false; + sb.append("longs "); + boolean isFirstValue = true; + for (int i = 0; i < columnSetInfo.longIndices.length; i++) { + if (isFirstValue) { + isFirstValue = false; + } else { + sb.append(", "); + } + int keyIndex = columnSetInfo.longIndices[i]; + if (isNull[keyIndex]) { + sb.append("null"); + } else { + sb.append(longValues[i]); + PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) columnSetInfo.typeInfos[keyIndex]; + // FUTURE: Add INTERVAL_YEAR_MONTH, etc, as desired. + switch (primitiveTypeInfo.getPrimitiveCategory()) { + case DATE: + { + Date dt = new Date(0); + dt.setTime(DateWritable.daysToMillis((int) longValues[i])); + sb.append(" date "); + sb.append(dt.toString()); + } + break; + default: + // Add nothing more. + break; + } + } + } + } + if (doubleValues.length > 0) { + if (isFirstKey) { + isFirstKey = false; + } else { + sb.append(", "); + } + sb.append("doubles "); + boolean isFirstValue = true; + for (int i = 0; i < columnSetInfo.doubleIndices.length; i++) { + if (isFirstValue) { + isFirstValue = false; + } else { + sb.append(", "); + } + int keyIndex = columnSetInfo.doubleIndices[i]; + if (isNull[keyIndex]) { + sb.append("null"); + } else { + sb.append(doubleValues[i]); + } + } + } + if (byteValues.length > 0) { + if (isFirstKey) { + isFirstKey = false; + } else { + sb.append(", "); + } + sb.append("byte lengths "); + boolean isFirstValue = true; + for (int i = 0; i < columnSetInfo.stringIndices.length; i++) { + if (isFirstValue) { + isFirstValue = false; + } else { + sb.append(", "); + } + int keyIndex = columnSetInfo.stringIndices[i]; + if (isNull[keyIndex]) { + sb.append("null"); + } else { + sb.append(byteLengths[i]); + } + } + } + if (decimalValues.length > 0) { + if (isFirstKey) { + isFirstKey = true; + } else { + sb.append(", "); + } + sb.append("decimals "); + boolean isFirstValue = true; + for (int i = 0; i < columnSetInfo.decimalIndices.length; i++) { + if (isFirstValue) { + isFirstValue = false; + } else { + sb.append(", "); + } + int keyIndex = columnSetInfo.decimalIndices[i]; + if (isNull[keyIndex]) { + sb.append("null"); + } else { + sb.append(decimalValues[i]); + } + } + } + if (timestampValues.length > 0) { + if (isFirstKey) { + isFirstKey = false; + } else { + sb.append(", "); + } + sb.append("timestamps "); + boolean isFirstValue = true; + for (int i = 0; i < columnSetInfo.timestampIndices.length; i++) { + if (isFirstValue) { + isFirstValue = false; + } else { + sb.append(", "); + } + int keyIndex = columnSetInfo.timestampIndices[i]; + if (isNull[keyIndex]) { + sb.append("null"); + } else { + sb.append(timestampValues[i]); + } + } + } + if (intervalDayTimeValues.length > 0) { + if (isFirstKey) { + isFirstKey = false; + } else { + sb.append(", "); + } + sb.append("interval day times "); + boolean isFirstValue = true; + for (int i = 0; i < columnSetInfo.intervalDayTimeIndices.length; i++) { + if (isFirstValue) { + isFirstValue = false; + } else { + sb.append(", "); + } + int keyIndex = columnSetInfo.intervalDayTimeIndices[i]; + if (isNull[keyIndex]) { + sb.append("null"); + } else { + sb.append(intervalDayTimeValues[i]); + } + } + } + + return sb.toString(); } @Override diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java index 2b401ac..0e6f8c5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.util.JavaDataModel; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector.Type; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; /** * Class for handling vectorized hash map key wrappers. It evaluates the key columns in a @@ -106,133 +107,48 @@ public void evaluateBatch(VectorizedRowBatch batch) throws HiveException { keyIndex = longIndices[i]; columnIndex = keyExpressions[keyIndex].getOutputColumnNum(); LongColumnVector columnVector = (LongColumnVector) batch.cols[columnIndex]; - if (columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) { - assignLongNoNullsNoRepeatingNoSelection(i, batch.size, columnVector); - } else if (columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) { - assignLongNoNullsNoRepeatingSelection(i, batch.size, columnVector, batch.selected); - } else if ((columnVector.noNulls || !columnVector.isNull[0]) && columnVector.isRepeating) { - assignLongNoNullsRepeating(i, batch.size, columnVector); - } else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) { - assignLongNullsNoRepeatingNoSelection(keyIndex, i, batch.size, columnVector); - } else if (!columnVector.noNulls && columnVector.isRepeating) { - assignLongNullsRepeating(keyIndex, i, batch.size, columnVector); - } else if (!columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) { - assignLongNullsNoRepeatingSelection (keyIndex, i, batch.size, columnVector, batch.selected); - } else { - throw new HiveException (String.format( - "Unimplemented Long null/repeat/selected combination %b/%b/%b", - columnVector.noNulls, columnVector.isRepeating, batch.selectedInUse)); - } + + evaluateLongColumnVector(batch, columnVector, keyIndex, i); } + for(int i=0;i] Filter Operator - Filter Vectorization: - className: VectorFilterOperator - native: true - predicateExpression: FilterExprAndExpr(children: SelectColumnIsNotNull(col 0:decimal(8,1)), SelectColumnIsNotNull(col 1:int)) - predicate: (decimal0801_col is not null and int_col_1 is not null) (type: boolean) - Statistics: Num rows: 4 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE + predicate: decimal0801_col is not null (type: boolean) + Statistics: Num rows: 5 Data size: 580 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: decimal0801_col (type: decimal(8,1)), int_col_1 (type: int) outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [0, 1] - Statistics: Num rows: 4 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 580 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: - Inner Join 0 to 1 + Left Outer Join 0 to 1 keys: - 0 _col1 (type: int) - 1 _col0 (type: int) - Map Join Vectorization: - bigTableKeyColumnNums: [1] - bigTableRetainedColumnNums: [0] - bigTableValueColumnNums: [0] - className: VectorMapJoinInnerBigOnlyLongOperator - native: true - nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Optimized Table and Supports Key Types IS true - projectedOutputColumnNums: [0] - outputColumnNames: _col0 + 0 + 1 + outputColumnNames: _col0, _col1, _col2 input vertices: 1 Reducer 3 - Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - File Sink Vectorization: - className: VectorFileSinkOperator - native: false - Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized, llap + Statistics: Num rows: 5 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (_col2) IN (_col1) (type: boolean) + Statistics: Num rows: 2 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: decimal(8,1)) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap LLAP IO: all inputs Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - rowBatchContext: - dataColumnCount: 2 - includeColumns: [0, 1] - dataColumns: decimal0801_col:decimal(8,1), int_col_1:int - partitionColumnCount: 0 - scratchColumnTypeNames: [] + notVectorizedReason: FILTER operator: Vectorizing IN expression only supported for constant values + vectorized: false Map 2 Map Operator Tree: TableScan @@ -266,24 +244,16 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - Filter Vectorization: - className: VectorFilterOperator + Reduce Output Operator + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + keyColumnNums: [] native: true - predicateExpression: SelectColumnIsNotNull(col 0:int) - predicate: _col0 is not null (type: boolean) + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumnNums: [0] Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Reduce Sink Vectorization: - className: VectorReduceSinkLongOperator - keyColumnNums: [0] - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumnNums: [] - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/parquet_vectorization_15.q.out ql/src/test/results/clientpositive/parquet_vectorization_15.q.out index 71ed777..6ada288 100644 --- ql/src/test/results/clientpositive/parquet_vectorization_15.q.out +++ ql/src/test/results/clientpositive/parquet_vectorization_15.q.out @@ -288,15 +288,15 @@ POSTHOOK: Input: default@alltypesparquet -51.0 true NULL QiOcvR0kt6r7f0R7fiPxQTCU -51 266531954 1969-12-31 16:00:08.451 NULL -266531980.28 NULL NULL 33.0 NULL 0.0 NULL 51 NULL 2.66532E8 -23 266531980.28 0.0 -51.0 true NULL Ybpj38RTTYl7CnJXPNx1g4C -51 -370919370 1969-12-31 16:00:08.451 NULL 370919343.72 NULL NULL 33.0 NULL 0.0 NULL 51 NULL -3.70919296E8 -23 -370919343.72 0.0 -6.0 NULL -200.0 NULL -6 NULL 1969-12-31 15:59:56.094 NULL NULL -200.0 -15910.599999999999 3.0 NULL 0.0 -23.0 6 NULL NULL -5 NULL NULL --62.0 NULL 15601.0 NULL -62 NULL 1969-12-31 15:59:56.527 NULL NULL 15601.0 1241106.353 33.0 NULL 0.0 -23.0 62 NULL NULL -23 NULL NULL +-62.0 NULL 15601.0 NULL -62 NULL 1969-12-31 16:00:09.889 NULL NULL 15601.0 1241106.353 33.0 NULL 0.0 -23.0 62 NULL NULL -23 NULL NULL 11.0 false NULL 10pO8p1LNx4Y 11 271296824 1969-12-31 16:00:02.351 NULL -271296850.28 NULL NULL 0.0 NULL 0.0 NULL -11 NULL 2.71296832E8 -1 271296850.28 0.0 11.0 false NULL 1H6wGP 11 -560827082 1969-12-31 16:00:02.351 NULL 560827055.72 NULL NULL 0.0 NULL 0.0 NULL -11 NULL -5.6082707E8 -1 -560827055.72 0.0 11.0 false NULL 2a7V63IL7jK3o 11 -325931647 1969-12-31 16:00:02.351 NULL 325931620.72 NULL NULL 0.0 NULL 0.0 NULL -11 NULL -3.25931648E8 -1 -325931620.72 0.0 11.0 true NULL 10 11 92365813 1969-12-31 16:00:02.351 NULL -92365839.28 NULL NULL 0.0 NULL 0.0 NULL -11 NULL 9.2365808E7 -1 92365839.28 0.0 -21.0 NULL 15601.0 NULL 21 NULL 1969-12-31 15:59:56.527 NULL NULL 15601.0 1241106.353 12.0 NULL 0.0 -23.0 -21 NULL NULL -2 NULL NULL +21.0 NULL 15601.0 NULL 21 NULL 1969-12-31 16:00:14.256 NULL NULL 15601.0 1241106.353 12.0 NULL 0.0 -23.0 -21 NULL NULL -2 NULL NULL 32.0 NULL -200.0 NULL 32 NULL 1969-12-31 16:00:02.445 NULL NULL -200.0 -15910.599999999999 1.0 NULL 0.0 -23.0 -32 NULL NULL -23 NULL NULL 36.0 NULL -200.0 NULL 36 NULL 1969-12-31 16:00:00.554 NULL NULL -200.0 -15910.599999999999 33.0 NULL 0.0 -23.0 -36 NULL NULL -23 NULL NULL -5.0 NULL 15601.0 NULL 5 NULL 1969-12-31 15:59:56.527 NULL NULL 15601.0 1241106.353 3.0 NULL 0.0 -23.0 -5 NULL NULL -3 NULL NULL +5.0 NULL 15601.0 NULL 5 NULL 1969-12-31 16:00:00.959 NULL NULL 15601.0 1241106.353 3.0 NULL 0.0 -23.0 -5 NULL NULL -3 NULL NULL 58.0 NULL 15601.0 NULL 58 NULL 1969-12-31 15:59:56.527 NULL NULL 15601.0 1241106.353 33.0 NULL 0.0 -23.0 -58 NULL NULL -23 NULL NULL 8.0 false NULL 10V3pN5r5lI2qWl2lG103 8 -362835731 1969-12-31 16:00:15.892 NULL 362835704.72 NULL NULL 1.0 NULL 0.0 NULL -8 NULL -3.62835744E8 -7 -362835704.72 0.0 8.0 false NULL 10c4qt584m5y6uWT 8 -183000142 1969-12-31 16:00:15.892 NULL 183000115.72 NULL NULL 1.0 NULL 0.0 NULL -8 NULL -1.8300016E8 -7 -183000115.72 0.0 diff --git ql/src/test/results/clientpositive/spark/parquet_vectorization_15.q.out ql/src/test/results/clientpositive/spark/parquet_vectorization_15.q.out index c8ac3f2..97cc8de 100644 --- ql/src/test/results/clientpositive/spark/parquet_vectorization_15.q.out +++ ql/src/test/results/clientpositive/spark/parquet_vectorization_15.q.out @@ -263,15 +263,15 @@ POSTHOOK: Input: default@alltypesparquet -51.0 true NULL QiOcvR0kt6r7f0R7fiPxQTCU -51 266531954 1969-12-31 16:00:08.451 NULL -266531980.28 NULL NULL 33.0 NULL 0.0 NULL 51 NULL 2.66532E8 -23 266531980.28 0.0 -51.0 true NULL Ybpj38RTTYl7CnJXPNx1g4C -51 -370919370 1969-12-31 16:00:08.451 NULL 370919343.72 NULL NULL 33.0 NULL 0.0 NULL 51 NULL -3.70919296E8 -23 -370919343.72 0.0 -6.0 NULL -200.0 NULL -6 NULL 1969-12-31 15:59:56.094 NULL NULL -200.0 -15910.599999999999 3.0 NULL 0.0 -23.0 6 NULL NULL -5 NULL NULL --62.0 NULL 15601.0 NULL -62 NULL 1969-12-31 15:59:56.527 NULL NULL 15601.0 1241106.353 33.0 NULL 0.0 -23.0 62 NULL NULL -23 NULL NULL +-62.0 NULL 15601.0 NULL -62 NULL 1969-12-31 16:00:09.889 NULL NULL 15601.0 1241106.353 33.0 NULL 0.0 -23.0 62 NULL NULL -23 NULL NULL 11.0 false NULL 10pO8p1LNx4Y 11 271296824 1969-12-31 16:00:02.351 NULL -271296850.28 NULL NULL 0.0 NULL 0.0 NULL -11 NULL 2.71296832E8 -1 271296850.28 0.0 11.0 false NULL 1H6wGP 11 -560827082 1969-12-31 16:00:02.351 NULL 560827055.72 NULL NULL 0.0 NULL 0.0 NULL -11 NULL -5.6082707E8 -1 -560827055.72 0.0 11.0 false NULL 2a7V63IL7jK3o 11 -325931647 1969-12-31 16:00:02.351 NULL 325931620.72 NULL NULL 0.0 NULL 0.0 NULL -11 NULL -3.25931648E8 -1 -325931620.72 0.0 11.0 true NULL 10 11 92365813 1969-12-31 16:00:02.351 NULL -92365839.28 NULL NULL 0.0 NULL 0.0 NULL -11 NULL 9.2365808E7 -1 92365839.28 0.0 -21.0 NULL 15601.0 NULL 21 NULL 1969-12-31 15:59:56.527 NULL NULL 15601.0 1241106.353 12.0 NULL 0.0 -23.0 -21 NULL NULL -2 NULL NULL +21.0 NULL 15601.0 NULL 21 NULL 1969-12-31 16:00:14.256 NULL NULL 15601.0 1241106.353 12.0 NULL 0.0 -23.0 -21 NULL NULL -2 NULL NULL 32.0 NULL -200.0 NULL 32 NULL 1969-12-31 16:00:02.445 NULL NULL -200.0 -15910.599999999999 1.0 NULL 0.0 -23.0 -32 NULL NULL -23 NULL NULL 36.0 NULL -200.0 NULL 36 NULL 1969-12-31 16:00:00.554 NULL NULL -200.0 -15910.599999999999 33.0 NULL 0.0 -23.0 -36 NULL NULL -23 NULL NULL -5.0 NULL 15601.0 NULL 5 NULL 1969-12-31 15:59:56.527 NULL NULL 15601.0 1241106.353 3.0 NULL 0.0 -23.0 -5 NULL NULL -3 NULL NULL +5.0 NULL 15601.0 NULL 5 NULL 1969-12-31 16:00:00.959 NULL NULL 15601.0 1241106.353 3.0 NULL 0.0 -23.0 -5 NULL NULL -3 NULL NULL 58.0 NULL 15601.0 NULL 58 NULL 1969-12-31 15:59:56.527 NULL NULL 15601.0 1241106.353 33.0 NULL 0.0 -23.0 -58 NULL NULL -23 NULL NULL 8.0 false NULL 10V3pN5r5lI2qWl2lG103 8 -362835731 1969-12-31 16:00:15.892 NULL 362835704.72 NULL NULL 1.0 NULL 0.0 NULL -8 NULL -3.62835744E8 -7 -362835704.72 0.0 8.0 false NULL 10c4qt584m5y6uWT 8 -183000142 1969-12-31 16:00:15.892 NULL 183000115.72 NULL NULL 1.0 NULL 0.0 NULL -8 NULL -1.8300016E8 -7 -183000115.72 0.0 diff --git ql/src/test/results/clientpositive/spark/vectorization_15.q.out ql/src/test/results/clientpositive/spark/vectorization_15.q.out index e0953d3..3e7aa0a 100644 --- ql/src/test/results/clientpositive/spark/vectorization_15.q.out +++ ql/src/test/results/clientpositive/spark/vectorization_15.q.out @@ -263,15 +263,15 @@ POSTHOOK: Input: default@alltypesorc -51.0 true NULL QiOcvR0kt6r7f0R7fiPxQTCU -51 266531954 1969-12-31 16:00:08.451 NULL -266531980.28 NULL NULL 33.0 NULL 0.0 NULL 51 NULL 2.66532E8 -23 266531980.28 0.0 -51.0 true NULL Ybpj38RTTYl7CnJXPNx1g4C -51 -370919370 1969-12-31 16:00:08.451 NULL 370919343.72 NULL NULL 33.0 NULL 0.0 NULL 51 NULL -3.70919296E8 -23 -370919343.72 0.0 -6.0 NULL -200.0 NULL -6 NULL 1969-12-31 15:59:56.094 NULL NULL -200.0 -15910.599999999999 3.0 NULL 0.0 -23.0 6 NULL NULL -5 NULL NULL --62.0 NULL 15601.0 NULL -62 NULL 1969-12-31 15:59:56.527 NULL NULL 15601.0 1241106.353 33.0 NULL 0.0 -23.0 62 NULL NULL -23 NULL NULL +-62.0 NULL 15601.0 NULL -62 NULL 1969-12-31 16:00:09.889 NULL NULL 15601.0 1241106.353 33.0 NULL 0.0 -23.0 62 NULL NULL -23 NULL NULL 11.0 false NULL 10pO8p1LNx4Y 11 271296824 1969-12-31 16:00:02.351 NULL -271296850.28 NULL NULL 0.0 NULL 0.0 NULL -11 NULL 2.71296832E8 -1 271296850.28 0.0 11.0 false NULL 1H6wGP 11 -560827082 1969-12-31 16:00:02.351 NULL 560827055.72 NULL NULL 0.0 NULL 0.0 NULL -11 NULL -5.6082707E8 -1 -560827055.72 0.0 11.0 false NULL 2a7V63IL7jK3o 11 -325931647 1969-12-31 16:00:02.351 NULL 325931620.72 NULL NULL 0.0 NULL 0.0 NULL -11 NULL -3.25931648E8 -1 -325931620.72 0.0 11.0 true NULL 10 11 92365813 1969-12-31 16:00:02.351 NULL -92365839.28 NULL NULL 0.0 NULL 0.0 NULL -11 NULL 9.2365808E7 -1 92365839.28 0.0 -21.0 NULL 15601.0 NULL 21 NULL 1969-12-31 15:59:56.527 NULL NULL 15601.0 1241106.353 12.0 NULL 0.0 -23.0 -21 NULL NULL -2 NULL NULL +21.0 NULL 15601.0 NULL 21 NULL 1969-12-31 16:00:14.256 NULL NULL 15601.0 1241106.353 12.0 NULL 0.0 -23.0 -21 NULL NULL -2 NULL NULL 32.0 NULL -200.0 NULL 32 NULL 1969-12-31 16:00:02.445 NULL NULL -200.0 -15910.599999999999 1.0 NULL 0.0 -23.0 -32 NULL NULL -23 NULL NULL 36.0 NULL -200.0 NULL 36 NULL 1969-12-31 16:00:00.554 NULL NULL -200.0 -15910.599999999999 33.0 NULL 0.0 -23.0 -36 NULL NULL -23 NULL NULL -5.0 NULL 15601.0 NULL 5 NULL 1969-12-31 15:59:56.527 NULL NULL 15601.0 1241106.353 3.0 NULL 0.0 -23.0 -5 NULL NULL -3 NULL NULL +5.0 NULL 15601.0 NULL 5 NULL 1969-12-31 16:00:00.959 NULL NULL 15601.0 1241106.353 3.0 NULL 0.0 -23.0 -5 NULL NULL -3 NULL NULL 58.0 NULL 15601.0 NULL 58 NULL 1969-12-31 15:59:56.527 NULL NULL 15601.0 1241106.353 33.0 NULL 0.0 -23.0 -58 NULL NULL -23 NULL NULL 8.0 false NULL 10V3pN5r5lI2qWl2lG103 8 -362835731 1969-12-31 16:00:15.892 NULL 362835704.72 NULL NULL 1.0 NULL 0.0 NULL -8 NULL -3.62835744E8 -7 -362835704.72 0.0 8.0 false NULL 10c4qt584m5y6uWT 8 -183000142 1969-12-31 16:00:15.892 NULL 183000115.72 NULL NULL 1.0 NULL 0.0 NULL -8 NULL -1.8300016E8 -7 -183000115.72 0.0 diff --git ql/src/test/results/clientpositive/vectorization_15.q.out ql/src/test/results/clientpositive/vectorization_15.q.out index e020bb8..3e3ceba 100644 --- ql/src/test/results/clientpositive/vectorization_15.q.out +++ ql/src/test/results/clientpositive/vectorization_15.q.out @@ -288,15 +288,15 @@ POSTHOOK: Input: default@alltypesorc -51.0 true NULL QiOcvR0kt6r7f0R7fiPxQTCU -51 266531954 1969-12-31 16:00:08.451 NULL -266531980.28 NULL NULL 33.0 NULL 0.0 NULL 51 NULL 2.66532E8 -23 266531980.28 0.0 -51.0 true NULL Ybpj38RTTYl7CnJXPNx1g4C -51 -370919370 1969-12-31 16:00:08.451 NULL 370919343.72 NULL NULL 33.0 NULL 0.0 NULL 51 NULL -3.70919296E8 -23 -370919343.72 0.0 -6.0 NULL -200.0 NULL -6 NULL 1969-12-31 15:59:56.094 NULL NULL -200.0 -15910.599999999999 3.0 NULL 0.0 -23.0 6 NULL NULL -5 NULL NULL --62.0 NULL 15601.0 NULL -62 NULL 1969-12-31 15:59:56.527 NULL NULL 15601.0 1241106.353 33.0 NULL 0.0 -23.0 62 NULL NULL -23 NULL NULL +-62.0 NULL 15601.0 NULL -62 NULL 1969-12-31 16:00:09.889 NULL NULL 15601.0 1241106.353 33.0 NULL 0.0 -23.0 62 NULL NULL -23 NULL NULL 11.0 false NULL 10pO8p1LNx4Y 11 271296824 1969-12-31 16:00:02.351 NULL -271296850.28 NULL NULL 0.0 NULL 0.0 NULL -11 NULL 2.71296832E8 -1 271296850.28 0.0 11.0 false NULL 1H6wGP 11 -560827082 1969-12-31 16:00:02.351 NULL 560827055.72 NULL NULL 0.0 NULL 0.0 NULL -11 NULL -5.6082707E8 -1 -560827055.72 0.0 11.0 false NULL 2a7V63IL7jK3o 11 -325931647 1969-12-31 16:00:02.351 NULL 325931620.72 NULL NULL 0.0 NULL 0.0 NULL -11 NULL -3.25931648E8 -1 -325931620.72 0.0 11.0 true NULL 10 11 92365813 1969-12-31 16:00:02.351 NULL -92365839.28 NULL NULL 0.0 NULL 0.0 NULL -11 NULL 9.2365808E7 -1 92365839.28 0.0 -21.0 NULL 15601.0 NULL 21 NULL 1969-12-31 15:59:56.527 NULL NULL 15601.0 1241106.353 12.0 NULL 0.0 -23.0 -21 NULL NULL -2 NULL NULL +21.0 NULL 15601.0 NULL 21 NULL 1969-12-31 16:00:14.256 NULL NULL 15601.0 1241106.353 12.0 NULL 0.0 -23.0 -21 NULL NULL -2 NULL NULL 32.0 NULL -200.0 NULL 32 NULL 1969-12-31 16:00:02.445 NULL NULL -200.0 -15910.599999999999 1.0 NULL 0.0 -23.0 -32 NULL NULL -23 NULL NULL 36.0 NULL -200.0 NULL 36 NULL 1969-12-31 16:00:00.554 NULL NULL -200.0 -15910.599999999999 33.0 NULL 0.0 -23.0 -36 NULL NULL -23 NULL NULL -5.0 NULL 15601.0 NULL 5 NULL 1969-12-31 15:59:56.527 NULL NULL 15601.0 1241106.353 3.0 NULL 0.0 -23.0 -5 NULL NULL -3 NULL NULL +5.0 NULL 15601.0 NULL 5 NULL 1969-12-31 16:00:00.959 NULL NULL 15601.0 1241106.353 3.0 NULL 0.0 -23.0 -5 NULL NULL -3 NULL NULL 58.0 NULL 15601.0 NULL 58 NULL 1969-12-31 15:59:56.527 NULL NULL 15601.0 1241106.353 33.0 NULL 0.0 -23.0 -58 NULL NULL -23 NULL NULL 8.0 false NULL 10V3pN5r5lI2qWl2lG103 8 -362835731 1969-12-31 16:00:15.892 NULL 362835704.72 NULL NULL 1.0 NULL 0.0 NULL -8 NULL -3.62835744E8 -7 -362835704.72 0.0 8.0 false NULL 10c4qt584m5y6uWT 8 -183000142 1969-12-31 16:00:15.892 NULL 183000115.72 NULL NULL 1.0 NULL 0.0 NULL -8 NULL -1.8300016E8 -7 -183000115.72 0.0 diff --git storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java index ea13c24..bebf769 100644 --- storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java +++ storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java @@ -180,12 +180,12 @@ public String stringifyColumn(int columnNum) { return b.toString(); } - @Override - public String toString() { + public String stringify(String prefix) { if (size == 0) { return ""; } StringBuilder b = new StringBuilder(); + b.append(prefix); b.append("Column vector types: "); for (int k = 0; k < projectionSize; k++) { int projIndex = projectedColumns[k]; @@ -233,11 +233,18 @@ public String toString() { if (k > 0) { b.append(", "); } - cv.stringifyValue(b, i); + if (cv != null) { + try { + cv.stringifyValue(b, i); + } catch (Exception ex) { + b.append(""); + } + } } b.append(']'); if (j < size - 1) { b.append('\n'); + b.append(prefix); } } } else { @@ -260,6 +267,7 @@ public String toString() { b.append(']'); if (i < size - 1) { b.append('\n'); + b.append(prefix); } } } @@ -267,6 +275,11 @@ public String toString() { } @Override + public String toString() { + return stringify(""); + } + + @Override public void readFields(DataInput arg0) throws IOException { throw new UnsupportedOperationException("Do you really need me?"); }