diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java index c41efb1..b3ef642 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java @@ -145,7 +145,9 @@ public int bufferSize() { * @param length length of source byte sequence */ public void setVal(int elementNum, byte[] sourceBuf, int start, int length) { - if ((nextFree + length) > buffer.length) { + if (buffer == null) { + initBuffer(length); + } else if ((nextFree + length) > buffer.length) { increaseBufferSpace(length); } System.arraycopy(sourceBuf, start, buffer, nextFree, length); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java index eb8c4c5..b963438 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java @@ -307,6 +307,10 @@ public VectorExpression getVectorExpression(ExprNodeDesc exprDesc, Mode mode) th if (ve == null) { throw new HiveException("Could not vectorize expression: "+exprDesc.getName()); } + if (LOG.isDebugEnabled()) { + LOG.debug("Input Expression = " + exprDesc.getTypeInfo() + +", Vectorized Expression = "+ ve.toString()); + } return ve; } @@ -597,7 +601,7 @@ private ExprNodeDesc foldConstantsForUnaryExpression(ExprNodeDesc exprDesc) thro if (!(exprDesc instanceof ExprNodeGenericFuncDesc)) { return exprDesc; } - + if (exprDesc.getChildren() == null || (exprDesc.getChildren().size() != 1) || (!( exprDesc.getChildren().get(0) instanceof ExprNodeConstantDesc))) { return exprDesc; @@ -605,10 +609,11 @@ private ExprNodeDesc foldConstantsForUnaryExpression(ExprNodeDesc exprDesc) thro GenericUDF gudf = ((ExprNodeGenericFuncDesc) exprDesc).getGenericUDF(); if (gudf instanceof GenericUDFOPNegative || gudf instanceof GenericUDFOPPositive - || castExpressionUdfs.contains(gudf) + || castExpressionUdfs.contains(gudf.getClass()) || ((gudf instanceof GenericUDFBridge) && castExpressionUdfs.contains(((GenericUDFBridge) gudf).getUdfClass()))) { - ExprNodeEvaluator evaluator = ExprNodeEvaluatorFactory.get(exprDesc); + + ExprNodeEvaluator evaluator = ExprNodeEvaluatorFactory.get(exprDesc); ObjectInspector output = evaluator.initialize(null); Object constant = evaluator.evaluate(null); Object java = ObjectInspectorUtils.copyToStandardJavaObject(constant, output); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java index 901005e..f89f5c9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; +import java.util.Arrays; + import org.apache.hadoop.hive.common.type.Decimal128; import org.apache.hadoop.hive.ql.exec.vector.*; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; @@ -93,14 +95,14 @@ private void evaluateBytes(VectorizedRowBatch vrg) { BytesColumnVector cv = (BytesColumnVector) vrg.cols[outputColumn]; cv.isRepeating = true; cv.noNulls = true; - cv.setRef(0, bytesValue, 0, bytesValueLength); + cv.setVal(0, bytesValue, 0, bytesValueLength); } private void evaluateDecimal(VectorizedRowBatch vrg) { DecimalColumnVector dcv = (DecimalColumnVector) vrg.cols[outputColumn]; dcv.isRepeating = true; dcv.noNulls = true; - dcv.vector[0] = decimalValue; + dcv.vector[0].update(decimalValue); } @Override @@ -152,12 +154,12 @@ public void setDoubleValue(double doubleValue) { } public void setBytesValue(byte[] bytesValue) { - this.bytesValue = bytesValue; + this.bytesValue = Arrays.copyOf(bytesValue, bytesValue.length); this.bytesValueLength = bytesValue.length; } public void setDecimalValue(Decimal128 decimalValue) { - this.decimalValue = decimalValue; + this.decimalValue = new Decimal128(decimalValue); } public String getTypeString() { diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestConstantVectorExpression.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestConstantVectorExpression.java index 4321545..6df4917 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestConstantVectorExpression.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestConstantVectorExpression.java @@ -23,7 +23,9 @@ import java.util.Arrays; +import org.apache.hadoop.hive.common.type.Decimal128; import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; @@ -39,27 +41,57 @@ public void testConstantExpression() { ConstantVectorExpression longCve = new ConstantVectorExpression(0, 17); ConstantVectorExpression doubleCve = new ConstantVectorExpression(1, 17.34); - ConstantVectorExpression bytesCve = new ConstantVectorExpression(2, "alpha".getBytes()); + String str = "alpha"; + ConstantVectorExpression bytesCve = new ConstantVectorExpression(2, str.getBytes()); + Decimal128 decVal = new Decimal128(25.8, (short) 1); + ConstantVectorExpression decimalCve = new ConstantVectorExpression(3, decVal); int size = 20; - VectorizedRowBatch vrg = VectorizedRowGroupGenUtil.getVectorizedRowBatch(size, 3, 0); + VectorizedRowBatch vrg = VectorizedRowGroupGenUtil.getVectorizedRowBatch(size, 4, 0); LongColumnVector lcv = (LongColumnVector) vrg.cols[0]; DoubleColumnVector dcv = new DoubleColumnVector(size); BytesColumnVector bcv = new BytesColumnVector(size); + DecimalColumnVector dv = new DecimalColumnVector(5, 1); vrg.cols[1] = dcv; vrg.cols[2] = bcv; + vrg.cols[3] = dv; longCve.evaluate(vrg); doubleCve.evaluate(vrg); - bytesCve.evaluate(vrg); - + bytesCve.evaluate(vrg); + decimalCve.evaluate(vrg); assertTrue(lcv.isRepeating); assertTrue(dcv.isRepeating); assertTrue(bcv.isRepeating); assertEquals(17, lcv.vector[0]); assertTrue(17.34 == dcv.vector[0]); - assertTrue(Arrays.equals("alpha".getBytes(), bcv.vector[0])); + + byte[] alphaBytes = "alpha".getBytes(); + assertTrue(bcv.length[0] == alphaBytes.length); + assertTrue(sameFirstKBytes(alphaBytes, bcv.vector[0], alphaBytes.length)); + // Evaluation of the bytes Constant Vector Expression after the value of the original string is + // modified. This should not change the value of vector column in bcv. + str = "beta"; + bytesCve.evaluate(vrg); + assertTrue(bcv.length[0] == alphaBytes.length); + assertTrue(sameFirstKBytes(alphaBytes, bcv.vector[0], alphaBytes.length)); + + assertTrue(25.8 == dv.vector[0].doubleValue()); + // Evaluation of the decimal Constant Vector Expression after the original decimal value object is + // modified. This should not change the value of vector column in dv. + decVal.update(29); + decimalCve.evaluate(vrg); + assertTrue(25.8 == dv.vector[0].doubleValue()); + } + + private boolean sameFirstKBytes(byte[] o1, byte[] o2, int k) { + for (int i = 0; i != k; i++) { + if (o1[i] != o2[i]) { + return false; + } + } + return true; } }