diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java index d7baeaf..e4368e6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java @@ -96,13 +96,11 @@ public void processOp(Object row, int tag) throws HiveException { } } - //Prepare output, set the projections + // Prepare output, set the projections int[] originalProjections = vrg.projectedColumns; int originalProjectionSize = vrg.projectionSize; vrg.projectionSize = vExpressions.length; - for (int i = 0; i < vExpressions.length; i++) { - vrg.projectedColumns[i] = vExpressions[i].getOutputColumn(); - } + vrg.projectedColumns = this.projectedColumns; forward(vrg, outputObjInspector); // Revert the projected columns back, because vrg will be re-used. diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java index d58a8c1..1d3db58 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java @@ -29,6 +29,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; +import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory; import org.apache.hadoop.hive.ql.exec.UDF; import org.apache.hadoop.hive.ql.exec.vector.expressions.ConstantVectorExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.FilterExprAndExpr; @@ -89,8 +91,7 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNull; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; /** * Context class for vectorization execution. @@ -246,6 +247,44 @@ public VectorExpression getVectorExpression(ExprNodeDesc exprDesc) throws HiveEx return ve; } + /** + * Handles only the special case of unary operators on a constant. + * @param exprDesc + * @return The same expression if no folding done, else return the constant + * expression. + * @throws HiveException + */ + private ExprNodeDesc foldConstantsForUnaryExpression(ExprNodeDesc exprDesc) throws HiveException { + if (!(exprDesc instanceof ExprNodeGenericFuncDesc)) { + return exprDesc; + } + + if (exprDesc.getChildren() == null || (exprDesc.getChildren().size() != 1) || + (!( exprDesc.getChildren().get(0) instanceof ExprNodeConstantDesc))) { + return exprDesc; + } + + GenericUDF gudf = ((ExprNodeGenericFuncDesc) exprDesc).getGenericUDF(); + if (!(gudf instanceof GenericUDFBridge)) { + return exprDesc; + } + + Class cl = ((GenericUDFBridge) gudf).getUdfClass(); + + ExprNodeConstantDesc constExpr = (ExprNodeConstantDesc) exprDesc.getChildren().get(0); + + if (cl.equals(UDFOPNegative.class) || cl.equals(UDFOPPositive.class)) { + ExprNodeEvaluator evaluator = ExprNodeEvaluatorFactory.get(exprDesc); + ObjectInspector output = evaluator.initialize(null); + + Object constant = evaluator.evaluate(null); + Object java = ObjectInspectorUtils.copyToStandardJavaObject(constant, output); + return new ExprNodeConstantDesc(java); + } else { + return exprDesc; + } + } + private VectorExpression getConstantVectorExpression(ExprNodeConstantDesc exprDesc) throws HiveException { String type = exprDesc.getTypeString(); @@ -378,6 +417,10 @@ private VectorExpression getBinaryArithmeticExpression(String method, ExprNodeDesc leftExpr = childExpr.get(0); ExprNodeDesc rightExpr = childExpr.get(1); + // TODO: Remove this when constant folding is fixed in the optimizer. + leftExpr = foldConstantsForUnaryExpression(leftExpr); + rightExpr = foldConstantsForUnaryExpression(rightExpr); + VectorExpression v1 = null; VectorExpression v2 = null; @@ -655,6 +698,10 @@ private VectorExpression getVectorBinaryComparisonFilterExpression(String ExprNodeDesc leftExpr = childExpr.get(0); ExprNodeDesc rightExpr = childExpr.get(1); + // TODO: Remove this when constant folding is fixed in the optimizer. + leftExpr = foldConstantsForUnaryExpression(leftExpr); + rightExpr = foldConstantsForUnaryExpression(rightExpr); + VectorExpression expr = null; VectorExpression v1 = null; VectorExpression v2 = null; diff --git ql/src/java/org/apache/hadoop/hive/ql/io/VectorizedRCFileRecordReader.java ql/src/java/org/apache/hadoop/hive/ql/io/VectorizedRCFileRecordReader.java index 54c2438..25b3aed 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/VectorizedRCFileRecordReader.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/VectorizedRCFileRecordReader.java @@ -168,6 +168,11 @@ public boolean next(NullWritable key, VectorizedRowBatch value) throws IOExcepti // Reset column fields noNull values to true VectorizedBatchUtil.SetNoNullFields(true, value); + value.selectedInUse = false; + for (int i = 0; i < value.numCols; i++) { + value.cols[i].isRepeating = false; + } + int i = 0; try { for (; i < VectorizedRowBatch.DEFAULT_SIZE; i++) { diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java index c29d97f..790c8c7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java @@ -1568,6 +1568,7 @@ public VectorizedRowBatch nextBatch(VectorizedRowBatch previous) throws IOExcept result.cols = cols; } else { result = (VectorizedRowBatch) previous; + result.selectedInUse = false; reader.nextVector(result.cols, (int) batchSize); } diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java index 7b24749..5f54712 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java @@ -9,6 +9,7 @@ import java.util.Map; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColGreaterLongScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringColGreaterStringScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.LongColAddLongColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.LongColModuloLongColumn; @@ -143,4 +144,29 @@ public void testStringFilterExpressions() throws HiveException { assertTrue(ve instanceof FilterStringColGreaterStringScalar); } + + @Test + public void testFilterWithNegativeScalar() throws HiveException { + ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(Integer.class, "col1", "table", false); + ExprNodeConstantDesc constDesc = new ExprNodeConstantDesc(new Integer(-10)); + + GenericUDFOPGreaterThan udf = new GenericUDFOPGreaterThan(); + ExprNodeGenericFuncDesc exprDesc = new ExprNodeGenericFuncDesc(); + exprDesc.setGenericUDF(udf); + List children1 = new ArrayList(2); + children1.add(col1Expr); + children1.add(constDesc); + exprDesc.setChildExprs(children1); + + Map columnMap = new HashMap(); + columnMap.put("col1", 1); + columnMap.put("col2", 2); + + VectorizationContext vc = new VectorizationContext(columnMap, 2); + vc.setOperatorType(OperatorType.FILTER); + + VectorExpression ve = vc.getVectorExpression(exprDesc); + + assertTrue(ve instanceof FilterLongColGreaterLongScalar); + } }