Index: hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java =================================================================== --- hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java (revision 1528446) +++ hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java (working copy) @@ -43,6 +43,7 @@ import org.apache.hadoop.hive.ql.index.IndexSearchCondition; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.ByteStream; @@ -253,8 +254,8 @@ if (filterExprSerialized == null) { return scan; } - ExprNodeDesc filterExpr = - Utilities.deserializeExpression(filterExprSerialized, jobConf); + ExprNodeGenericFuncDesc filterExpr = + Utilities.deserializeExpression(filterExprSerialized); String colName = jobConf.get(serdeConstants.LIST_COLUMNS).split(",")[iKey]; String colType = jobConf.get(serdeConstants.LIST_COLUMN_TYPES).split(",")[iKey]; Index: hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java =================================================================== --- hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java (revision 1528446) +++ hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java (working copy) @@ -50,6 +50,7 @@ import org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler; import org.apache.hadoop.hive.ql.metadata.HiveStoragePredicateHandler; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.SerDe; @@ -410,8 +411,8 @@ hbaseSerde.getStorageFormatOfCol(keyColPos).get(0)); List searchConditions = new ArrayList(); - ExprNodeDesc residualPredicate = - analyzer.analyzePredicate(predicate, searchConditions); + ExprNodeGenericFuncDesc residualPredicate = + (ExprNodeGenericFuncDesc)analyzer.analyzePredicate(predicate, searchConditions); int scSize = searchConditions.size(); if (scSize < 1 || 2 < scSize) { // Either there was nothing which could be pushed down (size = 0), Index: ql/src/test/results/compiler/plan/input2.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input2.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/input2.q.xml (working copy) @@ -1939,7 +1939,7 @@ - + @@ -2259,11 +2259,11 @@ - + - + @@ -2300,7 +2300,7 @@ - + @@ -2593,7 +2593,7 @@ - + Index: ql/src/test/results/compiler/plan/join2.q.xml =================================================================== --- ql/src/test/results/compiler/plan/join2.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/join2.q.xml (working copy) @@ -345,7 +345,7 @@ - + @@ -663,7 +663,7 @@ - + Index: ql/src/test/results/compiler/plan/input3.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input3.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/input3.q.xml (working copy) @@ -2325,7 +2325,7 @@ - + @@ -2645,11 +2645,11 @@ - + - + @@ -2686,7 +2686,7 @@ - + @@ -2979,11 +2979,11 @@ - + - + @@ -3020,7 +3020,7 @@ - + @@ -3271,7 +3271,7 @@ - + Index: ql/src/test/results/compiler/plan/input4.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input4.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/input4.q.xml (working copy) @@ -550,7 +550,7 @@ - + Index: ql/src/test/results/compiler/plan/join4.q.xml =================================================================== --- ql/src/test/results/compiler/plan/join4.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/join4.q.xml (working copy) @@ -644,11 +644,11 @@ - + - + @@ -693,7 +693,7 @@ - + @@ -1275,11 +1275,11 @@ - + - + @@ -1316,7 +1316,7 @@ - + Index: ql/src/test/results/compiler/plan/join5.q.xml =================================================================== --- ql/src/test/results/compiler/plan/join5.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/join5.q.xml (working copy) @@ -644,11 +644,11 @@ - + - + @@ -693,7 +693,7 @@ - + @@ -1275,11 +1275,11 @@ - + - + @@ -1316,7 +1316,7 @@ - + Index: ql/src/test/results/compiler/plan/input_testxpath2.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input_testxpath2.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/input_testxpath2.q.xml (working copy) @@ -324,7 +324,7 @@ _col2 - + @@ -363,7 +363,7 @@ _col1 - + @@ -422,7 +422,7 @@ _col0 - + @@ -573,11 +573,11 @@ - + - + @@ -608,11 +608,11 @@ - + - + Index: ql/src/test/results/compiler/plan/input6.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input6.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/input6.q.xml (working copy) @@ -887,7 +887,7 @@ - + Index: ql/src/test/results/compiler/plan/join6.q.xml =================================================================== --- ql/src/test/results/compiler/plan/join6.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/join6.q.xml (working copy) @@ -644,11 +644,11 @@ - + - + @@ -693,7 +693,7 @@ - + @@ -1275,11 +1275,11 @@ - + - + @@ -1316,7 +1316,7 @@ - + Index: ql/src/test/results/compiler/plan/join7.q.xml =================================================================== --- ql/src/test/results/compiler/plan/join7.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/join7.q.xml (working copy) @@ -780,11 +780,11 @@ - + - + @@ -829,7 +829,7 @@ - + @@ -1411,11 +1411,11 @@ - + - + @@ -1452,7 +1452,7 @@ - + @@ -2030,11 +2030,11 @@ - + - + @@ -2071,7 +2071,7 @@ - + Index: ql/src/test/results/compiler/plan/input8.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input8.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/input8.q.xml (working copy) @@ -320,7 +320,7 @@ _col2 - + @@ -351,7 +351,7 @@ _col1 - + @@ -396,7 +396,7 @@ _col0 - + Index: ql/src/test/results/compiler/plan/join8.q.xml =================================================================== --- ql/src/test/results/compiler/plan/join8.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/join8.q.xml (working copy) @@ -644,15 +644,15 @@ - + - + - + @@ -697,7 +697,7 @@ - + @@ -744,7 +744,7 @@ - + @@ -1316,15 +1316,15 @@ - + - + - + @@ -1361,7 +1361,7 @@ - + @@ -1408,7 +1408,7 @@ - + @@ -2196,7 +2196,7 @@ - + Index: ql/src/test/results/compiler/plan/union.q.xml =================================================================== --- ql/src/test/results/compiler/plan/union.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/union.q.xml (working copy) @@ -1046,7 +1046,7 @@ - + @@ -1492,7 +1492,7 @@ - + Index: ql/src/test/results/compiler/plan/input9.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input9.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/input9.q.xml (working copy) @@ -882,7 +882,7 @@ - + Index: ql/src/test/results/compiler/plan/udf1.q.xml =================================================================== --- ql/src/test/results/compiler/plan/udf1.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/udf1.q.xml (working copy) @@ -544,7 +544,7 @@ _col8 - + @@ -589,7 +589,7 @@ _col7 - + @@ -634,7 +634,7 @@ _col6 - + @@ -679,7 +679,7 @@ _col5 - + @@ -724,7 +724,7 @@ _col4 - + @@ -769,7 +769,7 @@ _col3 - + @@ -814,7 +814,7 @@ _col2 - + @@ -859,7 +859,7 @@ _col1 - + @@ -904,7 +904,7 @@ _col9 - + @@ -949,7 +949,7 @@ _col13 - + @@ -1001,7 +1001,7 @@ _col12 - + @@ -1046,7 +1046,7 @@ _col11 - + @@ -1091,7 +1091,7 @@ _col10 - + @@ -1136,7 +1136,7 @@ _col16 - + @@ -1188,7 +1188,7 @@ _col15 - + @@ -1240,7 +1240,7 @@ _col14 - + @@ -1292,7 +1292,7 @@ _col0 - + @@ -1764,7 +1764,7 @@ - + Index: ql/src/test/results/compiler/plan/udf4.q.xml =================================================================== --- ql/src/test/results/compiler/plan/udf4.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/udf4.q.xml (working copy) @@ -556,7 +556,7 @@ _col8 - + @@ -588,11 +588,11 @@ _col7 - + - + @@ -644,7 +644,7 @@ _col6 - + @@ -676,11 +676,11 @@ _col5 - + - + @@ -732,7 +732,7 @@ _col4 - + @@ -764,7 +764,7 @@ _col3 - + @@ -796,11 +796,11 @@ _col2 - + - + @@ -852,7 +852,7 @@ _col1 - + @@ -884,7 +884,7 @@ _col9 - + @@ -916,7 +916,7 @@ _col13 - + @@ -948,7 +948,7 @@ _col12 - + @@ -980,11 +980,11 @@ _col11 - + - + @@ -1036,7 +1036,7 @@ _col10 - + @@ -1068,7 +1068,7 @@ _col17 - + @@ -1082,7 +1082,7 @@ - + @@ -1137,7 +1137,7 @@ _col16 - + @@ -1182,7 +1182,7 @@ _col15 - + @@ -1228,7 +1228,7 @@ _col0 - + @@ -1260,7 +1260,7 @@ _col18 - + Index: ql/src/test/results/compiler/plan/input_testxpath.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input_testxpath.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/input_testxpath.q.xml (working copy) @@ -324,7 +324,7 @@ _col2 - + @@ -371,7 +371,7 @@ - + @@ -451,7 +451,7 @@ _col0 - + Index: ql/src/test/results/compiler/plan/udf6.q.xml =================================================================== --- ql/src/test/results/compiler/plan/udf6.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/udf6.q.xml (working copy) @@ -300,7 +300,7 @@ _col1 - + @@ -349,7 +349,7 @@ _col0 - + Index: ql/src/test/results/compiler/plan/input_part1.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input_part1.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/input_part1.q.xml (working copy) @@ -584,7 +584,7 @@ - + Index: ql/src/test/results/compiler/plan/groupby1.q.xml =================================================================== --- ql/src/test/results/compiler/plan/groupby1.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/groupby1.q.xml (working copy) @@ -561,7 +561,7 @@ - + Index: ql/src/test/results/compiler/plan/groupby2.q.xml =================================================================== --- ql/src/test/results/compiler/plan/groupby2.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/groupby2.q.xml (working copy) @@ -462,7 +462,7 @@ _col1 - + @@ -511,7 +511,7 @@ _col0 - + @@ -589,7 +589,7 @@ - + @@ -650,7 +650,7 @@ - + @@ -1480,7 +1480,7 @@ _col2 - + Index: ql/src/test/results/compiler/plan/udf_case.q.xml =================================================================== --- ql/src/test/results/compiler/plan/udf_case.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/udf_case.q.xml (working copy) @@ -377,7 +377,7 @@ _col1 - + @@ -442,7 +442,7 @@ _col0 - + Index: ql/src/test/results/compiler/plan/subq.q.xml =================================================================== --- ql/src/test/results/compiler/plan/subq.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/subq.q.xml (working copy) @@ -777,7 +777,7 @@ - + Index: ql/src/test/results/compiler/plan/groupby3.q.xml =================================================================== --- ql/src/test/results/compiler/plan/groupby3.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/groupby3.q.xml (working copy) @@ -542,7 +542,7 @@ _col0 - + @@ -611,7 +611,7 @@ - + @@ -672,7 +672,7 @@ - + @@ -736,7 +736,7 @@ - + @@ -797,7 +797,7 @@ - + @@ -858,7 +858,7 @@ - + Index: ql/src/test/results/compiler/plan/groupby4.q.xml =================================================================== --- ql/src/test/results/compiler/plan/groupby4.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/groupby4.q.xml (working copy) @@ -341,7 +341,7 @@ _col0 - + Index: ql/src/test/results/compiler/plan/groupby5.q.xml =================================================================== --- ql/src/test/results/compiler/plan/groupby5.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/groupby5.q.xml (working copy) @@ -414,7 +414,7 @@ - + Index: ql/src/test/results/compiler/plan/groupby6.q.xml =================================================================== --- ql/src/test/results/compiler/plan/groupby6.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/groupby6.q.xml (working copy) @@ -341,7 +341,7 @@ _col0 - + Index: ql/src/test/results/compiler/plan/case_sensitivity.q.xml =================================================================== --- ql/src/test/results/compiler/plan/case_sensitivity.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/case_sensitivity.q.xml (working copy) @@ -772,7 +772,7 @@ - + @@ -856,7 +856,7 @@ _col0 - + @@ -989,11 +989,11 @@ - + - + Index: ql/src/test/results/compiler/plan/udf_when.q.xml =================================================================== --- ql/src/test/results/compiler/plan/udf_when.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/udf_when.q.xml (working copy) @@ -377,11 +377,11 @@ _col1 - + - + @@ -429,7 +429,7 @@ - + @@ -484,11 +484,11 @@ _col0 - + - + @@ -532,7 +532,7 @@ - + Index: ql/src/test/results/compiler/plan/input20.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input20.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/input20.q.xml (working copy) @@ -555,7 +555,7 @@ _col1 - + @@ -611,7 +611,7 @@ _col0 - + Index: ql/src/test/results/compiler/plan/sample1.q.xml =================================================================== --- ql/src/test/results/compiler/plan/sample1.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/sample1.q.xml (working copy) @@ -590,23 +590,23 @@ - + - + - + - + - + Index: ql/src/test/results/compiler/plan/sample2.q.xml =================================================================== --- ql/src/test/results/compiler/plan/sample2.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/sample2.q.xml (working copy) @@ -905,19 +905,19 @@ - + - + - + - + Index: ql/src/test/results/compiler/plan/sample3.q.xml =================================================================== --- ql/src/test/results/compiler/plan/sample3.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/sample3.q.xml (working copy) @@ -905,19 +905,19 @@ - + - + - + - + Index: ql/src/test/results/compiler/plan/sample4.q.xml =================================================================== --- ql/src/test/results/compiler/plan/sample4.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/sample4.q.xml (working copy) @@ -905,19 +905,19 @@ - + - + - + - + Index: ql/src/test/results/compiler/plan/sample5.q.xml =================================================================== --- ql/src/test/results/compiler/plan/sample5.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/sample5.q.xml (working copy) @@ -905,19 +905,19 @@ - + - + - + - + Index: ql/src/test/results/compiler/plan/sample6.q.xml =================================================================== --- ql/src/test/results/compiler/plan/sample6.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/sample6.q.xml (working copy) @@ -905,19 +905,19 @@ - + - + - + - + Index: ql/src/test/results/compiler/plan/sample7.q.xml =================================================================== --- ql/src/test/results/compiler/plan/sample7.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/sample7.q.xml (working copy) @@ -902,23 +902,23 @@ - + - + - + - + - + @@ -1029,7 +1029,7 @@ - + Index: ql/src/test/results/compiler/plan/cast1.q.xml =================================================================== --- ql/src/test/results/compiler/plan/cast1.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/cast1.q.xml (working copy) @@ -388,7 +388,7 @@ _col6 - + @@ -420,7 +420,7 @@ _col5 - + @@ -452,7 +452,7 @@ _col4 - + @@ -466,7 +466,7 @@ - + @@ -518,7 +518,7 @@ _col3 - + @@ -563,7 +563,7 @@ _col2 - + @@ -608,7 +608,7 @@ _col1 - + @@ -653,7 +653,7 @@ _col0 - + @@ -905,7 +905,7 @@ - + Index: ql/src/test/results/compiler/plan/input1.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input1.q.xml (revision 1528446) +++ ql/src/test/results/compiler/plan/input1.q.xml (working copy) @@ -887,7 +887,7 @@ - + Index: ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java (revision 1528446) +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java (working copy) @@ -101,23 +101,23 @@ children1.add(minusExpr); children1.add(multiplyExpr); - sumExpr.setChildExprs(children1); + sumExpr.setChildren(children1); children2.add(sum2Expr); children2.add(col3Expr); - minusExpr.setChildExprs(children2); + minusExpr.setChildren(children2); children3.add(col1Expr); children3.add(col2Expr); - sum2Expr.setChildExprs(children3); + sum2Expr.setChildren(children3); children4.add(col4Expr); children4.add(modExpr); - multiplyExpr.setChildExprs(children4); + multiplyExpr.setChildren(children4); children5.add(col5Expr); children5.add(col6Expr); - modExpr.setChildExprs(children5); + modExpr.setChildren(children5); Map columnMap = new HashMap(); columnMap.put("col1", 1); @@ -163,7 +163,7 @@ List children1 = new ArrayList(2); children1.add(col1Expr); children1.add(constDesc); - exprDesc.setChildExprs(children1); + exprDesc.setChildren(children1); Map columnMap = new HashMap(); columnMap.put("col1", 1); @@ -188,7 +188,7 @@ List children1 = new ArrayList(2); children1.add(col1Expr); children1.add(col2Expr); - exprDesc.setChildExprs(children1); + exprDesc.setChildren(children1); Map columnMap = new HashMap(); columnMap.put("col1", 1); @@ -214,7 +214,7 @@ List children1 = new ArrayList(2); children1.add(col1Expr); children1.add(constDesc); - exprDesc.setChildExprs(children1); + exprDesc.setChildren(children1); Map columnMap = new HashMap(); columnMap.put("col1", 0); @@ -238,7 +238,7 @@ List children1 = new ArrayList(2); children1.add(col1Expr); children1.add(constDesc); - greaterExprDesc.setChildExprs(children1); + greaterExprDesc.setChildren(children1); ExprNodeColumnDesc col2Expr = new ExprNodeColumnDesc(Float.class, "col2", "table", false); ExprNodeConstantDesc const2Desc = new ExprNodeConstantDesc(new Float(1.0)); @@ -249,7 +249,7 @@ List children2 = new ArrayList(2); children2.add(col2Expr); children2.add(const2Desc); - lessExprDesc.setChildExprs(children2); + lessExprDesc.setChildren(children2); GenericUDFOPAnd andUdf = new GenericUDFOPAnd(); ExprNodeGenericFuncDesc andExprDesc = new ExprNodeGenericFuncDesc(); @@ -257,7 +257,7 @@ List children3 = new ArrayList(2); children3.add(greaterExprDesc); children3.add(lessExprDesc); - andExprDesc.setChildExprs(children3); + andExprDesc.setChildren(children3); Map columnMap = new HashMap(); columnMap.put("col1", 0); @@ -278,7 +278,7 @@ List children4 = new ArrayList(2); children4.add(greaterExprDesc); children4.add(lessExprDesc); - orExprDesc.setChildExprs(children4); + orExprDesc.setChildren(children4); VectorExpression veOr = vc.getVectorExpression(orExprDesc); @@ -300,7 +300,7 @@ children.add(constDesc); children.add(colDesc); - scalarMinusConstant.setChildExprs(children); + scalarMinusConstant.setChildren(children); Map columnMap = new HashMap(); columnMap.put("a", 0); @@ -322,7 +322,7 @@ List children1 = new ArrayList(2); children1.add(col1Expr); children1.add(constDesc); - exprDesc.setChildExprs(children1); + exprDesc.setChildren(children1); Map columnMap = new HashMap(); columnMap.put("col1", 1); @@ -344,7 +344,7 @@ negExprDesc.setGenericUDF(gudf); List children = new ArrayList(1); children.add(col1Expr); - negExprDesc.setChildExprs(children); + negExprDesc.setChildren(children); Map columnMap = new HashMap(); columnMap.put("col1", 1); VectorizationContext vc = new VectorizationContext(columnMap, 1); @@ -363,7 +363,7 @@ negExprDesc.setGenericUDF(gudf); List children = new ArrayList(1); children.add(col1Expr); - negExprDesc.setChildExprs(children); + negExprDesc.setChildren(children); Map columnMap = new HashMap(); columnMap.put("col1", 1); VectorizationContext vc = new VectorizationContext(columnMap, 1); @@ -388,7 +388,7 @@ children.add(constDesc); children.add(colDesc); - scalarGreaterColExpr.setChildExprs(children); + scalarGreaterColExpr.setChildren(children); Map columnMap = new HashMap(); columnMap.put("a", 0); Index: ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSelectOperator.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSelectOperator.java (revision 1528446) +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSelectOperator.java (working copy) @@ -100,7 +100,7 @@ List children = new ArrayList(); children.add(colDesc1); children.add(colDesc2); - plusDesc.setChildExprs(children); + plusDesc.setChildren(children); plusDesc.setTypeInfo(TypeInfoFactory.longTypeInfo); colList.add(plusDesc); Index: ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java (revision 1528446) +++ ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java (working copy) @@ -18,17 +18,19 @@ package org.apache.hadoop.hive.ql.exec; -import java.sql.Date; +import static org.apache.hadoop.hive.ql.exec.Utilities.getFileExtension; + import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.List; -import static org.apache.hadoop.hive.ql.exec.Utilities.getFileExtension; - import junit.framework.TestCase; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFFromUtcTimestamp; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.mapred.JobConf; @@ -64,10 +66,12 @@ public void testSerializeTimestamp() { Timestamp ts = new Timestamp(1374554702000L); ts.setNanos(123456); - ExprNodeConstantDesc constant = new ExprNodeConstantDesc( - TypeInfoFactory.timestampTypeInfo, ts); - String serialized = Utilities.serializeExpression(constant); - ExprNodeDesc deserialized = Utilities.deserializeExpression(serialized, new Configuration()); - assertEquals(constant.getExprString(), deserialized.getExprString()); + ExprNodeConstantDesc constant = new ExprNodeConstantDesc(ts); + List children = new ArrayList(1); + children.add(constant); + ExprNodeGenericFuncDesc desc = new ExprNodeGenericFuncDesc(TypeInfoFactory.timestampTypeInfo, + new GenericUDFFromUtcTimestamp(), children); + assertEquals(desc.getExprString(), Utilities.deserializeExpression( + Utilities.serializeExpression(desc)).getExprString()); } } Index: ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java (revision 1528446) +++ ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java (working copy) @@ -18,20 +18,21 @@ package org.apache.hadoop.hive.ql.io.sarg; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.ql.exec.Utilities; +import static junit.framework.Assert.assertEquals; +import static junit.framework.Assert.assertTrue; + +import java.beans.XMLDecoder; +import java.io.ByteArrayInputStream; +import java.io.UnsupportedEncodingException; +import java.util.List; + import org.apache.hadoop.hive.ql.io.sarg.SearchArgument.TruthValue; import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentImpl.ExpressionBuilder; import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentImpl.ExpressionTree; -import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.mina.util.IdentityHashSet; import org.junit.Test; -import java.util.List; - -import static junit.framework.Assert.assertEquals; -import static junit.framework.Assert.assertTrue; - /** * These test the SARG implementation. * The xml files were generated by setting hive.optimize.index.filter @@ -224,6 +225,23 @@ } } + private ExprNodeGenericFuncDesc getFuncDesc (String xmlSerialized) { + byte[] bytes; + try { + bytes = xmlSerialized.getBytes("UTF-8"); + } catch (UnsupportedEncodingException ex) { + throw new RuntimeException("UTF-8 support required", ex); + } + + ByteArrayInputStream bais = new ByteArrayInputStream(bytes); + XMLDecoder decoder = new XMLDecoder(bais, null, null); + + try { + return (ExprNodeGenericFuncDesc) decoder.readObject(); + } finally { + decoder.close(); + } + } @Test public void testExpression1() throws Exception { // first_name = 'john' or @@ -237,35 +255,35 @@ String exprStr = " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -310,7 +328,7 @@ " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -357,7 +375,7 @@ " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -404,7 +422,7 @@ " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -455,7 +473,7 @@ " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -502,7 +520,7 @@ " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -549,7 +567,7 @@ " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -596,11 +614,11 @@ " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -637,7 +655,7 @@ " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -692,10 +710,9 @@ " \n" + " \n" + " \n"; - Configuration conf = new Configuration(); - ExprNodeDesc expr = Utilities.deserializeExpression(exprStr, conf); + SearchArgumentImpl sarg = - (SearchArgumentImpl) SearchArgument.FACTORY.create(expr); + (SearchArgumentImpl) SearchArgument.FACTORY.create(getFuncDesc(exprStr)); List leaves = sarg.getLeaves(); assertEquals(9, leaves.size()); @@ -771,19 +788,19 @@ String exprStr = " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -818,7 +835,7 @@ " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -865,7 +882,7 @@ " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -916,7 +933,7 @@ " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -961,10 +978,9 @@ " \n" + " \n" + " \n"; - Configuration conf = new Configuration(); - ExprNodeDesc expr = Utilities.deserializeExpression(exprStr, conf); + SearchArgumentImpl sarg = - (SearchArgumentImpl) SearchArgument.FACTORY.create(expr); + (SearchArgumentImpl) SearchArgument.FACTORY.create(getFuncDesc(exprStr)); List leaves = sarg.getLeaves(); assertEquals(4, leaves.size()); @@ -1048,23 +1064,23 @@ String exprStr = " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -1129,7 +1145,7 @@ " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -1180,11 +1196,11 @@ " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -1210,9 +1226,6 @@ " \n" + " \n" + " \n" + - " \n" + - " org.apache.hadoop.hive.ql.udf.UDFSubstr \n" + - " \n" + " \n" + " org.apache.hadoop.hive.ql.udf.UDFSubstr \n" + " \n" + @@ -1261,7 +1274,7 @@ " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -1308,11 +1321,11 @@ " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -1341,9 +1354,6 @@ " \n" + " \n" + " \n" + - " \n" + - " org.apache.hadoop.hive.ql.udf.UDFSubstr \n" + - " \n" + " \n" + " org.apache.hadoop.hive.ql.udf.UDFSubstr \n" + " \n" + @@ -1387,10 +1397,9 @@ " \n" + " \n" + " \n"; - Configuration conf = new Configuration(); - ExprNodeDesc expr = Utilities.deserializeExpression(exprStr, conf); + SearchArgumentImpl sarg = - (SearchArgumentImpl) SearchArgument.FACTORY.create(expr); + (SearchArgumentImpl) SearchArgument.FACTORY.create(getFuncDesc(exprStr)); List leaves = sarg.getLeaves(); assertEquals(3, leaves.size()); @@ -1428,15 +1437,15 @@ String exprStr = " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -1481,7 +1490,7 @@ " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -1542,7 +1551,7 @@ " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -1598,10 +1607,9 @@ " \n" + " \n" + "\n"; - Configuration conf = new Configuration(); - ExprNodeDesc expr = Utilities.deserializeExpression(exprStr, conf); + SearchArgumentImpl sarg = - (SearchArgumentImpl) SearchArgument.FACTORY.create(expr); + (SearchArgumentImpl) SearchArgument.FACTORY.create(getFuncDesc(exprStr)); List leaves = sarg.getLeaves(); assertEquals(3, leaves.size()); @@ -1654,15 +1662,15 @@ String exprStr = " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -1707,7 +1715,7 @@ " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -1721,7 +1729,7 @@ " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -1754,9 +1762,6 @@ " \n" + " \n" + " \n" + - " \n" + - " org.apache.hadoop.hive.ql.udf.UDFSubstr \n" + - " \n" + " \n" + " org.apache.hadoop.hive.ql.udf.UDFSubstr \n" + " \n" + @@ -1792,7 +1797,7 @@ " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -1857,10 +1862,9 @@ " \n" + " \n" + " \n"; - Configuration conf = new Configuration(); - ExprNodeDesc expr = Utilities.deserializeExpression(exprStr, conf); + SearchArgumentImpl sarg = - (SearchArgumentImpl) SearchArgument.FACTORY.create(expr); + (SearchArgumentImpl) SearchArgument.FACTORY.create(getFuncDesc(exprStr)); List leaves = sarg.getLeaves(); assertEquals(1, leaves.size()); @@ -1884,27 +1888,27 @@ String exprStr = " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -1949,7 +1953,7 @@ " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -1996,7 +2000,7 @@ " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -2043,15 +2047,15 @@ " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -2088,7 +2092,7 @@ " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -2135,7 +2139,7 @@ " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -2192,11 +2196,11 @@ " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -2233,7 +2237,7 @@ " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -2290,7 +2294,7 @@ " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -2335,10 +2339,9 @@ " \n" + " \n" + ""; - Configuration conf = new Configuration(); - ExprNodeDesc expr = Utilities.deserializeExpression(exprStr, conf); + SearchArgumentImpl sarg = - (SearchArgumentImpl) SearchArgument.FACTORY.create(expr); + (SearchArgumentImpl) SearchArgument.FACTORY.create(getFuncDesc(exprStr)); List leaves = sarg.getLeaves(); assertEquals(9, leaves.size()); @@ -2424,7 +2427,7 @@ String exprStr = " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -2470,10 +2473,9 @@ " \n" + " \n" + " "; - Configuration conf = new Configuration(); - ExprNodeDesc expr = Utilities.deserializeExpression(exprStr, conf); + SearchArgumentImpl sarg = - (SearchArgumentImpl) SearchArgument.FACTORY.create(expr); + (SearchArgumentImpl) SearchArgument.FACTORY.create(getFuncDesc(exprStr)); List leaves = sarg.getLeaves(); assertEquals(0, leaves.size()); @@ -2487,7 +2489,7 @@ String exprStr = " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -2508,11 +2510,11 @@ " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -2541,9 +2543,6 @@ " \n" + " true \n" + " \n" + - " \n" + - " org.apache.hadoop.hive.ql.udf.UDFOPPlus \n" + - " \n" + " \n" + " org.apache.hadoop.hive.ql.udf.UDFOPPlus \n" + " \n" + @@ -2574,9 +2573,6 @@ " \n" + " true \n" + " \n" + - " \n" + - " org.apache.hadoop.hive.ql.udf.UDFOPPlus \n" + - " \n" + " \n" + " org.apache.hadoop.hive.ql.udf.UDFOPPlus \n" + " \n" + @@ -2604,10 +2600,9 @@ " \n" + " \n" + " "; - Configuration conf = new Configuration(); - ExprNodeDesc expr = Utilities.deserializeExpression(exprStr, conf); + SearchArgumentImpl sarg = - (SearchArgumentImpl) SearchArgument.FACTORY.create(expr); + (SearchArgumentImpl) SearchArgument.FACTORY.create(getFuncDesc(exprStr)); List leaves = sarg.getLeaves(); assertEquals(0, leaves.size()); @@ -2622,11 +2617,11 @@ String exprStr = " \n" + " \n"+ " \n"+ - " \n"+ + " \n"+ " \n"+ " \n"+ " \n"+ - " \n"+ + " \n"+ " \n"+ " \n"+ " \n"+ @@ -2671,11 +2666,11 @@ " \n"+ " \n"+ " \n"+ - " \n"+ + " \n"+ " \n"+ " \n"+ " \n"+ - " \n"+ + " \n"+ " \n"+ " \n"+ " \n"+ @@ -2730,10 +2725,9 @@ " \n"+ " \n"+ ""; - Configuration conf = new Configuration(); - ExprNodeDesc expr = Utilities.deserializeExpression(exprStr, conf); + SearchArgumentImpl sarg = - (SearchArgumentImpl) SearchArgument.FACTORY.create(expr); + (SearchArgumentImpl) SearchArgument.FACTORY.create(getFuncDesc(exprStr)); List leaves = sarg.getLeaves(); assertEquals(1, leaves.size()); Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java (revision 1528446) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java (working copy) @@ -26,7 +26,7 @@ import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; /** * The basic implementation of PartitionExpressionProxy that uses ql package classes. @@ -42,7 +42,7 @@ @Override public boolean filterPartitionsByExpr(List columnNames, byte[] exprBytes, String defaultPartitionName, List partitionNames) throws MetaException { - ExprNodeDesc expr = deserializeExpr(exprBytes); + ExprNodeGenericFuncDesc expr = deserializeExpr(exprBytes); try { long startTime = System.nanoTime(), len = partitionNames.size(); boolean result = PartitionPruner.prunePartitionNames( @@ -56,8 +56,8 @@ } } - private ExprNodeDesc deserializeExpr(byte[] exprBytes) throws MetaException { - ExprNodeDesc expr = null; + private ExprNodeGenericFuncDesc deserializeExpr(byte[] exprBytes) throws MetaException { + ExprNodeGenericFuncDesc expr = null; try { expr = Utilities.deserializeExpressionFromKryo(exprBytes); } catch (Exception ex) { Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java (revision 1528446) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java (working copy) @@ -21,12 +21,10 @@ import java.util.AbstractSequentialList; import java.util.ArrayList; import java.util.Iterator; -import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -40,7 +38,6 @@ import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.TableScanOperator; -import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.log.PerfLogger; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -192,7 +189,8 @@ if (((ExprNodeConstantDesc)expr).getValue() == null) { return null; } else { - return expr; + throw new IllegalStateException("Unexpected non-null ExprNodeConstantDesc: " + + expr.getExprString()); } } else if (expr instanceof ExprNodeGenericFuncDesc) { GenericUDF udf = ((ExprNodeGenericFuncDesc)expr).getGenericUDF(); @@ -213,9 +211,10 @@ return isAnd ? children.get(0) : null; } } - return expr; + return (ExprNodeGenericFuncDesc)expr; + } else { + throw new IllegalStateException("Unexpected type of ExprNodeDesc: " + expr.getExprString()); } - return expr; } /** @@ -247,10 +246,16 @@ * @return True iff expr contains any non-native user-defined functions. */ static private boolean hasUserFunctions(ExprNodeDesc expr) { - if (!(expr instanceof ExprNodeGenericFuncDesc)) return false; - if (!FunctionRegistry.isNativeFuncExpr((ExprNodeGenericFuncDesc)expr)) return true; + if (!(expr instanceof ExprNodeGenericFuncDesc)) { + return false; + } + if (!FunctionRegistry.isNativeFuncExpr((ExprNodeGenericFuncDesc)expr)) { + return true; + } for (ExprNodeDesc child : expr.getChildren()) { - if (hasUserFunctions(child)) return true; + if (hasUserFunctions(child)) { + return true; + } } return false; } @@ -279,7 +284,7 @@ // Replace virtual columns with nulls. See javadoc for details. prunerExpr = removeNonPartCols(prunerExpr, extractPartColNames(tab)); // Remove all parts that are not partition columns. See javadoc for details. - ExprNodeDesc compactExpr = compactExpr(prunerExpr.clone()); + ExprNodeGenericFuncDesc compactExpr = (ExprNodeGenericFuncDesc)compactExpr(prunerExpr.clone()); String oldFilter = prunerExpr.getExprString(); if (compactExpr == null) { // Non-strict mode, and all the predicates are on non-partition columns - get everything. @@ -338,7 +343,7 @@ * @return true iff the partition pruning expression contains non-partition columns. */ static private boolean pruneBySequentialScan(Table tab, List partitions, - ExprNodeDesc prunerExpr, HiveConf conf) throws HiveException, MetaException { + ExprNodeGenericFuncDesc prunerExpr, HiveConf conf) throws HiveException, MetaException { PerfLogger perfLogger = PerfLogger.getPerfLogger(); perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.PRUNE_LISTING); @@ -377,7 +382,7 @@ * @param partNames Partition names to filter. The list is modified in place. * @return Whether the list has any partitions for which the expression may or may not match. */ - public static boolean prunePartitionNames(List columnNames, ExprNodeDesc prunerExpr, + public static boolean prunePartitionNames(List columnNames, ExprNodeGenericFuncDesc prunerExpr, String defaultPartitionName, List partNames) throws HiveException, MetaException { // Prepare the expression to filter on the columns. ObjectPair handle = Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartExprEvalUtils.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartExprEvalUtils.java (revision 1528446) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartExprEvalUtils.java (working copy) @@ -29,6 +29,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; @@ -84,7 +85,7 @@ } static synchronized public ObjectPair prepareExpr( - ExprNodeDesc expr, List partNames) throws HiveException { + ExprNodeGenericFuncDesc expr, List partNames) throws HiveException { // Create the row object List partObjectInspectors = new ArrayList(); for (int i = 0; i < partNames.size(); i++) { Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcFactory.java (revision 1528446) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcFactory.java (working copy) @@ -209,7 +209,7 @@ children.add(wrapper.outExpr); } } - funcExpr.setChildExprs(children); + funcExpr.setChildren(children); return funcExpr; } Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyProcFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyProcFactory.java (revision 1528446) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyProcFactory.java (working copy) @@ -176,7 +176,7 @@ canApplyCtx.getGbKeyNameList().addAll(expr.getCols()); }else if(expr instanceof ExprNodeGenericFuncDesc){ ExprNodeGenericFuncDesc funcExpr = (ExprNodeGenericFuncDesc)expr; - List childExprs = funcExpr.getChildExprs(); + List childExprs = funcExpr.getChildren(); for (ExprNodeDesc childExpr : childExprs) { if(childExpr instanceof ExprNodeColumnDesc){ canApplyCtx.getGbKeyNameList().addAll(expr.getCols()); Index: ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java (revision 1528446) +++ ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java (working copy) @@ -171,7 +171,7 @@ ExprNodeDesc ch = (ExprNodeDesc) nd.getChildren().get(i); ExprNodeDesc newCh = ctx.getConvertedNode(ch); if (newCh != null) { - expr.getChildExprs().set(i, newCh); + expr.getChildren().set(i, newCh); ch = newCh; } String chAlias = ctx.getAlias(ch); Index: ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java (revision 1528446) +++ ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java (working copy) @@ -55,6 +55,7 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.plan.FilterDesc; import org.apache.hadoop.hive.ql.plan.JoinCondDesc; import org.apache.hadoop.hive.ql.plan.JoinDesc; @@ -695,6 +696,9 @@ } ExprNodeDesc condn = ExprNodeDescUtils.mergePredicates(preds); + if(!(condn instanceof ExprNodeGenericFuncDesc)) { + return null; + } if (op instanceof TableScanOperator) { boolean pushFilterToStorage; @@ -704,7 +708,7 @@ if (pushFilterToStorage) { condn = pushFilterToStorageHandler( (TableScanOperator) op, - condn, + (ExprNodeGenericFuncDesc)condn, owi, hiveConf); if (condn == null) { @@ -769,9 +773,9 @@ * by Hive as a post-filter, or null if it was possible * to push down the entire predicate */ - private static ExprNodeDesc pushFilterToStorageHandler( + private static ExprNodeGenericFuncDesc pushFilterToStorageHandler( TableScanOperator tableScanOp, - ExprNodeDesc originalPredicate, + ExprNodeGenericFuncDesc originalPredicate, OpWalkerInfo owi, HiveConf hiveConf) { @@ -832,7 +836,7 @@ } } tableScanDesc.setFilterExpr(decomposed.pushedPredicate); - return decomposed.residualPredicate; + return (ExprNodeGenericFuncDesc)decomposed.residualPredicate; } public static NodeProcessor getFilterProc() { Index: ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java (revision 1528446) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java (working copy) @@ -61,7 +61,7 @@ * exactly what we want. */ private GenericUDF genericUDF; - private List childExprs; + private List chidren; private transient String funcText; /** * This class uses a writableObjectInspector rather than a TypeInfo to store @@ -93,7 +93,7 @@ ObjectInspectorUtils.getWritableObjectInspector(oi); assert (genericUDF != null); this.genericUDF = genericUDF; - this.childExprs = children; + this.chidren = children; this.funcText = funcText; } @@ -121,17 +121,13 @@ this.genericUDF = genericUDF; } - public List getChildExprs() { - return childExprs; + public void setChildren(List children) { + chidren = children; } - public void setChildExprs(List children) { - childExprs = children; - } - @Override public List getChildren() { - return childExprs; + return chidren; } @Override @@ -139,11 +135,11 @@ StringBuilder sb = new StringBuilder(); sb.append(genericUDF.getClass().toString()); sb.append("("); - for (int i = 0; i < childExprs.size(); i++) { + for (int i = 0; i < chidren.size(); i++) { if (i > 0) { sb.append(", "); } - sb.append(childExprs.get(i).toString()); + sb.append(chidren.get(i).toString()); } sb.append("("); sb.append(")"); @@ -154,9 +150,9 @@ @Override public String getExprString() { // Get the children expr strings - String[] childrenExprStrings = new String[childExprs.size()]; + String[] childrenExprStrings = new String[chidren.size()]; for (int i = 0; i < childrenExprStrings.length; i++) { - childrenExprStrings[i] = childExprs.get(i).getExprString(); + childrenExprStrings[i] = chidren.get(i).getExprString(); } return genericUDF.getDisplayString(childrenExprStrings); @@ -165,10 +161,10 @@ @Override public List getCols() { List colList = new ArrayList(); - if (childExprs != null) { + if (chidren != null) { int pos = 0; - while (pos < childExprs.size()) { - List colCh = childExprs.get(pos).getCols(); + while (pos < chidren.size()) { + List colCh = chidren.get(pos).getCols(); colList = Utilities.mergeUniqElems(colList, colCh); pos++; } @@ -179,8 +175,8 @@ @Override public ExprNodeDesc clone() { - List cloneCh = new ArrayList(childExprs.size()); - for (ExprNodeDesc ch : childExprs) { + List cloneCh = new ArrayList(chidren.size()); + for (ExprNodeDesc ch : chidren) { cloneCh.add(ch.clone()); } ExprNodeGenericFuncDesc clone = new ExprNodeGenericFuncDesc(typeInfo, @@ -286,12 +282,12 @@ } } - if (childExprs.size() != dest.getChildExprs().size()) { + if (chidren.size() != dest.getChildren().size()) { return false; } - for (int pos = 0; pos < childExprs.size(); pos++) { - if (!childExprs.get(pos).isSame(dest.getChildExprs().get(pos))) { + for (int pos = 0; pos < chidren.size(); pos++) { + if (!chidren.get(pos).isSame(dest.getChildren().get(pos))) { return false; } } @@ -304,7 +300,7 @@ int superHashCode = super.hashCode(); HashCodeBuilder builder = new HashCodeBuilder(); builder.appendSuper(superHashCode); - builder.append(childExprs); + builder.append(chidren); return builder.toHashCode(); } Index: ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java (revision 1528446) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java (working copy) @@ -68,7 +68,7 @@ } // duplicate function with possibily replaced children ExprNodeGenericFuncDesc clone = (ExprNodeGenericFuncDesc) func.clone(); - clone.setChildExprs(children); + clone.setChildren(children); return clone; } // constant or null, just return it @@ -94,7 +94,7 @@ /** * bind two predicates by AND op */ - public static ExprNodeDesc mergePredicates(ExprNodeDesc prev, ExprNodeDesc next) { + public static ExprNodeGenericFuncDesc mergePredicates(ExprNodeDesc prev, ExprNodeDesc next) { List children = new ArrayList(2); children.add(prev); children.add(next); @@ -196,7 +196,7 @@ if (source instanceof ExprNodeGenericFuncDesc) { // all children expression should be resolved ExprNodeGenericFuncDesc function = (ExprNodeGenericFuncDesc) source.clone(); - function.setChildExprs(backtrack(function.getChildren(), current, terminal)); + function.setChildren(backtrack(function.getChildren(), current, terminal)); return function; } if (source instanceof ExprNodeColumnDesc) { Index: ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java (revision 1528446) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java (working copy) @@ -60,7 +60,7 @@ private boolean statsReliable; private int maxStatsKeyPrefixLength = -1; - private ExprNodeDesc filterExpr; + private ExprNodeGenericFuncDesc filterExpr; public static final String FILTER_EXPR_CONF_STR = "hive.io.filter.expr.serialized"; @@ -96,11 +96,11 @@ } @Explain(displayName = "filterExpr") - public ExprNodeDesc getFilterExpr() { + public ExprNodeGenericFuncDesc getFilterExpr() { return filterExpr; } - public void setFilterExpr(ExprNodeDesc filterExpr) { + public void setFilterExpr(ExprNodeGenericFuncDesc filterExpr) { this.filterExpr = filterExpr; } Index: ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (revision 1528446) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (working copy) @@ -4322,7 +4322,7 @@ if (exprNode instanceof ExprNodeGenericFuncDesc) { ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) exprNode; - for (ExprNodeDesc childExpr : funcDesc.getChildExprs()) { + for (ExprNodeDesc childExpr : funcDesc.getChildren()) { extractColumns(colNamesExprs, childExpr); } } Index: ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStoragePredicateHandler.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStoragePredicateHandler.java (revision 1528446) +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStoragePredicateHandler.java (working copy) @@ -18,9 +18,10 @@ package org.apache.hadoop.hive.ql.metadata; -import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.serde2.Deserializer; +import org.apache.hadoop.mapred.JobConf; /** * HiveStoragePredicateHandler is an optional companion to {@link @@ -65,12 +66,12 @@ * Portion of predicate to be evaluated by storage handler. Hive * will pass this into the storage handler's input format. */ - public ExprNodeDesc pushedPredicate; + public ExprNodeGenericFuncDesc pushedPredicate; /** * Portion of predicate to be post-evaluated by Hive for any rows * which are returned by storage handler. */ - public ExprNodeDesc residualPredicate; + public ExprNodeGenericFuncDesc residualPredicate; } } Index: ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (revision 1528446) +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (working copy) @@ -81,7 +81,7 @@ import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.index.HiveIndexHandler; import org.apache.hadoop.hive.ql.optimizer.listbucketingpruner.ListBucketingPrunerUtils; -import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.session.CreateTableAutomaticGrant; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.serde2.Deserializer; @@ -1927,7 +1927,7 @@ * @param result the resulting list of partitions * @return whether the resulting list contains partitions which may or may not match the expr */ - public boolean getPartitionsByExpr(Table tbl, ExprNodeDesc expr, HiveConf conf, + public boolean getPartitionsByExpr(Table tbl, ExprNodeGenericFuncDesc expr, HiveConf conf, List result) throws HiveException, TException { assert result != null; byte[] exprBytes = Utilities.serializeExpressionToKryo(expr); Index: ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java (revision 1528446) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java (working copy) @@ -83,10 +83,10 @@ public ExprNodeGenericFuncEvaluator(ExprNodeGenericFuncDesc expr) throws HiveException { super(expr); - children = new ExprNodeEvaluator[expr.getChildExprs().size()]; + children = new ExprNodeEvaluator[expr.getChildren().size()]; isEager = false; for (int i = 0; i < children.length; i++) { - ExprNodeDesc child = expr.getChildExprs().get(i); + ExprNodeDesc child = expr.getChildren().get(i); ExprNodeEvaluator nodeEvaluator = ExprNodeEvaluatorFactory.get(child); children[i] = nodeEvaluator; // If we have eager evaluators anywhere below us, then we are eager too. Index: ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java (revision 1528446) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java (working copy) @@ -272,7 +272,7 @@ ve = getCustomUDFExpression(expr); } else { ve = getVectorExpression(expr.getGenericUDF(), - expr.getChildExprs()); + expr.getChildren()); } } else if (exprDesc instanceof ExprNodeConstantDesc) { ve = getConstantVectorExpression((ExprNodeConstantDesc) exprDesc); @@ -622,10 +622,10 @@ throws HiveException { //GenericUDFBridge udfBridge = (GenericUDFBridge) expr.getGenericUDF(); - List childExprList = expr.getChildExprs(); + List childExprList = expr.getChildren(); // argument descriptors - VectorUDFArgDesc[] argDescs = new VectorUDFArgDesc[expr.getChildExprs().size()]; + VectorUDFArgDesc[] argDescs = new VectorUDFArgDesc[expr.getChildren().size()]; for (int i = 0; i < argDescs.length; i++) { argDescs[i] = new VectorUDFArgDesc(); } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFAdaptor.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFAdaptor.java (revision 1528446) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFAdaptor.java (working copy) @@ -92,9 +92,9 @@ // Initialize transient fields. To be called after deserialization of other fields. public void init() throws HiveException, UDFArgumentException { genericUDF = expr.getGenericUDF(); - deferredChildren = new GenericUDF.DeferredObject[expr.getChildExprs().size()]; - childrenOIs = new ObjectInspector[expr.getChildExprs().size()]; - writers = VectorExpressionWriterFactory.getExpressionWriters(expr.getChildExprs()); + deferredChildren = new GenericUDF.DeferredObject[expr.getChildren().size()]; + childrenOIs = new ObjectInspector[expr.getChildren().size()]; + writers = VectorExpressionWriterFactory.getExpressionWriters(expr.getChildren()); for (int i = 0; i < childrenOIs.length; i++) { childrenOIs[i] = writers[i].getObjectInspector(); } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (revision 1528446) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (working copy) @@ -80,6 +80,7 @@ import java.util.regex.Pattern; import org.antlr.runtime.CommonToken; +import org.apache.commons.codec.binary.Base64; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.WordUtils; import org.apache.commons.logging.Log; @@ -99,7 +100,6 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.ql.Context; -import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryPlan; import org.apache.hadoop.hive.ql.exec.mr.ExecDriver; @@ -129,7 +129,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.BaseWork; import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx; -import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.plan.FileSinkDesc; import org.apache.hadoop.hive.ql.plan.GroupByDesc; import org.apache.hadoop.hive.ql.plan.MapWork; @@ -582,10 +582,10 @@ * @param expr Expression. * @return Bytes. */ - public static byte[] serializeExpressionToKryo(ExprNodeDesc expr) { + public static byte[] serializeExpressionToKryo(ExprNodeGenericFuncDesc expr) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); Output output = new Output(baos); - runtimeSerializationKryo.get().writeClassAndObject(output, expr); + runtimeSerializationKryo.get().writeObject(output, expr); output.close(); return baos.toByteArray(); } @@ -595,47 +595,30 @@ * @param bytes Bytes containing the expression. * @return Expression; null if deserialization succeeded, but the result type is incorrect. */ - public static ExprNodeDesc deserializeExpressionFromKryo(byte[] bytes) { + public static ExprNodeGenericFuncDesc deserializeExpressionFromKryo(byte[] bytes) { Input inp = new Input(new ByteArrayInputStream(bytes)); - Object o = runtimeSerializationKryo.get().readClassAndObject(inp); + ExprNodeGenericFuncDesc func = runtimeSerializationKryo.get(). + readObject(inp,ExprNodeGenericFuncDesc.class); inp.close(); - return (o instanceof ExprNodeDesc) ? (ExprNodeDesc)o : null; + return func; } - public static String serializeExpression(ExprNodeDesc expr) { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - XMLEncoder encoder = new XMLEncoder(baos); - encoder.setPersistenceDelegate(java.sql.Date.class, new DatePersistenceDelegate()); - encoder.setPersistenceDelegate(Timestamp.class, new TimestampPersistenceDelegate()); + public static String serializeExpression(ExprNodeGenericFuncDesc expr) { try { - encoder.writeObject(expr); - } finally { - encoder.close(); - } - try { - return baos.toString("UTF-8"); + return new String(Base64.encodeBase64(serializeExpressionToKryo(expr)), "UTF-8"); } catch (UnsupportedEncodingException ex) { throw new RuntimeException("UTF-8 support required", ex); } } - public static ExprNodeDesc deserializeExpression(String s, Configuration conf) { + public static ExprNodeGenericFuncDesc deserializeExpression(String s) { byte[] bytes; try { - bytes = s.getBytes("UTF-8"); + bytes = Base64.decodeBase64(s.getBytes("UTF-8")); } catch (UnsupportedEncodingException ex) { throw new RuntimeException("UTF-8 support required", ex); } - - ByteArrayInputStream bais = new ByteArrayInputStream(bytes); - - XMLDecoder decoder = new XMLDecoder(bais, null, null); - try { - ExprNodeDesc expr = (ExprNodeDesc) decoder.readObject(); - return expr; - } finally { - decoder.close(); - } + return deserializeExpressionFromKryo(bytes); } public static class CollectionPersistenceDelegate extends DefaultPersistenceDelegate { @@ -653,6 +636,26 @@ } } + /** + * Kryo serializer for timestamp. + */ + private static class TimestampSerializer extends + com.esotericsoftware.kryo.Serializer { + + @Override + public Timestamp read(Kryo kryo, Input input, Class clazz) { + Timestamp ts = new Timestamp(input.readLong()); + ts.setNanos(input.readInt()); + return ts; + } + + @Override + public void write(Kryo kryo, Output output, Timestamp ts) { + output.writeLong(ts.getTime()); + output.writeInt(ts.getNanos()); + } + } + /** Custom Kryo serializer for sql date, otherwise Kryo gets confused between java.sql.Date and java.util.Date while deserializing */ @@ -822,6 +825,7 @@ Kryo kryo = new Kryo(); kryo.setClassLoader(Thread.currentThread().getContextClassLoader()); kryo.register(java.sql.Date.class, new SqlDateSerializer()); + kryo.register(java.sql.Timestamp.class, new TimestampSerializer()); removeField(kryo, Operator.class, "colExprMap"); removeField(kryo, ColumnInfo.class, "objectInspector"); removeField(kryo, MapWork.class, "opParseCtxMap"); @@ -842,6 +846,7 @@ kryo.setClassLoader(Thread.currentThread().getContextClassLoader()); kryo.register(CommonToken.class, new CommonTokenSerializer()); kryo.register(java.sql.Date.class, new SqlDateSerializer()); + kryo.register(java.sql.Timestamp.class, new TimestampSerializer()); return kryo; }; }; Index: ql/src/java/org/apache/hadoop/hive/ql/index/IndexSearchCondition.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/index/IndexSearchCondition.java (revision 1528446) +++ ql/src/java/org/apache/hadoop/hive/ql/index/IndexSearchCondition.java (working copy) @@ -19,7 +19,7 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; /** * IndexSearchCondition represents an individual search condition @@ -31,7 +31,7 @@ private ExprNodeColumnDesc columnDesc; private String comparisonOp; private ExprNodeConstantDesc constantDesc; - private ExprNodeDesc comparisonExpr; + private ExprNodeGenericFuncDesc comparisonExpr; /** * Constructs a search condition, which takes the form @@ -50,7 +50,7 @@ ExprNodeColumnDesc columnDesc, String comparisonOp, ExprNodeConstantDesc constantDesc, - ExprNodeDesc comparisonExpr) { + ExprNodeGenericFuncDesc comparisonExpr) { this.columnDesc = columnDesc; this.comparisonOp = comparisonOp; @@ -82,11 +82,11 @@ return constantDesc; } - public void setComparisonExpr(ExprNodeDesc comparisonExpr) { + public void setComparisonExpr(ExprNodeGenericFuncDesc comparisonExpr) { this.comparisonExpr = comparisonExpr; } - public ExprNodeDesc getComparisonExpr() { + public ExprNodeGenericFuncDesc getComparisonExpr() { return comparisonExpr; } Index: ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java (revision 1528446) +++ ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java (working copy) @@ -316,7 +316,8 @@ IndexPredicateAnalyzer analyzer = getIndexPredicateAnalyzer(index, queryPartitions); List searchConditions = new ArrayList(); // split predicate into pushed (what we can handle), and residual (what we can't handle) - ExprNodeDesc residualPredicate = analyzer.analyzePredicate(predicate, searchConditions); + ExprNodeGenericFuncDesc residualPredicate = (ExprNodeGenericFuncDesc)analyzer. + analyzePredicate(predicate, searchConditions); if (searchConditions.size() == 0) { return null; Index: ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java (revision 1528446) +++ ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java (working copy) @@ -26,6 +26,8 @@ import java.util.Set; import java.util.Stack; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; @@ -44,12 +46,10 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.util.ReflectionUtils; /** @@ -62,7 +62,7 @@ public class IndexPredicateAnalyzer { private static final Log LOG = LogFactory.getLog(IndexPredicateAnalyzer.class.getName()); - private Set udfNames; + private final Set udfNames; private Set allowedColumnNames; @@ -135,7 +135,7 @@ } } - return analyzeExpr((ExprNodeDesc) nd, searchConditions, nodeOutputs); + return analyzeExpr((ExprNodeGenericFuncDesc) nd, searchConditions, nodeOutputs); } }; @@ -155,13 +155,11 @@ } private ExprNodeDesc analyzeExpr( - ExprNodeDesc expr, + ExprNodeGenericFuncDesc expr, List searchConditions, Object... nodeOutputs) { - if (!(expr instanceof ExprNodeGenericFuncDesc)) { - return expr; - } + expr = (ExprNodeGenericFuncDesc) expr; if (FunctionRegistry.isOpAnd(expr)) { assert(nodeOutputs.length == 2); ExprNodeDesc residual1 = (ExprNodeDesc) nodeOutputs[0]; @@ -182,12 +180,11 @@ } String udfName; - ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) expr; - if (funcDesc.getGenericUDF() instanceof GenericUDFBridge) { - GenericUDFBridge func = (GenericUDFBridge) funcDesc.getGenericUDF(); + if (expr.getGenericUDF() instanceof GenericUDFBridge) { + GenericUDFBridge func = (GenericUDFBridge) expr.getGenericUDF(); udfName = func.getUdfName(); } else { - udfName = funcDesc.getGenericUDF().getClass().getName(); + udfName = expr.getGenericUDF().getClass().getName(); } if (!udfNames.contains(udfName)) { return expr; @@ -255,7 +252,7 @@ } } - for (ExprNodeDesc child : func.getChildExprs()) { + for (ExprNodeDesc child : func.getChildren()) { if (child instanceof ExprNodeConstantDesc) { continue; } else if (child instanceof ExprNodeGenericFuncDesc) { @@ -283,12 +280,12 @@ * * @param searchConditions (typically produced by analyzePredicate) * - * @return ExprNodeDesc form of search conditions + * @return ExprNodeGenericFuncDesc form of search conditions */ - public ExprNodeDesc translateSearchConditions( + public ExprNodeGenericFuncDesc translateSearchConditions( List searchConditions) { - ExprNodeDesc expr = null; + ExprNodeGenericFuncDesc expr = null; for (IndexSearchCondition searchCondition : searchConditions) { if (expr == null) { expr = searchCondition.getComparisonExpr(); Index: ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java (revision 1528446) +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java (working copy) @@ -109,7 +109,7 @@ boolean[] includeColumn = findIncludedColumns(types, conf); if (serializedPushdown != null && columnNamesString != null) { sarg = SearchArgument.FACTORY.create - (Utilities.deserializeExpression(serializedPushdown, conf)); + (Utilities.deserializeExpression(serializedPushdown)); LOG.info("ORC pushdown predicate: " + sarg); String[] neededColumnNames = columnNamesString.split(","); int i = 0; Index: ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgument.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgument.java (revision 1528446) +++ ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgument.java (working copy) @@ -18,10 +18,10 @@ package org.apache.hadoop.hive.ql.io.sarg; -import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; - import java.util.List; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; + /** * Primary interface for * SearchArgument, which are the subset of predicates @@ -170,7 +170,7 @@ * in interfaces. *DOH* */ public static class Factory { - public SearchArgument create(ExprNodeDesc expression) { + public SearchArgument create(ExprNodeGenericFuncDesc expression) { return new SearchArgumentImpl(expression); } Index: ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java (revision 1528446) +++ ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java (working copy) @@ -18,6 +18,14 @@ package org.apache.hadoop.hive.ql.io.sarg; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; @@ -40,14 +48,6 @@ import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Deque; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - /** * The implementation of SearchArguments. */ @@ -221,6 +221,7 @@ } } + @Override public String toString() { StringBuilder buffer = new StringBuilder(); switch (operator) { @@ -471,49 +472,49 @@ return new ExpressionTree(TruthValue.YES_NO_NULL); } // get the kind of expression - ExprNodeGenericFuncDesc typed = (ExprNodeGenericFuncDesc) expression; - Class op = typed.getGenericUDF().getClass(); + ExprNodeGenericFuncDesc expr = (ExprNodeGenericFuncDesc) expression; + Class op = expr.getGenericUDF().getClass(); ExpressionTree result; // handle the logical operators if (op == GenericUDFOPOr.class) { result = new ExpressionTree(ExpressionTree.Operator.OR); - addChildren(result, typed, leafCache); + addChildren(result, expr, leafCache); } else if (op == GenericUDFOPAnd.class) { result = new ExpressionTree(ExpressionTree.Operator.AND); - addChildren(result, typed, leafCache); + addChildren(result, expr, leafCache); } else if (op == GenericUDFOPNot.class) { result = new ExpressionTree(ExpressionTree.Operator.NOT); - addChildren(result, typed, leafCache); + addChildren(result, expr, leafCache); } else if (op == GenericUDFOPEqual.class) { - result = createLeaf(PredicateLeaf.Operator.EQUALS, typed, leafCache); + result = createLeaf(PredicateLeaf.Operator.EQUALS, expr, leafCache); } else if (op == GenericUDFOPNotEqual.class) { - result = negate(createLeaf(PredicateLeaf.Operator.EQUALS, typed, + result = negate(createLeaf(PredicateLeaf.Operator.EQUALS, expr, leafCache)); } else if (op == GenericUDFOPEqualNS.class) { - result = createLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, typed, + result = createLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, expr, leafCache); } else if (op == GenericUDFOPGreaterThan.class) { result = negate(createLeaf(PredicateLeaf.Operator.LESS_THAN_EQUALS, - typed, leafCache)); + expr, leafCache)); } else if (op == GenericUDFOPEqualOrGreaterThan.class) { - result = negate(createLeaf(PredicateLeaf.Operator.LESS_THAN, typed, + result = negate(createLeaf(PredicateLeaf.Operator.LESS_THAN, expr, leafCache)); } else if (op == GenericUDFOPLessThan.class) { - result = createLeaf(PredicateLeaf.Operator.LESS_THAN, typed, leafCache); + result = createLeaf(PredicateLeaf.Operator.LESS_THAN, expr, leafCache); } else if (op == GenericUDFOPEqualOrLessThan.class) { - result = createLeaf(PredicateLeaf.Operator.LESS_THAN_EQUALS, typed, + result = createLeaf(PredicateLeaf.Operator.LESS_THAN_EQUALS, expr, leafCache); } else if (op == GenericUDFIn.class) { - result = createLeaf(PredicateLeaf.Operator.IN, typed, leafCache, 0); + result = createLeaf(PredicateLeaf.Operator.IN, expr, leafCache, 0); } else if (op == GenericUDFBetween.class) { - result = createLeaf(PredicateLeaf.Operator.BETWEEN, typed, leafCache, + result = createLeaf(PredicateLeaf.Operator.BETWEEN, expr, leafCache, 1); } else if (op == GenericUDFOPNull.class) { - result = createLeaf(PredicateLeaf.Operator.IS_NULL, typed, leafCache, + result = createLeaf(PredicateLeaf.Operator.IS_NULL, expr, leafCache, 0); } else if (op == GenericUDFOPNotNull.class) { - result = negate(createLeaf(PredicateLeaf.Operator.IS_NULL, typed, + result = negate(createLeaf(PredicateLeaf.Operator.IS_NULL, expr, leafCache, 0)); // otherwise, we didn't understand it, so mark it maybe @@ -753,7 +754,7 @@ * @param expression the expression to translate * @return The normalized expression. */ - ExpressionTree expression(ExprNodeDesc expression) { + ExpressionTree expression(ExprNodeGenericFuncDesc expression) { List leafCache = new ArrayList(); ExpressionTree expr = parse(expression, leafCache); return expression(expr, leafCache); @@ -786,7 +787,7 @@ private final List leaves; private final ExpressionTree expression; - SearchArgumentImpl(ExprNodeDesc expr) { + SearchArgumentImpl(ExprNodeGenericFuncDesc expr) { if (expr == null) { leaves = new ArrayList(); expression = null; Index: ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java (revision 1528446) +++ ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java (working copy) @@ -37,7 +37,7 @@ import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.log.PerfLogger; -import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.plan.MapWork; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.PartitionDesc; @@ -347,7 +347,7 @@ Utilities.setColumnNameList(jobConf, tableScan); Utilities.setColumnTypeList(jobConf, tableScan); // push down filters - ExprNodeDesc filterExpr = scanDesc.getFilterExpr(); + ExprNodeGenericFuncDesc filterExpr = (ExprNodeGenericFuncDesc)scanDesc.getFilterExpr(); if (filterExpr == null) { return; }