Index: ql/src/test/results/clientnegative/udf_size_wrong_args_len.q.out =================================================================== --- ql/src/test/results/clientnegative/udf_size_wrong_args_len.q.out (revision 790697) +++ ql/src/test/results/clientnegative/udf_size_wrong_args_len.q.out (working copy) @@ -1 +1 @@ -FAILED: Error in semantic analysis: line 2:7 Arguments Length Mismatch lintstring: The functin SIZE only accepts 1 argument. +FAILED: Error in semantic analysis: line 2:7 Arguments Length Mismatch lintstring: The function SIZE only accepts 1 argument. Index: ql/src/test/results/compiler/plan/input_testxpath.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input_testxpath.q.xml (revision 790697) +++ ql/src/test/results/compiler/plan/input_testxpath.q.xml (working copy) @@ -68,7 +68,7 @@ location - file:/data/users/njain/hive1/hive1/build/ql/test/data/warehouse/src_thrift + file:/data/users/zshao/tools/deploy-trunk-apache-hive/build/ql/test/data/warehouse/src_thrift @@ -96,7 +96,7 @@ - file:/data/users/njain/hive1/hive1/build/ql/tmp/1178517652/10001 + file:/data/users/zshao/tools/deploy-trunk-apache-hive/build/ql/tmp/1490863661/10001 @@ -190,33 +190,40 @@ _col2 - - - - - mstringstring + + + + + + + mstringstring + + + + + + + + + + + + - - - + + + - - + + key_2 - - - - - - - key_2 - - + + org.apache.hadoop.hive.ql.udf.generic.GenericUDFIndex @@ -227,30 +234,37 @@ _col1 - - - - - lintstring + + + + + + + lintstring + + + + + + + + + - - - - + + + + + + 0 + - - - - - - - 0 - - + + org.apache.hadoop.hive.ql.udf.generic.GenericUDFIndex @@ -270,30 +284,37 @@ _col0 - - - - - lint + + + + + + + lint + + + + + + + + + - - - + + + + + 1 + - - - - - - - 1 - - + + org.apache.hadoop.hive.ql.udf.generic.GenericUDFIndex @@ -307,13 +328,13 @@ - + - + @@ -460,7 +481,7 @@ - file:/data/users/njain/hive1/hive1/build/ql/test/data/warehouse/src_thrift + file:/data/users/zshao/tools/deploy-trunk-apache-hive/build/ql/test/data/warehouse/src_thrift src_thrift @@ -472,7 +493,7 @@ - file:/data/users/njain/hive1/hive1/build/ql/test/data/warehouse/src_thrift + file:/data/users/zshao/tools/deploy-trunk-apache-hive/build/ql/test/data/warehouse/src_thrift Index: ql/src/test/results/compiler/plan/case_sensitivity.q.xml =================================================================== --- ql/src/test/results/compiler/plan/case_sensitivity.q.xml (revision 790697) +++ ql/src/test/results/compiler/plan/case_sensitivity.q.xml (working copy) @@ -30,7 +30,7 @@ true - file:/data/users/pchakka/workspace/oshive/build/ql/tmp/275561524/10000 + file:/data/users/zshao/tools/deploy-trunk-apache-hive/build/ql/tmp/1210903944/10000 @@ -83,7 +83,7 @@ location - file:/data/users/pchakka/workspace/oshive/build/ql/test/data/warehouse/dest1 + file:/data/users/zshao/tools/deploy-trunk-apache-hive/build/ql/test/data/warehouse/dest1 @@ -93,7 +93,7 @@ - file:/data/users/pchakka/workspace/oshive/build/ql/tmp/275561524/10001 + file:/data/users/zshao/tools/deploy-trunk-apache-hive/build/ql/tmp/1210903944/10001 @@ -121,10 +121,10 @@ true - file:/data/users/pchakka/workspace/oshive/build/ql/tmp/909472698/10002 + file:/data/users/zshao/tools/deploy-trunk-apache-hive/build/ql/tmp/1325032988/10002 - file:/data/users/pchakka/workspace/oshive/build/ql/tmp/275561524/10000 + file:/data/users/zshao/tools/deploy-trunk-apache-hive/build/ql/tmp/1210903944/10000 @@ -142,7 +142,7 @@ - file:/data/users/pchakka/workspace/oshive/build/ql/tmp/909472698/10002 + file:/data/users/zshao/tools/deploy-trunk-apache-hive/build/ql/tmp/1325032988/10002 @@ -342,10 +342,10 @@ - file:/data/users/pchakka/workspace/oshive/build/ql/tmp/909472698/10002 + file:/data/users/zshao/tools/deploy-trunk-apache-hive/build/ql/tmp/1325032988/10002 - file:/data/users/pchakka/workspace/oshive/build/ql/tmp/909472698/10002 + file:/data/users/zshao/tools/deploy-trunk-apache-hive/build/ql/tmp/1325032988/10002 @@ -354,7 +354,7 @@ - file:/data/users/pchakka/workspace/oshive/build/ql/tmp/909472698/10002 + file:/data/users/zshao/tools/deploy-trunk-apache-hive/build/ql/tmp/1325032988/10002 @@ -372,7 +372,7 @@ - file:/data/users/pchakka/workspace/oshive/build/ql/tmp/275561524/10000 + file:/data/users/zshao/tools/deploy-trunk-apache-hive/build/ql/tmp/1210903944/10000 @@ -549,7 +549,7 @@ location - file:/data/users/pchakka/workspace/oshive/build/ql/test/data/warehouse/src_thrift + file:/data/users/zshao/tools/deploy-trunk-apache-hive/build/ql/test/data/warehouse/src_thrift @@ -588,7 +588,7 @@ 1 - file:/data/users/pchakka/workspace/oshive/build/ql/tmp/909472698/10002 + file:/data/users/zshao/tools/deploy-trunk-apache-hive/build/ql/tmp/1325032988/10002 @@ -615,30 +615,37 @@ _col1 - - - - - lintstring + + + + + + + lintstring + + + + + + + + + - - - - + + + + + + 0 + - - - - - - - 0 - - + + org.apache.hadoop.hive.ql.udf.generic.GenericUDFIndex @@ -658,30 +665,37 @@ _col0 - - - - - lint + + + + + + + lint + + + + + + + + + - - - + + + + + 1 + - - - - - - - 1 - - + + org.apache.hadoop.hive.ql.udf.generic.GenericUDFIndex @@ -695,7 +709,7 @@ - + @@ -776,26 +790,33 @@ - - - - - lint + + + + + + + lint + + + + + - - + + + + + + + 0 + + - - - - - - - 0 - - + + org.apache.hadoop.hive.ql.udf.generic.GenericUDFIndex @@ -928,12 +949,33 @@ - - - + + + + + + + lint + + + + + + + + + + + + + 0 + + + + - - + + org.apache.hadoop.hive.ql.udf.generic.GenericUDFIndex @@ -991,7 +1033,7 @@ - file:/data/users/pchakka/workspace/oshive/build/ql/test/data/warehouse/src_thrift + file:/data/users/zshao/tools/deploy-trunk-apache-hive/build/ql/test/data/warehouse/src_thrift src_thrift @@ -1003,7 +1045,7 @@ - file:/data/users/pchakka/workspace/oshive/build/ql/test/data/warehouse/src_thrift + file:/data/users/zshao/tools/deploy-trunk-apache-hive/build/ql/test/data/warehouse/src_thrift Index: ql/src/test/org/apache/hadoop/hive/ql/exec/TestExpressionEvaluator.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/exec/TestExpressionEvaluator.java (revision 790697) +++ ql/src/test/org/apache/hadoop/hive/ql/exec/TestExpressionEvaluator.java (working copy) @@ -33,8 +33,9 @@ import org.apache.hadoop.hive.ql.plan.exprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.exprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.exprNodeDesc; -import org.apache.hadoop.hive.ql.plan.exprNodeIndexDesc; +import org.apache.hadoop.hive.ql.plan.exprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.plan.PlanUtils.ExpressionTypes; +import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; @@ -113,13 +114,24 @@ } } + private static exprNodeDesc getListIndexNode(exprNodeDesc node, int index) { + return getListIndexNode(node, new exprNodeConstantDesc(index)); + } + + private static exprNodeDesc getListIndexNode(exprNodeDesc node, exprNodeDesc index) { + return new exprNodeGenericFuncDesc( + ((ListTypeInfo)node.getTypeInfo()).getListElementTypeInfo(), + FunctionRegistry.getGenericUDFClassForIndex(), + Arrays.asList(new exprNodeDesc[]{node, index})); + } + public void testExprNodeFuncEvaluator() throws Throwable { try { // get a evaluator for a string concatenation expression exprNodeDesc col1desc = new exprNodeColumnDesc(col1Type, "col1"); exprNodeDesc coladesc = new exprNodeColumnDesc(colaType, "cola"); - exprNodeDesc col11desc = new exprNodeIndexDesc(col1desc, new exprNodeConstantDesc(new Integer(1))); - exprNodeDesc cola0desc = new exprNodeIndexDesc(coladesc, new exprNodeConstantDesc(new Integer(0))); + exprNodeDesc col11desc = getListIndexNode(col1desc, 1); + exprNodeDesc cola0desc = getListIndexNode(coladesc, 0); exprNodeDesc func1 = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("concat", col11desc, cola0desc); ExprNodeEvaluator eval = ExprNodeEvaluatorFactory.get(func1); @@ -139,7 +151,7 @@ try { // get a evaluator for a string concatenation expression exprNodeDesc col1desc = new exprNodeColumnDesc(col1Type, "col1"); - exprNodeDesc col11desc = new exprNodeIndexDesc(col1desc, new exprNodeConstantDesc(new Integer(1))); + exprNodeDesc col11desc = getListIndexNode(col1desc, 1); exprNodeDesc func1 = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc(Constants.DOUBLE_TYPE_NAME, col11desc); ExprNodeEvaluator eval = ExprNodeEvaluatorFactory.get(func1); @@ -241,8 +253,8 @@ basetimes * 10, ExprNodeEvaluatorFactory.get( TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("concat", - new exprNodeIndexDesc(new exprNodeColumnDesc(col1Type, "col1"), constant1), - new exprNodeIndexDesc(new exprNodeColumnDesc(colaType, "cola"), constant1))), + getListIndexNode(new exprNodeColumnDesc(col1Type, "col1"), constant1), + getListIndexNode(new exprNodeColumnDesc(colaType, "cola"), constant1))), r, "1b"); measureSpeed("concat(concat(col1[1], cola[1]), col1[2])", @@ -250,9 +262,9 @@ ExprNodeEvaluatorFactory.get( TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("concat", TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("concat", - new exprNodeIndexDesc(new exprNodeColumnDesc(col1Type, "col1"), constant1), - new exprNodeIndexDesc(new exprNodeColumnDesc(colaType, "cola"), constant1)), - new exprNodeIndexDesc(new exprNodeColumnDesc(col1Type, "col1"), constant2))), + getListIndexNode(new exprNodeColumnDesc(col1Type, "col1"), constant1), + getListIndexNode(new exprNodeColumnDesc(colaType, "cola"), constant1)), + getListIndexNode(new exprNodeColumnDesc(col1Type, "col1"), constant2))), r, "1b2"); measureSpeed("concat(concat(concat(col1[1], cola[1]), col1[2]), cola[2])", @@ -261,10 +273,10 @@ TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("concat", TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("concat", TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("concat", - new exprNodeIndexDesc(new exprNodeColumnDesc(col1Type, "col1"), constant1), - new exprNodeIndexDesc(new exprNodeColumnDesc(colaType, "cola"), constant1)), - new exprNodeIndexDesc(new exprNodeColumnDesc(col1Type, "col1"), constant2)), - new exprNodeIndexDesc(new exprNodeColumnDesc(colaType, "cola"), constant2))), + getListIndexNode(new exprNodeColumnDesc(col1Type, "col1"), constant1), + getListIndexNode(new exprNodeColumnDesc(colaType, "cola"), constant1)), + getListIndexNode(new exprNodeColumnDesc(col1Type, "col1"), constant2)), + getListIndexNode(new exprNodeColumnDesc(colaType, "cola"), constant2))), r, "1b2c"); Index: ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java (revision 790697) +++ ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java (working copy) @@ -41,7 +41,6 @@ import org.apache.hadoop.hive.ql.plan.exprNodeFieldDesc; import org.apache.hadoop.hive.ql.plan.exprNodeFuncDesc; import org.apache.hadoop.hive.ql.plan.exprNodeGenericFuncDesc; -import org.apache.hadoop.hive.ql.plan.exprNodeIndexDesc; import org.apache.hadoop.hive.ql.udf.UDFOPAnd; import org.apache.hadoop.hive.ql.udf.UDFType; @@ -230,50 +229,6 @@ } - public static class IndexExprProcessor implements NodeProcessor { - - @Override - public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, - Object... nodeOutputs) throws SemanticException { - ExprWalkerInfo ctx = (ExprWalkerInfo) procCtx; - exprNodeIndexDesc expr = (exprNodeIndexDesc) nd; - - // process the base array expr(or map) - exprNodeDesc desc = expr.getDesc(); - exprNodeDesc index = expr.getIndex(); - - exprNodeDesc newDesc = ctx.getConvertedNode(desc); - if (newDesc != null) { - expr.setDesc(newDesc); - desc = newDesc; - } - - exprNodeDesc newIndex = ctx.getConvertedNode(desc); - if (newIndex != null) { - expr.setIndex(newIndex); - index = newIndex; - } - if (!ctx.isCandidate(desc) || !ctx.isCandidate(index)) { - ctx.setIsCandidate(expr, false); - return false; - } - - String descAlias = ctx.getAlias(desc); - String indexAlias = ctx.getAlias(index); - if ((descAlias != null && indexAlias != null) - && (!descAlias.equals(indexAlias))) { - // aliases don't match - ctx.setIsCandidate(expr, false); - return false; - } - String alias = descAlias != null ? descAlias : indexAlias; - ctx.addAlias(expr, alias); - ctx.setIsCandidate(expr, true); - return true; - } - - } - /** * For constants and null expressions */ @@ -300,10 +255,6 @@ return new GenericFuncExprProcessor(); } - public static NodeProcessor getIndexProcessor() { - return new IndexExprProcessor(); - } - public static NodeProcessor getColumnProcessor() { return new ColumnExprProcessor(); } @@ -340,8 +291,7 @@ exprRules.put(new RuleRegExp("R1", exprNodeColumnDesc.class.getName() + "%"), getColumnProcessor()); exprRules.put(new RuleRegExp("R2", exprNodeFieldDesc.class.getName() + "%"), getFieldProcessor()); exprRules.put(new RuleRegExp("R3", exprNodeFuncDesc.class.getName() + "%"), getFuncProcessor()); - exprRules.put(new RuleRegExp("R4", exprNodeIndexDesc.class.getName() + "%"), getIndexProcessor()); - exprRules.put(new RuleRegExp("R5", exprNodeGenericFuncDesc.class.getName() + "%"), getGenericFuncProcessor()); + exprRules.put(new RuleRegExp("R4", exprNodeGenericFuncDesc.class.getName() + "%"), getGenericFuncProcessor()); // The dispatcher fires the processor corresponding to the closest matching rule and passes the context along Dispatcher disp = new DefaultRuleDispatcher(getDefaultExprProcessor(), exprRules, exprContext); Index: ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java (revision 790697) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java (working copy) @@ -25,7 +25,6 @@ import org.apache.hadoop.hive.ql.plan.exprNodeNullDesc; import org.apache.hadoop.hive.ql.plan.exprNodeFieldDesc; import org.apache.hadoop.hive.ql.plan.exprNodeFuncDesc; -import org.apache.hadoop.hive.ql.plan.exprNodeIndexDesc; public class ExprNodeEvaluatorFactory { @@ -52,10 +51,6 @@ if (desc instanceof exprNodeFieldDesc) { return new ExprNodeFieldEvaluator((exprNodeFieldDesc)desc); } - // Index node, e.g. get a[index] from a - if (desc instanceof exprNodeIndexDesc) { - return new ExprNodeIndexEvaluator((exprNodeIndexDesc)desc); - } // Null node, a constant node with value NULL and no type information if (desc instanceof exprNodeNullDesc) { return new ExprNodeNullEvaluator((exprNodeNullDesc)desc); Index: ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeIndexEvaluator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeIndexEvaluator.java (revision 790697) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeIndexEvaluator.java (working copy) @@ -1,95 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec; - -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.plan.exprNodeIndexDesc; -import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject; -import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector; - -/** - * This class can evaluate index operators on both list(array) and map. - */ -public class ExprNodeIndexEvaluator extends ExprNodeEvaluator { - - protected exprNodeIndexDesc expr; - transient ExprNodeEvaluator mainEvaluator; - transient ExprNodeEvaluator indexEvaluator; - - transient ObjectInspector mainInspector; - transient PrimitiveObjectInspector indexInspector; - - public ExprNodeIndexEvaluator(exprNodeIndexDesc expr) { - this.expr = expr; - mainEvaluator = ExprNodeEvaluatorFactory.get(expr.getDesc()); - indexEvaluator = ExprNodeEvaluatorFactory.get(expr.getIndex()); - } - - public ObjectInspector initialize(ObjectInspector rowInspector) - throws HiveException { - mainInspector = mainEvaluator.initialize(rowInspector); - indexInspector = (PrimitiveObjectInspector)indexEvaluator.initialize(rowInspector); - - if (mainInspector.getCategory() == Category.LIST) { - return ((ListObjectInspector)mainInspector).getListElementObjectInspector(); - } else if (mainInspector.getCategory() == Category.MAP) { - return ((MapObjectInspector)mainInspector).getMapValueObjectInspector(); - } else { - // Should never happen because we checked this in SemanticAnalyzer.getXpathOrFuncExprNodeDesc - throw new RuntimeException("Hive 2 Internal error: cannot evaluate index expression on " - + mainInspector.getTypeName()); - } - } - - public Object evaluate(Object row) throws HiveException { - - Object main = mainEvaluator.evaluate(row); - Object index = indexEvaluator.evaluate(row); - - if (mainInspector.getCategory() == Category.LIST) { - int intIndex = PrimitiveObjectInspectorUtils.getInt(index, indexInspector); - ListObjectInspector loi = (ListObjectInspector)mainInspector; - return loi.getListElement(main, intIndex); - } else if (mainInspector.getCategory() == Category.MAP) { - MapObjectInspector moi = (MapObjectInspector)mainInspector; - Object indexObject; - if (((PrimitiveObjectInspector)moi.getMapKeyObjectInspector()).preferWritable()) { - indexObject = indexInspector.getPrimitiveWritableObject(index); - } else { - indexObject = indexInspector.getPrimitiveJavaObject(index); - } - return moi.getMapValueElement(main, indexObject); - } - else { - // Should never happen because we checked this in SemanticAnalyzer.getXpathOrFuncExprNodeDesc - throw new RuntimeException("Hive 2 Internal error: cannot evaluate index expression on " - + mainInspector.getTypeName()); - } - } - -} Index: ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (revision 790697) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (working copy) @@ -164,6 +164,7 @@ registerGenericUDF("when", GenericUDFWhen.class); registerGenericUDF("hash", GenericUDFHash.class); registerGenericUDF("coalesce", GenericUDFCoalesce.class); + registerGenericUDF("index", GenericUDFIndex.class); } public static FunctionInfo getInfo(Class fClass) { @@ -521,4 +522,12 @@ } return udfMethod; } + + /** + * A shortcut to get the index GenericUDFClass. + * This is used for getting elements out of array and getting values out of map. + */ + public static Class getGenericUDFClassForIndex() { + return FunctionRegistry.getFunctionInfo("index").getGenericUDFClass(); + } } Index: ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeIndexDesc.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeIndexDesc.java (revision 790697) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeIndexDesc.java (working copy) @@ -1,100 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.plan; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; -import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.hive.ql.lib.Node; - - -/** - * exprNodeIndexDesc describes the operation of getting a list element out - * of a list, and getting a value out of a map. - */ -public class exprNodeIndexDesc extends exprNodeDesc implements Serializable { - private static final long serialVersionUID = 1L; - exprNodeDesc desc; - exprNodeDesc index; - - public exprNodeIndexDesc() {} - public exprNodeIndexDesc(TypeInfo typeInfo, exprNodeDesc desc, exprNodeDesc index) { - super(typeInfo); - this.desc = desc; - this.index = index; - } - public exprNodeIndexDesc(exprNodeDesc desc, exprNodeDesc index) { - super( desc.getTypeInfo().getCategory().equals(Category.LIST) - ? ((ListTypeInfo)desc.getTypeInfo()).getListElementTypeInfo() - : ((MapTypeInfo)desc.getTypeInfo()).getMapValueTypeInfo()); - this.desc = desc; - this.index = index; - } - - @Override - public List getChildren() { - List children = new ArrayList(2); - children.add(desc); - children.add(index); - return children; - } - - public exprNodeDesc getDesc() { - return this.desc; - } - public void setDesc(exprNodeDesc desc) { - this.desc = desc; - } - public exprNodeDesc getIndex() { - return this.index; - } - public void setIndex(exprNodeDesc index) { - this.index = index; - } - @Override - public String toString() { - return this.desc.toString() + "[" + this.index + "]"; - } - - @explain(displayName="expr") - @Override - public String getExprString() { - return this.desc.getExprString() + "[" + this.index.getExprString() + "]"; - } - - public List getCols() { - List colList = new ArrayList(); - if (desc != null) - colList = Utilities.mergeUniqElems(colList, desc.getCols()); - if (index != null) - colList = Utilities.mergeUniqElems(colList, index.getCols()); - - return colList; - } - @Override - public exprNodeDesc clone() { - return new exprNodeIndexDesc(this.typeInfo, this.desc, this.index); - } -} Index: ql/src/java/org/apache/hadoop/hive/ql/parse/PartitionPruner.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/PartitionPruner.java (revision 790697) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/PartitionPruner.java (working copy) @@ -37,7 +37,6 @@ import org.apache.hadoop.hive.ql.plan.exprNodeFieldDesc; import org.apache.hadoop.hive.ql.plan.exprNodeFuncDesc; import org.apache.hadoop.hive.ql.plan.exprNodeGenericFuncDesc; -import org.apache.hadoop.hive.ql.plan.exprNodeIndexDesc; import org.apache.hadoop.hive.ql.plan.exprNodeNullDesc; import org.apache.hadoop.hive.ql.udf.UDFOPAnd; import org.apache.hadoop.hive.ql.udf.UDFOPNot; @@ -322,9 +321,6 @@ return d.getValue() == null; } else if (desc instanceof exprNodeNullDesc) { return false; - } else if (desc instanceof exprNodeIndexDesc) { - exprNodeIndexDesc d = (exprNodeIndexDesc)desc; - return mightBeUnknown(d.getDesc()) || mightBeUnknown(d.getIndex()); } else if (desc instanceof exprNodeFieldDesc) { exprNodeFieldDesc d = (exprNodeFieldDesc)desc; return mightBeUnknown(d.getDesc()); Index: ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java (revision 790697) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java (working copy) @@ -43,7 +43,6 @@ import org.apache.hadoop.hive.ql.plan.exprNodeFieldDesc; import org.apache.hadoop.hive.ql.plan.exprNodeFuncDesc; import org.apache.hadoop.hive.ql.plan.exprNodeGenericFuncDesc; -import org.apache.hadoop.hive.ql.plan.exprNodeIndexDesc; import org.apache.hadoop.hive.ql.plan.exprNodeNullDesc; import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; @@ -526,7 +525,9 @@ // Calculate TypeInfo TypeInfo t = ((ListTypeInfo)myt).getListElementTypeInfo(); - desc = new exprNodeIndexDesc(t, children.get(0), children.get(1)); + desc = new exprNodeGenericFuncDesc(t, + FunctionRegistry.getGenericUDFClassForIndex(), + children); } else if (myt.getCategory() == Category.MAP) { // Only allow only constant indexes for now @@ -539,8 +540,9 @@ } // Calculate TypeInfo TypeInfo t = ((MapTypeInfo)myt).getMapValueTypeInfo(); - - desc = new exprNodeIndexDesc(t, children.get(0), children.get(1)); + desc = new exprNodeGenericFuncDesc(t, + FunctionRegistry.getGenericUDFClassForIndex(), + children); } else { throw new SemanticException(ErrorMsg.NON_COLLECTION_TYPE.getMsg(expr, Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIndex.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIndex.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIndex.java (revision 0) @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.udf.generic; + +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde.Constants; +import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.io.IntWritable; + +public class GenericUDFIndex extends GenericUDF { + private MapObjectInspector mapOI; + private boolean mapKeyPreferWritable; + private ListObjectInspector listOI; + private PrimitiveObjectInspector indexOI; + private ObjectInspector returnOI; + private IntWritable result = new IntWritable(-1); + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) + throws UDFArgumentException { + if (arguments.length != 2) { + throw new UDFArgumentLengthException( + "The function INDEX accepts exactly 2 arguments."); + } + + if (arguments[0] instanceof MapObjectInspector) { + // index into a map + mapOI = (MapObjectInspector)arguments[0]; + listOI = null; + } else if (arguments[0] instanceof ListObjectInspector) { + // index into a list + listOI = (ListObjectInspector)arguments[0]; + mapOI = null; + } else { + throw new UDFArgumentTypeException(0, + "\"" + Category.MAP.toString().toLowerCase() + + "\" or \"" + Category.LIST.toString().toLowerCase() + + "\" is expected at function INDEX, but \"" + + arguments[0].getTypeName() + "\" is found"); + } + + // index has to be a primitive + if (arguments[1] instanceof PrimitiveObjectInspector) { + indexOI = (PrimitiveObjectInspector) arguments[1]; + } else { + throw new UDFArgumentTypeException(1, + "Primitive Type is expected but " + arguments[1].getTypeName() + + "\" is found"); + } + + if (mapOI != null) { + returnOI = mapOI.getMapValueObjectInspector(); + ObjectInspector keyOI = mapOI.getMapKeyObjectInspector(); + mapKeyPreferWritable = ((PrimitiveObjectInspector)keyOI).preferWritable(); + } else { + returnOI = listOI.getListElementObjectInspector(); + } + + return returnOI; + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + assert(arguments.length == 2); + Object main = arguments[0].get(); + Object index = arguments[1].get(); + + if (mapOI != null) { + + Object indexObject; + if (mapKeyPreferWritable) { + indexObject = indexOI.getPrimitiveWritableObject(index); + } else { + indexObject = indexOI.getPrimitiveJavaObject(index); + } + return mapOI.getMapValueElement(main, indexObject); + + } else { + + assert(listOI != null); + int intIndex = 0; + try { + intIndex = PrimitiveObjectInspectorUtils.getInt(index, indexOI); + } catch (NullPointerException e) { + // If index is null, we should return null. + return null; + } catch (NumberFormatException e) { + // If index is not a number, we should return null. + return null; + } + return listOI.getListElement(main, intIndex); + + } + } + + @Override + public String getDisplayString(String[] children) { + assert(children.length == 2); + return children[0] + "[" + children[1] + "]"; + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSize.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSize.java (revision 790697) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSize.java (working copy) @@ -39,7 +39,7 @@ throws UDFArgumentException { if (arguments.length != 1) { throw new UDFArgumentLengthException( - "The functin SIZE only accepts 1 argument."); + "The function SIZE only accepts 1 argument."); } Category category = arguments[0].getCategory(); String typeName = arguments[0].getTypeName();