diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java index f6b6447..5c51450 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java @@ -168,7 +168,7 @@ // private final Map columnMap; private int firstOutputColumnIndex; - private enum HiveVectorAdaptorUsageMode { + public enum HiveVectorAdaptorUsageMode { NONE, CHOSEN, ALL; diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index 2a99274..814fe6c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -67,6 +67,7 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOuterFilteredOperator; import org.apache.hadoop.hive.ql.exec.vector.VectorSMBMapJoinOperator; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; +import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext.HiveVectorAdaptorUsageMode; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext.InConstantType; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContextRegion; import org.apache.hadoop.hive.ql.exec.vector.expressions.IdentityExpression; @@ -240,8 +241,16 @@ boolean isSchemaEvolution; + HiveVectorAdaptorUsageMode hiveVectorAdaptorUsageMode; + public Vectorizer() { + /* + * We check UDFs against the supportedGenericUDFs when + * hive.vectorized.adaptor.usage.mode=chosen or none. + * + * We allow all UDFs for hive.vectorized.adaptor.usage.mode=all. + */ supportedGenericUDFs.add(GenericUDFOPPlus.class); supportedGenericUDFs.add(GenericUDFOPMinus.class); supportedGenericUDFs.add(GenericUDFOPMultiply.class); @@ -1394,6 +1403,8 @@ public PhysicalContext resolve(PhysicalContext physicalContext) throws SemanticE HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_SCHEMA_EVOLUTION); + hiveVectorAdaptorUsageMode = HiveVectorAdaptorUsageMode.getHiveConfValue(hiveConf); + // create dispatcher and graph walker Dispatcher disp = new VectorizationDispatcher(physicalContext); TaskGraphWalker ogw = new TaskGraphWalker(disp); @@ -1900,13 +1911,17 @@ private boolean validateGenericUdf(ExprNodeGenericFuncDesc genericUDFExpr) { if (VectorizationContext.isCustomUDF(genericUDFExpr)) { return true; } - GenericUDF genericUDF = genericUDFExpr.getGenericUDF(); - if (genericUDF instanceof GenericUDFBridge) { - Class udf = ((GenericUDFBridge) genericUDF).getUdfClass(); - return supportedGenericUDFs.contains(udf); - } else { - return supportedGenericUDFs.contains(genericUDF.getClass()); + if (hiveVectorAdaptorUsageMode == HiveVectorAdaptorUsageMode.NONE || + hiveVectorAdaptorUsageMode == HiveVectorAdaptorUsageMode.CHOSEN) { + GenericUDF genericUDF = genericUDFExpr.getGenericUDF(); + if (genericUDF instanceof GenericUDFBridge) { + Class udf = ((GenericUDFBridge) genericUDF).getUdfClass(); + return supportedGenericUDFs.contains(udf); + } else { + return supportedGenericUDFs.contains(genericUDF.getClass()); + } } + return true; } private boolean validateAggregationIsPrimitive(VectorAggregateExpression vectorAggrExpr) { diff --git ql/src/test/results/clientpositive/vector_complex_all.q.out ql/src/test/results/clientpositive/vector_complex_all.q.out index 69d5576..eb7fc55 100644 --- ql/src/test/results/clientpositive/vector_complex_all.q.out +++ ql/src/test/results/clientpositive/vector_complex_all.q.out @@ -75,9 +75,9 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_create_complex #### A masked pattern was here #### orc_create_complex.str orc_create_complex.mp orc_create_complex.lst orc_create_complex.strct orc_create_complex.val -line1 {"key13":"value13","key11":"value11","key12":"value12"} ["a","b","c"] {"a":"one","b":"two"} 0 -line2 {"key21":"value21","key22":"value22","key23":"value23"} ["d","e","f"] {"a":"three","b":"four"} 0 -line3 {"key31":"value31","key32":"value32","key33":"value33"} ["g","h","i"] {"a":"five","b":"six"} 0 +line1 {"key13":"value13","key12":"value12","key11":"value11"} ["a","b","c"] {"a":"one","b":"two"} 0 +line2 {"key21":"value21","key23":"value23","key22":"value22"} ["d","e","f"] {"a":"three","b":"four"} 0 +line3 {"key33":"value33","key31":"value31","key32":"value32"} ["g","h","i"] {"a":"five","b":"six"} 0 PREHOOK: query: SELECT str FROM orc_create_complex PREHOOK: type: QUERY PREHOOK: Input: default@orc_create_complex @@ -99,9 +99,9 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_create_complex #### A masked pattern was here #### strct mp lst -{"a":"one","b":"two"} {"key13":"value13","key11":"value11","key12":"value12"} ["a","b","c"] -{"a":"three","b":"four"} {"key21":"value21","key22":"value22","key23":"value23"} ["d","e","f"] -{"a":"five","b":"six"} {"key31":"value31","key32":"value32","key33":"value33"} ["g","h","i"] +{"a":"one","b":"two"} {"key13":"value13","key12":"value12","key11":"value11"} ["a","b","c"] +{"a":"three","b":"four"} {"key21":"value21","key23":"value23","key22":"value22"} ["d","e","f"] +{"a":"five","b":"six"} {"key33":"value33","key31":"value31","key32":"value32"} ["g","h","i"] PREHOOK: query: SELECT lst, str FROM orc_create_complex PREHOOK: type: QUERY PREHOOK: Input: default@orc_create_complex @@ -123,9 +123,9 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_create_complex #### A masked pattern was here #### mp str -{"key13":"value13","key11":"value11","key12":"value12"} line1 -{"key21":"value21","key22":"value22","key23":"value23"} line2 -{"key31":"value31","key32":"value32","key33":"value33"} line3 +{"key13":"value13","key12":"value12","key11":"value11"} line1 +{"key21":"value21","key23":"value23","key22":"value22"} line2 +{"key33":"value33","key31":"value31","key32":"value32"} line3 PREHOOK: query: SELECT strct, str FROM orc_create_complex PREHOOK: type: QUERY PREHOOK: Input: default@orc_create_complex @@ -224,6 +224,6 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_create_complex #### A masked pattern was here #### strct mp lst str c4 -{"a":"one","b":"two"} {"key11":"value11","key12":"value12","key13":"value13"} ["a","b","c"] line1 4501 -{"a":"three","b":"four"} {"key21":"value21","key22":"value22","key23":"value23"} ["d","e","f"] line2 4501 -{"a":"five","b":"six"} {"key31":"value31","key32":"value32","key33":"value33"} ["g","h","i"] line3 4501 +{"a":"one","b":"two"} {"key12":"value12","key11":"value11","key13":"value13"} ["a","b","c"] line1 4501 +{"a":"three","b":"four"} {"key21":"value21","key23":"value23","key22":"value22"} ["d","e","f"] line2 4501 +{"a":"five","b":"six"} {"key33":"value33","key31":"value31","key32":"value32"} ["g","h","i"] line3 4501