diff --git data/conf/tez/hive-site.xml data/conf/tez/hive-site.xml index 41b248d..20b3abd 100644 --- data/conf/tez/hive-site.xml +++ data/conf/tez/hive-site.xml @@ -44,11 +44,6 @@ - hive.llap.execution.mode - auto - - - mapred.tez.java.opts -Xmx128m diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java index 2127abc..a9b30cb 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java @@ -1574,9 +1574,16 @@ public static boolean isRankingFunction(String name) throws SemanticException { * @return True iff the fnExpr represents a hive built-in function (native, non-permanent) */ public static boolean isBuiltInFuncExpr(ExprNodeGenericFuncDesc fnExpr) { - Class udfClass = FunctionRegistry.getGenericUDFClassFromExprDesc(fnExpr); - if (udfClass != null) { - return system.isBuiltInFunc(udfClass); + GenericUDF udf = fnExpr.getGenericUDF(); + if (udf == null) return false; + + Class clazz = udf.getClass(); + if (udf instanceof GenericUDFBridge) { + clazz = ((GenericUDFBridge)udf).getUdfClass(); + } + + if (clazz != null) { + return system.isBuiltInFunc(clazz); } return false; } @@ -1590,4 +1597,4 @@ public static void setupPermissionsForBuiltinUDFs(String whiteListStr, String blackListStr) { system.setupPermissionsForUDFs(whiteListStr, blackListStr); } -} \ No newline at end of file +} diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java index 09aca38..79308b0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java @@ -65,6 +65,7 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.plan.MapWork; import org.apache.hadoop.hive.ql.plan.PartitionDesc; +import org.apache.hadoop.hive.ql.plan.ReduceWork; import org.apache.hadoop.hive.ql.plan.Statistics; import org.apache.hadoop.hive.ql.plan.TezWork; @@ -131,6 +132,14 @@ private void handleWork(TezWork tezWork, BaseWork work) private void convertWork(TezWork tezWork, BaseWork work) throws SemanticException { + // let's see if we can go one step further and just uber this puppy + if (tezWork.getChildren(work).isEmpty() + && work instanceof ReduceWork + && ((ReduceWork) work).getNumReduceTasks() == 1) { + work.setUberMode(true); + } + + // always mark as llap work.setLlapMode(true); } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java index 6c64318..cf8ebfe 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java @@ -316,12 +316,16 @@ private void setAliases() { public String getExecutionMode() { if (vectorMode) { if (llapMode) { - return "vectorized, llap"; + if (uberMode) { + return "vectorized, uber"; + } else { + return "vectorized, llap"; + } } else { return "vectorized"; } } else if (llapMode) { - return "llap"; + return uberMode? "uber" : "llap"; } return null; } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java index 598f230..5e1a923 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java @@ -152,12 +152,16 @@ public void setTagToValueDesc(final List tagToValueDesc) { public String getExecutionMode() { if (vectorMode) { if (llapMode) { - return "vectorized, llap"; + if (uberMode) { + return "vectorized, uber"; + } else { + return "vectorized, llap"; + } } else { return "vectorized"; } } else if (llapMode) { - return "llap"; + return uberMode? "uber" : "llap"; } return null; } diff --git ql/src/test/results/clientpositive/tez/llapdecider.q.out ql/src/test/results/clientpositive/tez/llapdecider.q.out index a698284..3f364dd 100644 --- ql/src/test/results/clientpositive/tez/llapdecider.q.out +++ ql/src/test/results/clientpositive/tez/llapdecider.q.out @@ -203,7 +203,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) Reducer 3 - Execution mode: llap + Execution mode: uber Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: bigint) @@ -284,7 +284,7 @@ STAGE PLANS: Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) Reducer 3 - Execution mode: llap + Execution mode: uber Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: bigint) @@ -462,7 +462,7 @@ STAGE PLANS: Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) Reducer 3 - Execution mode: llap + Execution mode: uber Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY.reducesinkkey0 (type: string) @@ -551,7 +551,7 @@ STAGE PLANS: Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) Reducer 3 - Execution mode: llap + Execution mode: uber Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY.reducesinkkey0 (type: string) @@ -991,7 +991,7 @@ STAGE PLANS: Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) Reducer 3 - Execution mode: llap + Execution mode: uber Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY.reducesinkkey0 (type: string) @@ -1055,7 +1055,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap Reducer 2 - Execution mode: llap + Execution mode: uber Reduce Operator Tree: Group By Operator aggregations: sum(VALUE._col0)