diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java index c257217..b96465e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java @@ -84,6 +84,7 @@ public class ExecMapper extends MapReduceBase implements Mapper { try { jc = job; execContext.setJc(jc); + UDFContext.addJobConf(jc); // create map and fetch operators MapredWork mrwork = Utilities.getMapRedWork(job); mo = new MapOperator(); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java index 3d7c798..d69fd78 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java @@ -87,7 +87,7 @@ public class ExecReducer extends MapReduceBase implements Reducer { TableDesc[] valueTableDesc; ObjectInspector[] rowObjectInspector; - + @Override public void configure(JobConf job) { rowObjectInspector = new ObjectInspector[Byte.MAX_VALUE]; @@ -108,6 +108,7 @@ public class ExecReducer extends MapReduceBase implements Reducer { l4j.info("cannot get classpath: " + e.getMessage()); } jc = job; + UDFContext.addJobConf(jc); MapredWork gWork = Utilities.getMapRedWork(job); reducer = gWork.getReducer(); reducer.setParentOperators(null); // clear out any parents as reducer is the @@ -247,7 +248,7 @@ public class ExecReducer extends MapReduceBase implements Reducer { try { rowString = SerDeUtils.getJSONString(row, rowObjectInspector[tag.get()]); } catch (Exception e2) { - rowString = "[Error getting row data with exception " + + rowString = "[Error getting row data with exception " + StringUtils.stringifyException(e2) + " ]"; } throw new HiveException("Hive Runtime Error while processing row (tag=" @@ -300,7 +301,7 @@ public class ExecReducer extends MapReduceBase implements Reducer { reducer.close(abort); reportStats rps = new reportStats(rp); reducer.preorderMap(rps); - + } catch (Exception e) { if (!abort) { // signal new failure to map-reduce diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/UDFContext.java ql/src/java/org/apache/hadoop/hive/ql/exec/UDFContext.java new file mode 100644 index 0000000..5b35ad3 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/UDFContext.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec; + +import org.apache.hadoop.mapred.JobConf; + +/** + * UDFContext. + * + * A singleton class that provides UDFs with runtime + * context. + */ +@SuppressWarnings("deprecation") +public class UDFContext { + private static final UDFContext INSTANCE = new UDFContext(); + + private JobConf conf = null; + + private UDFContext() {} + + /** + * Sets the JobConf for this singleton. This is called on + * the backend as part of the initialization of the + * Map and Reduce functions in order to provide UDFs with + * runtime access to the JobConf. + * @param conf + */ + public static void addJobConf(JobConf conf) { + INSTANCE.conf = conf; + } + + /** + * Get a copy of the JobConf. This method should only be + * called by UDFs executing on the backend. This method will + * return NULL if called on the frontend. + * @return A copy of the JobConf for this job. + */ + public static JobConf getJobConf() { + if (null != INSTANCE.conf) { + return new JobConf(INSTANCE.conf); + } + return null; + } +} diff --git ql/src/test/org/apache/hadoop/hive/ql/udf/UDFGetConfProp.java ql/src/test/org/apache/hadoop/hive/ql/udf/UDFGetConfProp.java new file mode 100644 index 0000000..03297bb --- /dev/null +++ ql/src/test/org/apache/hadoop/hive/ql/udf/UDFGetConfProp.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.udf; + +import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.ql.exec.UDFContext; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.JobConf; + +/** + * UDFGetConfProp. + * + * Maps a configuration property name to its + * corresponding value using a copy of the JobConf + * obtained from the UDFContext. + */ +public class UDFGetConfProp extends UDF { + + private final Text result = new Text(); + private boolean isInitialized = false; + private JobConf conf = null; + + public Text evaluate(Text propName) { + if (!isInitialized) { + conf = UDFContext.getJobConf(); + isInitialized = true; + } + if (propName == null || conf == null) { + return null; + } + result.set(conf.get(propName.toString())); + return result; + } +} diff --git ql/src/test/queries/clientpositive/udf_testcontext.q ql/src/test/queries/clientpositive/udf_testcontext.q new file mode 100644 index 0000000..78d383a --- /dev/null +++ ql/src/test/queries/clientpositive/udf_testcontext.q @@ -0,0 +1,10 @@ +-- Test to see if the UDFContext is properly initialized on the backend +-- and if a UDF can obtain the JobConf using the UDFContext. + +CREATE TEMPORARY FUNCTION getconfprop AS 'org.apache.hadoop.hive.ql.udf.UDFGetConfProp'; + +set test.udfcontext.prop="Can the UDF see this on the backend?"; + +select getconfprop('test.udfcontext.prop') FROM src LIMIT 10; + +DROP TEMPORARY FUNCTION getconfprop; diff --git ql/src/test/results/clientpositive/udf_testcontext.q.out ql/src/test/results/clientpositive/udf_testcontext.q.out new file mode 100644 index 0000000..919cb24 --- /dev/null +++ ql/src/test/results/clientpositive/udf_testcontext.q.out @@ -0,0 +1,32 @@ +PREHOOK: query: -- Test to see if the UDFContext is properly initialized on the backend +-- and if a UDF can obtain the JobConf using the UDFContext. + +CREATE TEMPORARY FUNCTION getconfprop AS 'org.apache.hadoop.hive.ql.udf.UDFGetConfProp' +PREHOOK: type: CREATEFUNCTION +POSTHOOK: query: -- Test to see if the UDFContext is properly initialized on the backend +-- and if a UDF can obtain the JobConf using the UDFContext. + +CREATE TEMPORARY FUNCTION getconfprop AS 'org.apache.hadoop.hive.ql.udf.UDFGetConfProp' +POSTHOOK: type: CREATEFUNCTION +PREHOOK: query: select getconfprop('test.udfcontext.prop') FROM src LIMIT 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: file:/var/folders/b7/b7UUwNZdF1KKHtM+5la6f++++TI/-Tmp-/carl/hive_2010-08-30_15-09-05_654_5544702033416876686/-mr-10000 +POSTHOOK: query: select getconfprop('test.udfcontext.prop') FROM src LIMIT 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: file:/var/folders/b7/b7UUwNZdF1KKHtM+5la6f++++TI/-Tmp-/carl/hive_2010-08-30_15-09-05_654_5544702033416876686/-mr-10000 +"Can the UDF see this on the backend?" +"Can the UDF see this on the backend?" +"Can the UDF see this on the backend?" +"Can the UDF see this on the backend?" +"Can the UDF see this on the backend?" +"Can the UDF see this on the backend?" +"Can the UDF see this on the backend?" +"Can the UDF see this on the backend?" +"Can the UDF see this on the backend?" +"Can the UDF see this on the backend?" +PREHOOK: query: DROP TEMPORARY FUNCTION getconfprop +PREHOOK: type: DROPFUNCTION +POSTHOOK: query: DROP TEMPORARY FUNCTION getconfprop +POSTHOOK: type: DROPFUNCTION