diff --git a/hive-1.1/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFJson.java b/hive-1.1/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFJson.java index 66778e0..9d11d75 100644 --- a/hive-1.1/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFJson.java +++ b/hive-1.1/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFJson.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import java.util.ArrayList; +import java.util.Collections; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; @@ -27,6 +28,7 @@ import java.util.regex.Pattern; import com.google.common.collect.Iterators; + import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; import org.apache.hadoop.io.Text; @@ -90,7 +92,7 @@ protected boolean removeEldestEntry(Map.Entry eldest) { } - static Map extractObjectCache = new HashCache(); + static Map extractObjectCache = Collections.synchronizedMap(new HashCache()); static Map pathExprCache = new HashCache(); static Map> indexListCache = new HashCache>(); @@ -138,16 +140,21 @@ public Text evaluate(String jsonString, String pathString) { if (!pathExpr[0].equalsIgnoreCase("$")) { return null; } + // Cache extractObject - Object extractObject = extractObjectCache.get(jsonString); - if (extractObject == null) { - try { - extractObject = MAPPER.readValue(jsonString, MAP_TYPE); - } catch (Exception e) { - return null; - } - extractObjectCache.put(jsonString, extractObject); + Object extractObject; + synchronized(extractObjectCache) { + extractObject = extractObjectCache.get(jsonString); + if (extractObject == null) { + try { + extractObject = MAPPER.readValue(jsonString, MAP_TYPE); + } catch (Exception e) { + return null; + } + extractObjectCache.put(jsonString, extractObject); + } } + for (int i = 1; i < pathExpr.length; i++) { if (extractObject == null) { return null; diff --git a/hive-1.1/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFJSONTuple.java b/hive-1.1/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFJSONTuple.java index ea5aeec..3cf6adf 100644 --- a/hive-1.1/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFJSONTuple.java +++ b/hive-1.1/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFJSONTuple.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf.generic; import java.util.ArrayList; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -91,8 +92,8 @@ protected boolean removeEldestEntry(Map.Entry eldest) { } - static Map jsonObjectCache = new HashCache(); - + static Map jsonObjectCache = Collections.synchronizedMap(new HashCache()); + @Override public void close() throws HiveException { } @@ -163,42 +164,44 @@ public void process(Object[] o) throws HiveException { return; } try { - Object jsonObj = jsonObjectCache.get(jsonStr); - if (jsonObj == null) { - try { - jsonObj = MAPPER.readValue(jsonStr, MAP_TYPE); - } catch (Exception e) { - reportInvalidJson(jsonStr); - forward(nullCols); - return; - } - jsonObjectCache.put(jsonStr, jsonObj); - } - - if (!(jsonObj instanceof Map)) { - reportInvalidJson(jsonStr); - forward(nullCols); - return; - } - - for (int i = 0; i < numCols; ++i) { - if (retCols[i] == null) { - retCols[i] = cols[i]; // use the object pool rather than creating a new object - } - Object extractObject = ((Map)jsonObj).get(paths[i]); - if (extractObject instanceof Map || extractObject instanceof List) { - retCols[i].set(MAPPER.writeValueAsString(extractObject)); - } else if (extractObject != null) { - retCols[i].set(extractObject.toString()); - } else { - retCols[i] = null; - } - } - forward(retCols); - return; + Object jsonObj; + synchronized(jsonObjectCache) { + jsonObj = jsonObjectCache.get(jsonStr); + if (jsonObj == null) { + try { + jsonObj = MAPPER.readValue(jsonStr, MAP_TYPE); + } catch (Exception e) { + reportInvalidJson(jsonStr); + forward(nullCols); + return; + } + jsonObjectCache.put(jsonStr, jsonObj); + } + } + if (!(jsonObj instanceof Map)) { + reportInvalidJson(jsonStr); + forward(nullCols); + return; + } + + for (int i = 0; i < numCols; ++i) { + if (retCols[i] == null) { + retCols[i] = cols[i]; // use the object pool rather than creating a new object + } + Object extractObject = ((Map)jsonObj).get(paths[i]); + if (extractObject instanceof Map || extractObject instanceof List) { + retCols[i].set(MAPPER.writeValueAsString(extractObject)); + } else if (extractObject != null) { + retCols[i].set(extractObject.toString()); + } else { + retCols[i] = null; + } + } + forward(retCols); + return; } catch (Throwable e) { - LOG.error("JSON parsing/evaluation exception" + e); - forward(nullCols); + LOG.error("JSON parsing/evaluation exception" + e); + forward(nullCols); } }