commit 6a0f3a11e91ab29ba241579c3edc32354e899a45 Author: Travis Crawford Date: Thu Jul 26 16:52:54 2012 -0700 Conditionalize binary support based on Hive version diff --git a/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatBaseStorer.java b/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatBaseStorer.java index 83e51bd..6c3082e 100644 --- a/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatBaseStorer.java +++ b/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatBaseStorer.java @@ -32,6 +32,7 @@ import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hcatalog.common.HCatException; +import org.apache.hcatalog.common.HCatUtil; import org.apache.hcatalog.data.DefaultHCatRecord; import org.apache.hcatalog.data.HCatRecord; import org.apache.hcatalog.data.schema.HCatFieldSchema; @@ -247,7 +248,21 @@ public abstract class HCatBaseStorer extends StoreFunc implements StoreMetadata switch(type){ case BINARY: - return (null == pigObj) ? new byte[0] : ((DataByteArray)pigObj).get(); + if (null == pigObj) { + return new byte[0]; + } + + if (HCatUtil.getHiveMajorVersion() != null && + HCatUtil.getHiveMinorVersion() != null && + HCatUtil.getHiveMajorVersion() == 0 && + HCatUtil.getHiveMinorVersion() <= 9) { + ByteArrayRef ba = new ByteArrayRef(); + byte[] bytes = ((DataByteArray) pigObj).get(); + ba.setData(bytes); + return ba; + } else { + return ((DataByteArray)pigObj).get(); + } case STRUCT: if (pigObj == null) { diff --git a/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/PigHCatUtil.java b/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/PigHCatUtil.java index 7fe459d..102edd1 100644 --- a/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/PigHCatUtil.java +++ b/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/PigHCatUtil.java @@ -320,7 +320,17 @@ public class PigHCatUtil { Type itemType = hfs.getType(); switch (itemType){ case BINARY: - result = (o == null) ? null : new DataByteArray((byte[]) o); + if (null == o) { + result = null; + } else if ( + HCatUtil.getHiveMajorVersion() != null && + HCatUtil.getHiveMinorVersion() != null && + HCatUtil.getHiveMajorVersion() == 0 && + HCatUtil.getHiveMinorVersion() <= 9) { + result = new DataByteArray(((ByteArrayRef)o).getData()); + } else { + result = ((DataByteArray)o).get(); + } break; case STRUCT: result = transformToTuple((List)o,hfs); diff --git a/src/java/org/apache/hcatalog/common/HCatUtil.java b/src/java/org/apache/hcatalog/common/HCatUtil.java index 5b744fe..dafb322 100644 --- a/src/java/org/apache/hcatalog/common/HCatUtil.java +++ b/src/java/org/apache/hcatalog/common/HCatUtil.java @@ -30,6 +30,8 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.jar.JarFile; +import java.util.jar.Manifest; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.permission.FsAction; @@ -64,6 +66,7 @@ import org.apache.hcatalog.mapreduce.InputJobInfo; import org.apache.hcatalog.mapreduce.OutputJobInfo; import org.apache.hcatalog.mapreduce.PartInfo; import org.apache.hcatalog.mapreduce.StorerInfo; +import org.apache.pig.impl.util.JarManager; import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -71,6 +74,9 @@ import org.slf4j.LoggerFactory; public class HCatUtil { private static final Logger LOG = LoggerFactory.getLogger(HCatUtil.class); + private static Integer HIVE_MAJOR_VERSION = null; + private static Integer HIVE_MINOR_VERSION = null; + public static boolean checkJobContextIfRunningFromBackend(JobContext j) { if (j.getConfiguration().get("mapred.task.id", "").equals("")) { @@ -591,4 +597,34 @@ public class HCatUtil { jobConf.set(entry.getKey(), entry.getValue()); } } + + public static void getHiveVersion() { + try { + String hiveJarFileName = JarManager.findContainingJar(HiveConf.class); + JarFile jar = new JarFile(hiveJarFileName); + Manifest manifest = jar.getManifest(); + String title = manifest.getMainAttributes().getValue("Implementation-Title"); + String version = manifest.getMainAttributes().getValue("Implementation-Version"); + if ("Hive".equals(title) && version != null) { + HIVE_MAJOR_VERSION = Integer.parseInt(version.split("\\.")[0]); + HIVE_MINOR_VERSION = Integer.parseInt(version.split("\\.")[1]); + } + } catch (IOException e) { + LOG.debug("Failed determining specific Hive version number."); + } + } + + public static Integer getHiveMajorVersion() { + if (HIVE_MAJOR_VERSION == null) { + getHiveVersion(); + } + return HIVE_MAJOR_VERSION; + } + + public static Integer getHiveMinorVersion() { + if (HIVE_MINOR_VERSION == null) { + getHiveVersion(); + } + return HIVE_MINOR_VERSION; + } }