diff --git hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatBaseStorer.java hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatBaseStorer.java index a19e98e..ead54e3 100644 --- hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatBaseStorer.java +++ hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatBaseStorer.java @@ -27,7 +27,6 @@ import java.util.Map; import java.util.Map.Entry; -import com.google.common.collect.Lists; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapreduce.Job; @@ -53,6 +52,10 @@ import org.apache.pig.impl.util.ObjectSerializer; import org.apache.pig.impl.util.UDFContext; import org.apache.pig.impl.util.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; /** * Base class for HCatStorer and HCatEximStorer @@ -61,6 +64,8 @@ abstract class HCatBaseStorer extends StoreFunc implements StoreMetadata { + private static final Logger LOG = LoggerFactory.getLogger( HCatBaseStorer.class ); + private static final List SUPPORTED_INTEGER_CONVERSIONS = Lists.newArrayList(Type.TINYINT, Type.SMALLINT, Type.INT); protected static final String COMPUTED_OUTPUT_SCHEMA = "hcat.output.schema"; @@ -175,6 +180,9 @@ private HCatFieldSchema getHCatFSFromPigFS(FieldSchema fSchema, HCatFieldSchema case DataType.BYTEARRAY: return new HCatFieldSchema(fSchema.alias, Type.BINARY, null); + case DataType.BOOLEAN: + return new HCatFieldSchema(fSchema.alias, Type.BOOLEAN, null); + case DataType.BAG: Schema bagSchema = fSchema.schema; List arrFields = new ArrayList(1); @@ -328,10 +336,30 @@ private Object getJavaObj(Object pigObj, HCatFieldSchema hcatFS) throws HCatExce } return ((Integer) pigObj).byteValue(); case BOOLEAN: - // would not pass schema validation anyway - throw new BackendException("Incompatible type " + type + " found in hcat table schema: " + hcatFS, PigHCatUtil.PIG_EXCEPTION_CODE); + if (pigObj == null) { + LOG.debug( "HCatBaseStorer.getJavaObj(BOOLEAN): obj null, bailing early" ); + return null; + } + + if( pigObj instanceof String ) { + if( ((String)pigObj).trim().compareTo("0") == 0 ) { + return Boolean.FALSE; + } + if( ((String)pigObj).trim().compareTo("1") == 0 ) { + return Boolean.TRUE; + } + + throw new BackendException( + "Unexpected type " + type + " for value " + pigObj + + (pigObj == null ? "" : " of class " + + pigObj.getClass().getName()), PigHCatUtil.PIG_EXCEPTION_CODE); + } + + return Boolean.parseBoolean( pigObj.toString() ); default: - throw new BackendException("Unexpected type " + type + " for value " + pigObj + (pigObj == null ? "" : " of class " + pigObj.getClass().getName()), PigHCatUtil.PIG_EXCEPTION_CODE); + throw new BackendException("Unexpected type " + type + " for value " + pigObj + + (pigObj == null ? "" : " of class " + + pigObj.getClass().getName()), PigHCatUtil.PIG_EXCEPTION_CODE); } } catch (BackendException e) { // provide the path to the field in the error message diff --git hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorer.java hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorer.java index 8019e3e..6074821 100644 --- hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorer.java +++ hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorer.java @@ -403,7 +403,7 @@ public void testBagNStruct() throws IOException, CommandNeedRetryException { public void testStoreFuncAllSimpleTypes() throws IOException, CommandNeedRetryException { driver.run("drop table junit_unparted"); - String createTable = "create table junit_unparted(a int, b float, c double, d bigint, e string, f binary, g binary) stored as RCFILE"; + String createTable = "create table junit_unparted(a int, b float, c double, d bigint, e string, h boolean, f binary, g binary) stored as RCFILE"; int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { throw new IOException("Failed to create table."); @@ -411,16 +411,16 @@ public void testStoreFuncAllSimpleTypes() throws IOException, CommandNeedRetryEx int i = 0; String[] input = new String[3]; - input[i++] = "0\t\t\t\t\t\t"; //Empty values except first column - input[i++] = "\t" + i * 2.1f + "\t" + i * 1.1d + "\t" + i * 2L + "\t" + "lets hcat" + "\tbinary-data"; //First column empty - input[i++] = i + "\t" + i * 2.1f + "\t" + i * 1.1d + "\t" + i * 2L + "\t" + "lets hcat" + "\tbinary-data"; + input[i++] = "0\t\t\t\t\t\t\t"; //Empty values except first column + input[i++] = "\t" + i * 2.1f + "\t" + i * 1.1d + "\t" + i * 2L + "\t" + "lets hcat" + "\t" + "true" + "\tbinary-data"; //First column empty + input[i++] = i + "\t" + i * 2.1f + "\t" + i * 1.1d + "\t" + i * 2L + "\t" + "lets hcat" + "\t" + "false" + "\tbinary-data"; HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, input); PigServer server = new PigServer(ExecType.LOCAL); server.setBatchOn(); - server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:float, c:double, d:long, e:chararray, f:bytearray);"); + server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:float, c:double, d:long, e:chararray, h:boolean, f:bytearray);"); //null gets stored into column g which is a binary field. - server.registerQuery("store A into 'default.junit_unparted' using " + HCatStorer.class.getName() + "('','a:int, b:float, c:double, d:long, e:chararray,f:bytearray');"); + server.registerQuery("store A into 'default.junit_unparted' using " + HCatStorer.class.getName() + "('','a:int, b:float, c:double, d:long, e:chararray, h:boolean, f:bytearray');"); server.executeBatch(); @@ -429,9 +429,10 @@ public void testStoreFuncAllSimpleTypes() throws IOException, CommandNeedRetryEx driver.getResults(res); Iterator itr = res.iterator(); - Assert.assertEquals("0\tNULL\tNULL\tNULL\tNULL\tNULL\tNULL", itr.next()); - Assert.assertEquals("NULL\t4.2\t2.2\t4\tlets hcat\tbinary-data\tNULL", itr.next()); - Assert.assertEquals("3\t6.2999997\t3.3000000000000003\t6\tlets hcat\tbinary-data\tNULL", itr.next()); + String next = itr.next(); + Assert.assertEquals("0\tNULL\tNULL\tNULL\tNULL\tNULL\tNULL\tNULL", next ); + Assert.assertEquals("NULL\t4.2\t2.2\t4\tlets hcat\ttrue\tbinary-data\tNULL", itr.next()); + Assert.assertEquals("3\t6.2999997\t3.3000000000000003\t6\tlets hcat\tfalse\tbinary-data\tNULL", itr.next()); Assert.assertFalse(itr.hasNext()); server.registerQuery("B = load 'junit_unparted' using " + HCatLoader.class.getName() + ";"); @@ -440,12 +441,12 @@ public void testStoreFuncAllSimpleTypes() throws IOException, CommandNeedRetryEx int num5nulls = 0; while (iter.hasNext()) { Tuple t = iter.next(); - if (t.get(5) == null) { + if (t.get(6) == null) { num5nulls++; } else { - Assert.assertTrue(t.get(5) instanceof DataByteArray); + Assert.assertTrue(t.get(6) instanceof DataByteArray); } - Assert.assertNull(t.get(6)); + Assert.assertNull(t.get(7)); count++; } Assert.assertEquals(3, count);