diff --git hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatBaseStorer.java hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatBaseStorer.java index a19e98e..a1b184a 100644 --- hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatBaseStorer.java +++ hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatBaseStorer.java @@ -53,6 +53,8 @@ import org.apache.pig.impl.util.ObjectSerializer; import org.apache.pig.impl.util.UDFContext; import org.apache.pig.impl.util.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Base class for HCatStorer and HCatEximStorer @@ -60,6 +62,8 @@ */ abstract class HCatBaseStorer extends StoreFunc implements StoreMetadata { + + private static final Logger LOG = LoggerFactory.getLogger( HCatBaseStorer.class ); private static final List SUPPORTED_INTEGER_CONVERSIONS = Lists.newArrayList(Type.TINYINT, Type.SMALLINT, Type.INT); @@ -174,6 +178,9 @@ private HCatFieldSchema getHCatFSFromPigFS(FieldSchema fSchema, HCatFieldSchema case DataType.BYTEARRAY: return new HCatFieldSchema(fSchema.alias, Type.BINARY, null); + + case DataType.BOOLEAN: + return new HCatFieldSchema(fSchema.alias, Type.BOOLEAN, null); case DataType.BAG: Schema bagSchema = fSchema.schema; @@ -328,8 +335,24 @@ private Object getJavaObj(Object pigObj, HCatFieldSchema hcatFS) throws HCatExce } return ((Integer) pigObj).byteValue(); case BOOLEAN: - // would not pass schema validation anyway - throw new BackendException("Incompatible type " + type + " found in hcat table schema: " + hcatFS, PigHCatUtil.PIG_EXCEPTION_CODE); + if (pigObj == null) { + LOG.debug( "HCatBaseStorer.getJavaObj(BOOLEAN): obj null, bailing early" ); + return null; + } + + if( pigObj instanceof String ) { + if( ((String)pigObj).trim().compareTo("0") == 0 ) { + return Boolean.FALSE; + } + if( ((String)pigObj).trim().compareTo("1") == 0 ) { + return Boolean.TRUE; + } + + throw new BackendException("Unexpected type " + type + " for value " + pigObj + (pigObj == null ? "" : " of class " + pigObj.getClass().getName()), PigHCatUtil.PIG_EXCEPTION_CODE); + } + + return Boolean.parseBoolean( pigObj.toString() ); + default: throw new BackendException("Unexpected type " + type + " for value " + pigObj + (pigObj == null ? "" : " of class " + pigObj.getClass().getName()), PigHCatUtil.PIG_EXCEPTION_CODE); } diff --git hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorer.java hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorer.java index 8019e3e..8ad0def 100644 --- hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorer.java +++ hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorer.java @@ -403,7 +403,7 @@ public void testBagNStruct() throws IOException, CommandNeedRetryException { public void testStoreFuncAllSimpleTypes() throws IOException, CommandNeedRetryException { driver.run("drop table junit_unparted"); - String createTable = "create table junit_unparted(a int, b float, c double, d bigint, e string, f binary, g binary) stored as RCFILE"; + String createTable = "create table junit_unparted(a int, b float, c double, d bigint, e string, h boolean, f binary, g binary) stored as RCFILE"; int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { throw new IOException("Failed to create table."); @@ -411,16 +411,16 @@ public void testStoreFuncAllSimpleTypes() throws IOException, CommandNeedRetryEx int i = 0; String[] input = new String[3]; - input[i++] = "0\t\t\t\t\t\t"; //Empty values except first column - input[i++] = "\t" + i * 2.1f + "\t" + i * 1.1d + "\t" + i * 2L + "\t" + "lets hcat" + "\tbinary-data"; //First column empty - input[i++] = i + "\t" + i * 2.1f + "\t" + i * 1.1d + "\t" + i * 2L + "\t" + "lets hcat" + "\tbinary-data"; + input[i++] = "0\t\t\t\t\t\t\t"; //Empty values except first column + input[i++] = "\t" + i * 2.1f + "\t" + i * 1.1d + "\t" + i * 2L + "\t" + "lets hcat" + "\t" + "true" + "\tbinary-data"; //First column empty + input[i++] = i + "\t" + i * 2.1f + "\t" + i * 1.1d + "\t" + i * 2L + "\t" + "lets hcat" + "\t" + "false" + "\tbinary-data"; HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, input); PigServer server = new PigServer(ExecType.LOCAL); server.setBatchOn(); - server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:float, c:double, d:long, e:chararray, f:bytearray);"); + server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:float, c:double, d:long, e:chararray, h:boolean, f:bytearray);"); //null gets stored into column g which is a binary field. - server.registerQuery("store A into 'default.junit_unparted' using " + HCatStorer.class.getName() + "('','a:int, b:float, c:double, d:long, e:chararray,f:bytearray');"); + server.registerQuery("store A into 'default.junit_unparted' using " + HCatStorer.class.getName() + "('','a:int, b:float, c:double, d:long, e:chararray, h:boolean, f:bytearray');"); server.executeBatch(); @@ -429,9 +429,10 @@ public void testStoreFuncAllSimpleTypes() throws IOException, CommandNeedRetryEx driver.getResults(res); Iterator itr = res.iterator(); - Assert.assertEquals("0\tNULL\tNULL\tNULL\tNULL\tNULL\tNULL", itr.next()); - Assert.assertEquals("NULL\t4.2\t2.2\t4\tlets hcat\tbinary-data\tNULL", itr.next()); - Assert.assertEquals("3\t6.2999997\t3.3000000000000003\t6\tlets hcat\tbinary-data\tNULL", itr.next()); + String next = itr.next(); + Assert.assertEquals("0\tNULL\tNULL\tNULL\tNULL\tNULL\tnull\tnull", next ); + Assert.assertEquals("NULL\t4.2\t2.2\t4\tlets hcat\ttrue\tbinary-data\tnull", itr.next()); + Assert.assertEquals("3\t6.2999997\t3.3000000000000003\t6\tlets hcat\tfalse\tbinary-data\tnull", itr.next()); Assert.assertFalse(itr.hasNext()); server.registerQuery("B = load 'junit_unparted' using " + HCatLoader.class.getName() + ";"); @@ -440,12 +441,12 @@ public void testStoreFuncAllSimpleTypes() throws IOException, CommandNeedRetryEx int num5nulls = 0; while (iter.hasNext()) { Tuple t = iter.next(); - if (t.get(5) == null) { + if (t.get(6) == null) { num5nulls++; } else { - Assert.assertTrue(t.get(5) instanceof DataByteArray); + Assert.assertTrue(t.get(6) instanceof DataByteArray); } - Assert.assertNull(t.get(6)); + Assert.assertNull(t.get(7)); count++; } Assert.assertEquals(3, count);