diff --git a/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatLoader.java b/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatLoader.java index bb5dd4f..d98f40c 100644 --- a/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatLoader.java +++ b/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatLoader.java @@ -38,18 +38,23 @@ import org.apache.hadoop.security.Credentials; import org.apache.hive.hcatalog.common.HCatConstants; import org.apache.hive.hcatalog.common.HCatContext; +import org.apache.hive.hcatalog.common.HCatException; import org.apache.hive.hcatalog.common.HCatUtil; import org.apache.hive.hcatalog.data.Pair; +import org.apache.hive.hcatalog.data.schema.HCatFieldSchema; import org.apache.hive.hcatalog.data.schema.HCatSchema; import org.apache.hive.hcatalog.mapreduce.HCatInputFormat; import org.apache.hive.hcatalog.mapreduce.InputJobInfo; import org.apache.hive.hcatalog.mapreduce.SpecialCases; import org.apache.pig.Expression; import org.apache.pig.Expression.BinaryExpression; +import org.apache.pig.Expression.Const; import org.apache.pig.PigException; import org.apache.pig.ResourceSchema; import org.apache.pig.ResourceStatistics; import org.apache.pig.impl.util.UDFContext; +import org.joda.time.DateTime; +import org.joda.time.format.DateTimeFormat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -273,6 +278,16 @@ private String getPartitionFilterString() { return partitionFilterString; } + private String getHCatConstString(Const con, HCatFieldSchema.Type type) { + Object value = con.getValue(); + switch (type) { + case DATE: + return ((DateTime)value).toString(DateTimeFormat.forPattern("YYYY-MM-dd")); + default: + return con.toString(); + } + } + private String getHCatComparisonString(Expression expr) { if (expr instanceof BinaryExpression) { // call getHCatComparisonString on lhs and rhs, and and join the @@ -290,6 +305,25 @@ private String getHCatComparisonString(Expression expr) { opStr = expr.getOpType().toString(); } BinaryExpression be = (BinaryExpression) expr; + if (be.getRhs() instanceof Const) { + // If the expr is column op const, will try to cast the const to string + // according to the data type of the column + UDFContext udfContext = UDFContext.getUDFContext(); + Properties udfProps = udfContext.getUDFProperties(this.getClass(), + new String[]{signature}); + HCatSchema hcatTableSchema = (HCatSchema) udfProps.get(HCatConstants.HCAT_TABLE_SCHEMA); + HCatFieldSchema fs = null; + try { + fs = hcatTableSchema.get(be.getLhs().toString()); + } catch (HCatException e) { + // Shall never happen + } + if (fs != null) { + return "(" + getHCatComparisonString(be.getLhs()) + + opStr + + getHCatConstString((Const)be.getRhs(), fs.getType()) + ")"; + } + } return "(" + getHCatComparisonString(be.getLhs()) + opStr + getHCatComparisonString(be.getRhs()) + ")"; diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java index 4e23fa2..ab13d22 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java @@ -87,11 +87,13 @@ private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse"; private static final String BASIC_FILE_NAME = TEST_DATA_DIR + "/basic.input.data"; private static final String COMPLEX_FILE_NAME = TEST_DATA_DIR + "/complex.input.data"; + private static final String DATE_FILE_NAME = TEST_DATA_DIR + "/datetimestamp.input.data"; private static final String BASIC_TABLE = "junit_unparted_basic"; private static final String COMPLEX_TABLE = "junit_unparted_complex"; private static final String PARTITIONED_TABLE = "junit_parted_basic"; private static final String SPECIFIC_SIZE_TABLE = "junit_specific_size"; + private static final String PARTITIONED_DATE_TABLE = "junit_parted_date"; private Driver driver; private Map> basicInputData; @@ -104,6 +106,8 @@ add("testProjectionsBasic"); add("testColumnarStorePushdown2"); add("testReadMissingPartitionBasicNeg"); + add("testDatePartitionPushUp"); + add("testTimestampPartitionPushUp"); }}); }}; @@ -197,6 +201,7 @@ public void setup() throws Exception { createTable(PARTITIONED_TABLE, "a int, b string", "bkt string"); createTable(SPECIFIC_SIZE_TABLE, "a int, b string"); + createTable(PARTITIONED_DATE_TABLE, "b string", "dt date"); AllTypesTable.setupAllTypesTable(driver); int LOOP_SIZE = 3; @@ -219,6 +224,12 @@ public void setup() throws Exception { "Edward Hyde\t1337\t(415-253-6367,anonymous@b44chan.org)\t{(CREATIVE_WRITING),(COPYRIGHT_LAW)}\t[CREATIVE_WRITING#A+,COPYRIGHT_LAW#D]\t{(415-253-6367,cell),(408-253-6367,landline)}", } ); + HcatTestUtils.createTestDataFile(DATE_FILE_NAME, + new String[]{ + "2016-07-14 08:10:15\tHenry Jekyll", + "2016-07-15 11:54:55\tEdward Hyde", + } + ); PigServer server = new PigServer(ExecType.LOCAL); server.setBatchOn(); int i = 0; @@ -236,6 +247,11 @@ public void setup() throws Exception { server.registerQuery("D = load '" + COMPLEX_FILE_NAME + "' as (name:chararray, studentid:int, contact:tuple(phno:chararray,email:chararray), currently_registered_courses:bag{innertup:tuple(course:chararray)}, current_grades:map[ ] , phnos :bag{innertup:tuple(phno:chararray,type:chararray)});", ++i); server.registerQuery("store D into '" + COMPLEX_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer();", ++i); + + server.registerQuery("E = load '" + DATE_FILE_NAME + "' as (dt:chararray, b:chararray);", ++i); + server.registerQuery("F = foreach E generate ToDate(dt, 'yyyy-MM-dd HH:mm:ss') as dt, b;", ++i); + server.registerQuery("store F into '" + PARTITIONED_DATE_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer();", ++i); + server.executeBatch(); } @@ -247,6 +263,7 @@ public void tearDown() throws Exception { dropTable(COMPLEX_TABLE); dropTable(PARTITIONED_TABLE); dropTable(SPECIFIC_SIZE_TABLE); + dropTable(PARTITIONED_DATE_TABLE); dropTable(AllTypesTable.ALL_PRIMITIVE_TYPES_TABLE); } } finally { @@ -658,6 +675,25 @@ public void testConvertBooleanToInt() throws Exception { } /** + * Test if we can read a date partitioned table + */ + @Test + public void testDatePartitionPushUp() throws Exception { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); + PigServer server = new PigServer(ExecType.LOCAL); + server.registerQuery("X = load '" + PARTITIONED_DATE_TABLE + "' using " + HCatLoader.class.getName() + "();"); + server.registerQuery("Y = filter X by dt == ToDate('2016-07-14','yyyy-MM-dd');"); + Iterator YIter = server.openIterator("Y"); + int numTuplesRead = 0; + while (YIter.hasNext()) { + Tuple t = YIter.next(); + assertEquals(t.size(), 2); + numTuplesRead++; + } + assertTrue("Expected " + 1 + "; found " + numTuplesRead, numTuplesRead == 1); + } + + /** * basic tests that cover each scalar type * https://issues.apache.org/jira/browse/HIVE-5814 */ diff --git a/pom.xml b/pom.xml index 7d22b4a..d185e34 100644 --- a/pom.xml +++ b/pom.xml @@ -168,7 +168,7 @@ 2.0.0-M5 4.0.23.Final 1.8.1 - 0.12.0 + 0.16.0 2.5.0 1.0.1 1.7.10