diff --git data/files/datatypes.txt data/files/datatypes.txt index 40e43ff..aabdb6e 100644 --- data/files/datatypes.txt +++ data/files/datatypes.txt @@ -1,3 +1,3 @@ -\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N --1false-1.1\N\N\N-1-1-1.0-1\N\N\N\N\N -1true1.11121x2ykva92.2111.01abcd1111213142212212x1abcd22012-04-22 09:00:00.123456789123456789.0123456YWJjZA== +\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N +-1false-1.1\N\N\N-1-1-1.0-1\N\N\N\N\N\N +1true1.11121x2ykva92.2111.01abcd1111213142212212x1abcd22012-04-22 09:00:00.123456789123456789.0123456YWJjZA==2013-01-01 diff --git data/files/flights_join.txt data/files/flights_join.txt new file mode 100644 index 0000000..664a06e --- /dev/null +++ data/files/flights_join.txt @@ -0,0 +1,20 @@ +BaltimoreNew York2010-10-20-30.01064 +BaltimoreNew York2010-10-2123.01142 +BaltimoreNew York2010-10-226.01599 +ChicagoNew York2010-10-2342.0361 +ChicagoNew York2010-10-2424.0897 +ChicagoNew York2010-10-2515.01531 +ChicagoNew York2010-10-26-6.01610 +ChicagoNew York2010-10-27-2.03198 +BaltimoreNew York2010-10-2817.01064 +BaltimoreNew York2010-10-29105.01142 +BaltimoreNew York2000-11-20-30.01064 +BaltimoreNew York2000-11-2123.01142 +BaltimoreNew York2000-11-226.01599 +ChicagoNew York2000-11-2342.0361 +ChicagoNew York2000-11-2424.0897 +ChicagoNew York2000-11-2515.01531 +ChicagoNew York2000-11-26-6.01610 +ChicagoNew York2000-11-27-2.03198 +BaltimoreNew York2000-11-2817.01064 +BaltimoreNew York2000-11-28105.01142 diff --git data/files/flights_tiny.txt.1 data/files/flights_tiny.txt.1 new file mode 100644 index 0000000..532848a --- /dev/null +++ data/files/flights_tiny.txt.1 @@ -0,0 +1,137 @@ +BaltimoreNew York2010-10-20-30.01064 +BaltimoreNew York2010-10-2023.01142 +BaltimoreNew York2010-10-206.01599 +ChicagoNew York2010-10-2042.0361 +ChicagoNew York2010-10-2024.0897 +ChicagoNew York2010-10-2015.01531 +ChicagoNew York2010-10-20-6.01610 +ChicagoNew York2010-10-20-2.03198 +BaltimoreNew York2010-10-2117.01064 +BaltimoreNew York2010-10-21105.01142 +BaltimoreNew York2010-10-2128.01599 +ChicagoNew York2010-10-21142.0361 +ChicagoNew York2010-10-2177.0897 +ChicagoNew York2010-10-2153.01531 +ChicagoNew York2010-10-21-5.01610 +ChicagoNew York2010-10-2151.03198 +BaltimoreNew York2010-10-22-12.01064 +BaltimoreNew York2010-10-2254.01142 +BaltimoreNew York2010-10-2218.01599 +ChicagoNew York2010-10-222.0361 +ChicagoNew York2010-10-2224.0897 +ChicagoNew York2010-10-2216.01531 +ChicagoNew York2010-10-22-6.01610 +ChicagoNew York2010-10-22-11.03198 +BaltimoreNew York2010-10-2318.0272 +BaltimoreNew York2010-10-23-10.01805 +BaltimoreNew York2010-10-236.03171 +ChicagoNew York2010-10-233.0384 +ChicagoNew York2010-10-2332.0426 +ChicagoNew York2010-10-231.0650 +ChicagoNew York2010-10-2311.03085 +BaltimoreNew York2010-10-2412.01599 +BaltimoreNew York2010-10-2420.02571 +ChicagoNew York2010-10-2410.0361 +ChicagoNew York2010-10-24113.0897 +ChicagoNew York2010-10-24-5.01531 +ChicagoNew York2010-10-24-17.01610 +ChicagoNew York2010-10-24-3.03198 +BaltimoreNew York2010-10-25-25.01064 +BaltimoreNew York2010-10-2592.01142 +BaltimoreNew York2010-10-25106.01599 +ChicagoNew York2010-10-2531.0361 +ChicagoNew York2010-10-25-1.0897 +ChicagoNew York2010-10-2543.01531 +ChicagoNew York2010-10-256.01610 +ChicagoNew York2010-10-25-16.03198 +BaltimoreNew York2010-10-26-22.01064 +BaltimoreNew York2010-10-26123.01142 +BaltimoreNew York2010-10-2690.01599 +ChicagoNew York2010-10-2612.0361 +ChicagoNew York2010-10-260.0897 +ChicagoNew York2010-10-2629.01531 +ChicagoNew York2010-10-26-17.01610 +ChicagoNew York2010-10-266.03198 +BaltimoreNew York2010-10-27-18.01064 +BaltimoreNew York2010-10-2749.01142 +BaltimoreNew York2010-10-2792.01599 +ChicagoNew York2010-10-27148.0361 +ChicagoNew York2010-10-27-11.0897 +ChicagoNew York2010-10-2770.01531 +ChicagoNew York2010-10-278.01610 +ChicagoNew York2010-10-2721.03198 +BaltimoreNew York2010-10-28-4.01064 +BaltimoreNew York2010-10-28-14.01142 +BaltimoreNew York2010-10-28-14.01599 +ChicagoNew York2010-10-282.0361 +ChicagoNew York2010-10-282.0897 +ChicagoNew York2010-10-28-11.01531 +ChicagoNew York2010-10-283.01610 +ChicagoNew York2010-10-28-18.03198 +BaltimoreNew York2010-10-29-24.01064 +BaltimoreNew York2010-10-2921.01142 +BaltimoreNew York2010-10-29-2.01599 +ChicagoNew York2010-10-29-12.0361 +ChicagoNew York2010-10-29-11.0897 +ChicagoNew York2010-10-2915.01531 +ChicagoNew York2010-10-29-18.01610 +ChicagoNew York2010-10-29-4.03198 +BaltimoreNew York2010-10-3014.0272 +BaltimoreNew York2010-10-30-1.01805 +BaltimoreNew York2010-10-305.03171 +ChicagoNew York2010-10-30-6.0384 +ChicagoNew York2010-10-30-10.0426 +ChicagoNew York2010-10-30-5.0650 +ChicagoNew York2010-10-30-5.03085 +BaltimoreNew York2010-10-31-1.01599 +BaltimoreNew York2010-10-31-14.02571 +ChicagoNew York2010-10-31-25.0361 +ChicagoNew York2010-10-31-18.0897 +ChicagoNew York2010-10-31-4.01531 +ChicagoNew York2010-10-31-22.01610 +ChicagoNew York2010-10-31-15.03198 +ClevelandNew York2010-10-30-23.02018 +ClevelandNew York2010-10-30-12.02932 +ClevelandNew York2010-10-29-4.02630 +ClevelandNew York2010-10-29-19.02646 +ClevelandNew York2010-10-29-12.03014 +ClevelandNew York2010-10-283.02630 +ClevelandNew York2010-10-28-6.02646 +ClevelandNew York2010-10-281.03014 +ClevelandNew York2010-10-2716.02630 +ClevelandNew York2010-10-2727.03014 +ClevelandNew York2010-10-264.02630 +ClevelandNew York2010-10-26-27.02646 +ClevelandNew York2010-10-26-11.02662 +ClevelandNew York2010-10-2613.03014 +ClevelandNew York2010-10-25-4.02630 +ClevelandNew York2010-10-2581.02646 +ClevelandNew York2010-10-2542.03014 +ClevelandNew York2010-10-245.02254 +ClevelandNew York2010-10-24-11.02630 +ClevelandNew York2010-10-24-20.02646 +ClevelandNew York2010-10-24-9.03014 +ClevelandNew York2010-10-23-21.02932 +ClevelandNew York2010-10-221.02630 +ClevelandNew York2010-10-22-25.02646 +ClevelandNew York2010-10-22-3.03014 +ClevelandNew York2010-10-213.02630 +ClevelandNew York2010-10-2129.02646 +ClevelandNew York2010-10-2172.03014 +ClevelandNew York2010-10-20-8.02630 +ClevelandNew York2010-10-20-15.03014 +WashingtonNew York2010-10-23-25.05832 +WashingtonNew York2010-10-23-21.05904 +WashingtonNew York2010-10-23-18.05917 +WashingtonNew York2010-10-30-27.05904 +WashingtonNew York2010-10-30-16.05917 +WashingtonNew York2010-10-20-2.07291 +WashingtonNew York2010-10-2122.07291 +WashingtonNew York2010-10-23-16.07274 +WashingtonNew York2010-10-24-26.07282 +WashingtonNew York2010-10-259.07291 +WashingtonNew York2010-10-264.07291 +WashingtonNew York2010-10-2726.07291 +WashingtonNew York2010-10-2845.07291 +WashingtonNew York2010-10-291.07291 +WashingtonNew York2010-10-31-18.07282 diff --git jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveBaseResultSet.java jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveBaseResultSet.java index 092fa28..d18a6a3 100644 --- jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveBaseResultSet.java +++ jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveBaseResultSet.java @@ -219,12 +219,20 @@ public Date getDate(int columnIndex) throws SQLException { return null; } + if (obj instanceof Date) { + return (Date) obj; + } + try { - return Date.valueOf((String) obj); + if (obj instanceof String) { + return Date.valueOf((String)obj); + } } catch (Exception e) { throw new SQLException("Cannot convert column " + columnIndex + " to date: " + e.toString()); } + + throw new SQLException("Illegal conversion"); } public Date getDate(String columnName) throws SQLException { diff --git jdbc/src/java/org/apache/hadoop/hive/jdbc/HivePreparedStatement.java jdbc/src/java/org/apache/hadoop/hive/jdbc/HivePreparedStatement.java index 2859859..827104a 100644 --- jdbc/src/java/org/apache/hadoop/hive/jdbc/HivePreparedStatement.java +++ jdbc/src/java/org/apache/hadoop/hive/jdbc/HivePreparedStatement.java @@ -496,8 +496,7 @@ public void setClob(int parameterIndex, Reader reader, long length) throws SQLEx */ public void setDate(int parameterIndex, Date x) throws SQLException { - // TODO Auto-generated method stub - throw new SQLException("Method not supported"); + this.parameters.put(parameterIndex, x.toString()); } /* diff --git jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveResultSetMetaData.java jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveResultSetMetaData.java index 48b2e86..94b6ecd 100644 --- jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveResultSetMetaData.java +++ jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveResultSetMetaData.java @@ -110,6 +110,8 @@ public String getColumnTypeName(int column) throws SQLException { return serdeConstants.INT_TYPE_NAME; } else if ("bigint".equalsIgnoreCase(type)) { return serdeConstants.BIGINT_TYPE_NAME; + } else if ("date".equalsIgnoreCase(type)) { + return serdeConstants.DATE_TYPE_NAME; } else if ("timestamp".equalsIgnoreCase(type)) { return serdeConstants.TIMESTAMP_TYPE_NAME; } else if ("decimal".equalsIgnoreCase(type)) { diff --git jdbc/src/java/org/apache/hadoop/hive/jdbc/JdbcColumn.java jdbc/src/java/org/apache/hadoop/hive/jdbc/JdbcColumn.java index 920a4ba..d3a87e2 100644 --- jdbc/src/java/org/apache/hadoop/hive/jdbc/JdbcColumn.java +++ jdbc/src/java/org/apache/hadoop/hive/jdbc/JdbcColumn.java @@ -74,6 +74,8 @@ static int columnDisplaySize(int columnType) throws SQLException { case Types.INTEGER: case Types.BIGINT: return columnPrecision(columnType) + 1; // allow +/- + case Types.DATE: + return 10; case Types.TIMESTAMP: return columnPrecision(columnType); // see http://download.oracle.com/javase/6/docs/api/constant-values.html#java.lang.Float.MAX_EXPONENT @@ -108,6 +110,8 @@ static int columnPrecision(int columnType) throws SQLException { return 7; case Types.DOUBLE: return 15; + case Types.DATE: + return 10; case Types.TIMESTAMP: return 29; case Types.DECIMAL: @@ -126,6 +130,7 @@ static int columnScale(int columnType) throws SQLException { case Types.SMALLINT: case Types.INTEGER: case Types.BIGINT: + case Types.DATE: return 0; case Types.FLOAT: return 7; diff --git jdbc/src/java/org/apache/hadoop/hive/jdbc/Utils.java jdbc/src/java/org/apache/hadoop/hive/jdbc/Utils.java index c93d00b..bd98274 100644 --- jdbc/src/java/org/apache/hadoop/hive/jdbc/Utils.java +++ jdbc/src/java/org/apache/hadoop/hive/jdbc/Utils.java @@ -46,6 +46,8 @@ public static int hiveTypeToSqlType(String type) throws SQLException { return Types.INTEGER; } else if ("bigint".equalsIgnoreCase(type)) { return Types.BIGINT; + } else if ("date".equalsIgnoreCase(type)) { + return Types.DATE; } else if ("timestamp".equalsIgnoreCase(type)) { return Types.TIMESTAMP; } else if ("decimal".equalsIgnoreCase(type)) { diff --git jdbc/src/java/org/apache/hive/jdbc/HiveBaseResultSet.java jdbc/src/java/org/apache/hive/jdbc/HiveBaseResultSet.java index 689ba14..f47df43 100644 --- jdbc/src/java/org/apache/hive/jdbc/HiveBaseResultSet.java +++ jdbc/src/java/org/apache/hive/jdbc/HiveBaseResultSet.java @@ -224,13 +224,20 @@ public Date getDate(int columnIndex) throws SQLException { if (obj == null) { return null; } - + if (obj instanceof Date) { + return (Date) obj; + } try { - return Date.valueOf((String) obj); + if (obj instanceof String) { + return Date.valueOf((String)obj); + } } catch (Exception e) { throw new SQLException("Cannot convert column " + columnIndex + " to date: " + e.toString(), e); } + // If we fell through to here this is not a valid type conversion + throw new SQLException("Cannot convert column " + columnIndex + + " to date: Illegal conversion"); } public Date getDate(String columnName) throws SQLException { @@ -434,6 +441,15 @@ private String getStringValue(TStringValue tStringValue) { return null; } + private Date getDateValue(TStringValue tStringValue) { + if (tStringValue.isSetValue()) { + wasNull = false; + return Date.valueOf(tStringValue.getValue()); + } + wasNull = true; + return null; + } + private Timestamp getTimestampValue(TStringValue tStringValue) { if (tStringValue.isSetValue()) { wasNull = false; @@ -495,6 +511,8 @@ private Object getColumnValue(int columnIndex) throws SQLException { return getStringValue(tColumnValue.getStringVal()); case BINARY_TYPE: return getBinaryValue(tColumnValue.getStringVal()); + case DATE_TYPE: + return getDateValue(tColumnValue.getStringVal()); case TIMESTAMP_TYPE: return getTimestampValue(tColumnValue.getStringVal()); case DECIMAL_TYPE: diff --git jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java index b040d39..dfcd536 100644 --- jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java +++ jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java @@ -43,10 +43,10 @@ import java.util.HashMap; import java.util.Map; +import org.apache.hive.service.cli.thrift.TCLIService; import org.apache.hive.service.cli.thrift.TExecuteStatementReq; import org.apache.hive.service.cli.thrift.TExecuteStatementResp; import org.apache.hive.service.cli.thrift.TOperationHandle; -import org.apache.hive.service.cli.thrift.TCLIService; import org.apache.hive.service.cli.thrift.TSessionHandle; /** @@ -510,8 +510,7 @@ public void setClob(int parameterIndex, Reader reader, long length) throws SQLEx */ public void setDate(int parameterIndex, Date x) throws SQLException { - // TODO Auto-generated method stub - throw new SQLException("Method not supported"); + this.parameters.put(parameterIndex, x.toString()); } /* diff --git jdbc/src/java/org/apache/hive/jdbc/HiveResultSetMetaData.java jdbc/src/java/org/apache/hive/jdbc/HiveResultSetMetaData.java index b2b91c8..ec6fee8 100644 --- jdbc/src/java/org/apache/hive/jdbc/HiveResultSetMetaData.java +++ jdbc/src/java/org/apache/hive/jdbc/HiveResultSetMetaData.java @@ -105,6 +105,8 @@ public String getColumnTypeName(int column) throws SQLException { return serdeConstants.BIGINT_TYPE_NAME; } else if ("timestamp".equalsIgnoreCase(type)) { return serdeConstants.TIMESTAMP_TYPE_NAME; + } else if ("date".equalsIgnoreCase(type)) { + return serdeConstants.DATE_TYPE_NAME; } else if ("decimal".equalsIgnoreCase(type)) { return serdeConstants.DECIMAL_TYPE_NAME; } else if ("binary".equalsIgnoreCase(type)) { diff --git jdbc/src/java/org/apache/hive/jdbc/JdbcColumn.java jdbc/src/java/org/apache/hive/jdbc/JdbcColumn.java index 25c3f3f..5553d7d 100644 --- jdbc/src/java/org/apache/hive/jdbc/JdbcColumn.java +++ jdbc/src/java/org/apache/hive/jdbc/JdbcColumn.java @@ -75,6 +75,8 @@ static int columnDisplaySize(int columnType) throws SQLException { case Types.INTEGER: case Types.BIGINT: return columnPrecision(columnType) + 1; // allow +/- + case Types.DATE: + return 10; case Types.TIMESTAMP: return columnPrecision(columnType); @@ -110,6 +112,8 @@ static int columnPrecision(int columnType) throws SQLException { return 7; case Types.DOUBLE: return 15; + case Types.DATE: + return 10; case Types.TIMESTAMP: return 29; case Types.DECIMAL: @@ -128,6 +132,7 @@ static int columnScale(int columnType) throws SQLException { case Types.SMALLINT: case Types.INTEGER: case Types.BIGINT: + case Types.DATE: return 0; case Types.FLOAT: return 7; diff --git jdbc/src/java/org/apache/hive/jdbc/Utils.java jdbc/src/java/org/apache/hive/jdbc/Utils.java index d907f0a..3df3bd7 100644 --- jdbc/src/java/org/apache/hive/jdbc/Utils.java +++ jdbc/src/java/org/apache/hive/jdbc/Utils.java @@ -128,6 +128,8 @@ public static int hiveTypeToSqlType(String type) throws SQLException { return Types.INTEGER; } else if ("bigint".equalsIgnoreCase(type)) { return Types.BIGINT; + } else if ("date".equalsIgnoreCase(type)) { + return Types.DATE; } else if ("timestamp".equalsIgnoreCase(type)) { return Types.TIMESTAMP; } else if ("decimal".equalsIgnoreCase(type)) { diff --git jdbc/src/test/org/apache/hadoop/hive/jdbc/TestJdbcDriver.java jdbc/src/test/org/apache/hadoop/hive/jdbc/TestJdbcDriver.java index 2e1e580..e1107dd 100644 --- jdbc/src/test/org/apache/hadoop/hive/jdbc/TestJdbcDriver.java +++ jdbc/src/test/org/apache/hadoop/hive/jdbc/TestJdbcDriver.java @@ -150,7 +150,9 @@ protected void setUp() throws Exception { + " c15 struct>," + " c16 array,n:int>>," + " c17 timestamp, " - + " c18 decimal) comment'" + dataTypeTableComment + + " c18 decimal," + + " c19 binary," + + " c20 date) comment'" + dataTypeTableComment +"' partitioned by (dt STRING)"); assertFalse(res.next()); @@ -226,6 +228,7 @@ public void testPrepareStatement() { + tableName + " where 'not?param?not?param' <> 'not_param??not_param' and ?=? " + " and 1=? and 2=? and 3.0=? and 4.0=? and 'test\\'string\"'=? and 5=? and ?=? " + + " and date '2012-01-01' = date ?" + " ) t select '2011-03-25' ddate,'China',true bv, 10 num limit 10"; /////////////////////////////////////////////// @@ -296,7 +299,7 @@ public void testPrepareStatement() { assertNotNull( "Execute the invalid setted sql statement should throw exception", expectedException); - + // setObject to the yet unknown type java.util.Date expectedException = null; try { @@ -326,6 +329,7 @@ private PreparedStatement createPreapredStatementUsingSetObject(String sql) thro ps.setObject(8, 5L); //setLong ps.setObject(9, (byte) 1); //setByte ps.setObject(10, (byte) 1); //setByte + ps.setString(11, "2012-01-01"); //setString ps.setMaxRows(2); return ps; @@ -345,6 +349,7 @@ private PreparedStatement createPreapredStatementUsingSetXXX(String sql) throws ps.setLong(8, 5L); //setLong ps.setByte(9, (byte) 1); //setByte ps.setByte(10, (byte) 1); //setByte + ps.setString(11, "2012-01-01"); //setString ps.setMaxRows(2); return ps; @@ -438,6 +443,8 @@ public void testDataTypes() throws Exception { assertEquals(null, res.getString(17)); assertEquals(null, res.getTimestamp(17)); assertEquals(null, res.getBigDecimal(18)); + assertEquals(null, res.getString(20)); + assertEquals(null, res.getDate(20)); // row 3 assertTrue(res.next()); @@ -460,6 +467,8 @@ public void testDataTypes() throws Exception { assertEquals("2012-04-22 09:00:00.123456789", res.getString(17)); assertEquals("2012-04-22 09:00:00.123456789", res.getTimestamp(17).toString()); assertEquals("123456789.0123456", res.getBigDecimal(18).toString()); + assertEquals("2013-01-01", res.getString(20)); + assertEquals("2013-01-01", res.getDate(20).toString()); // test getBoolean rules on non-boolean columns assertEquals(true, res.getBoolean(1)); @@ -850,13 +859,14 @@ public void testResultSetMetaData() throws SQLException { ResultSet res = stmt.executeQuery( "select c1, c2, c3, c4, c5 as a, c6, c7, c8, c9, c10, c11, c12, " + - "c1*2, sentences(null, null, null) as b, c17, c18 from " + dataTypeTableName + " limit 1"); + "c1*2, sentences(null, null, null) as b, c17, c18, c20 from " + dataTypeTableName + + " limit 1"); ResultSetMetaData meta = res.getMetaData(); ResultSet colRS = con.getMetaData().getColumns(null, null, dataTypeTableName.toLowerCase(), null); - assertEquals(16, meta.getColumnCount()); + assertEquals(17, meta.getColumnCount()); assertTrue(colRS.next()); @@ -1066,6 +1076,13 @@ public void testResultSetMetaData() throws SQLException { assertEquals(Integer.MAX_VALUE, meta.getPrecision(16)); assertEquals(Integer.MAX_VALUE, meta.getScale(16)); + assertEquals("c20", meta.getColumnName(17)); + assertEquals(Types.DATE, meta.getColumnType(17)); + assertEquals("date", meta.getColumnTypeName(17)); + assertEquals(10, meta.getColumnDisplaySize(17)); + assertEquals(10, meta.getPrecision(17)); + assertEquals(0, meta.getScale(17)); + for (int i = 1; i <= meta.getColumnCount(); i++) { assertFalse(meta.isAutoIncrement(i)); assertFalse(meta.isCurrency(i)); diff --git jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver2.java jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver2.java index fd990a8..1042125 100644 --- jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver2.java +++ jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver2.java @@ -150,7 +150,8 @@ protected void setUp() throws Exception { + " c16 array,n:int>>," + " c17 timestamp, " + " c18 decimal, " - + " c19 binary) comment'" + dataTypeTableComment + + " c19 binary, " + + " c20 date) comment'" + dataTypeTableComment +"' partitioned by (dt STRING)"); stmt.execute("load data local inpath '" @@ -278,6 +279,7 @@ public void testPrepareStatement() { + tableName + " where 'not?param?not?param' <> 'not_param??not_param' and ?=? " + " and 1=? and 2=? and 3.0=? and 4.0=? and 'test\\'string\"'=? and 5=? and ?=? " + + " and date '2012-01-01' = date ?" + " ) t select '2011-03-25' ddate,'China',true bv, 10 num limit 10"; /////////////////////////////////////////////// @@ -297,6 +299,7 @@ public void testPrepareStatement() { ps.setLong(8, 5L); ps.setByte(9, (byte) 1); ps.setByte(10, (byte) 1); + ps.setString(11, "2012-01-01"); ps.setMaxRows(2); @@ -445,6 +448,8 @@ public void testDataTypes() throws Exception { assertEquals(null, res.getString(17)); assertEquals(null, res.getString(18)); assertEquals(null, res.getString(19)); + assertEquals(null, res.getString(20)); + assertEquals(null, res.getDate(20)); // row 2 assertTrue(res.next()); @@ -468,6 +473,8 @@ public void testDataTypes() throws Exception { assertEquals(null, res.getTimestamp(17)); assertEquals(null, res.getBigDecimal(18)); assertEquals(null, res.getString(19)); + assertEquals(null, res.getString(20)); + assertEquals(null, res.getDate(20)); // row 3 assertTrue(res.next()); @@ -491,6 +498,8 @@ public void testDataTypes() throws Exception { assertEquals("2012-04-22 09:00:00.123456789", res.getTimestamp(17).toString()); assertEquals("123456789.0123456", res.getBigDecimal(18).toString()); assertEquals("abcd", res.getString(19)); + assertEquals("2013-01-01", res.getString(20)); + assertEquals("2013-01-01", res.getDate(20).toString()); // test getBoolean rules on non-boolean columns assertEquals(true, res.getBoolean(1)); @@ -899,13 +908,14 @@ public void testResultSetMetaData() throws SQLException { ResultSet res = stmt.executeQuery( "select c1, c2, c3, c4, c5 as a, c6, c7, c8, c9, c10, c11, c12, " + - "c1*2, sentences(null, null, null) as b, c17, c18 from " + dataTypeTableName + " limit 1"); + "c1*2, sentences(null, null, null) as b, c17, c18, c20 from " + dataTypeTableName + + " limit 1"); ResultSetMetaData meta = res.getMetaData(); ResultSet colRS = con.getMetaData().getColumns(null, null, dataTypeTableName.toLowerCase(), null); - assertEquals(16, meta.getColumnCount()); + assertEquals(17, meta.getColumnCount()); assertTrue(colRS.next()); @@ -1106,6 +1116,13 @@ public void testResultSetMetaData() throws SQLException { assertEquals(Integer.MAX_VALUE, meta.getPrecision(16)); assertEquals(Integer.MAX_VALUE, meta.getScale(16)); + assertEquals("c20", meta.getColumnName(17)); + assertEquals(Types.DATE, meta.getColumnType(17)); + assertEquals("date", meta.getColumnTypeName(17)); + assertEquals(10, meta.getColumnDisplaySize(17)); + assertEquals(10, meta.getPrecision(17)); + assertEquals(0, meta.getScale(17)); + for (int i = 1; i <= meta.getColumnCount(); i++) { assertFalse(meta.isAutoIncrement(i)); assertFalse(meta.isCurrency(i)); diff --git ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java index f6acc00..19a6d0e 100644 --- ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java +++ ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java @@ -2415,6 +2415,401 @@ void setSum(com.google.protobuf.ByteString value) { // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.ql.io.orc.DecimalStatistics) } + public interface DateStatisticsOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // optional sint32 minimum = 1; + boolean hasMinimum(); + int getMinimum(); + + // optional sint32 maximum = 2; + boolean hasMaximum(); + int getMaximum(); + } + public static final class DateStatistics extends + com.google.protobuf.GeneratedMessage + implements DateStatisticsOrBuilder { + // Use DateStatistics.newBuilder() to construct. + private DateStatistics(Builder builder) { + super(builder); + } + private DateStatistics(boolean noInit) {} + + private static final DateStatistics defaultInstance; + public static DateStatistics getDefaultInstance() { + return defaultInstance; + } + + public DateStatistics getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_DateStatistics_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_DateStatistics_fieldAccessorTable; + } + + private int bitField0_; + // optional sint32 minimum = 1; + public static final int MINIMUM_FIELD_NUMBER = 1; + private int minimum_; + public boolean hasMinimum() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public int getMinimum() { + return minimum_; + } + + // optional sint32 maximum = 2; + public static final int MAXIMUM_FIELD_NUMBER = 2; + private int maximum_; + public boolean hasMaximum() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public int getMaximum() { + return maximum_; + } + + private void initFields() { + minimum_ = 0; + maximum_ = 0; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeSInt32(1, minimum_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeSInt32(2, maximum_); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeSInt32Size(1, minimum_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeSInt32Size(2, maximum_); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatisticsOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_DateStatistics_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_DateStatistics_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + minimum_ = 0; + bitField0_ = (bitField0_ & ~0x00000001); + maximum_ = 0; + bitField0_ = (bitField0_ & ~0x00000002); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics.getDescriptor(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics getDefaultInstanceForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics.getDefaultInstance(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics build() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics buildPartial() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics result = new org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.minimum_ = minimum_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.maximum_ = maximum_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics) { + return mergeFrom((org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics other) { + if (other == org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics.getDefaultInstance()) return this; + if (other.hasMinimum()) { + setMinimum(other.getMinimum()); + } + if (other.hasMaximum()) { + setMaximum(other.getMaximum()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 8: { + bitField0_ |= 0x00000001; + minimum_ = input.readSInt32(); + break; + } + case 16: { + bitField0_ |= 0x00000002; + maximum_ = input.readSInt32(); + break; + } + } + } + } + + private int bitField0_; + + // optional sint32 minimum = 1; + private int minimum_ ; + public boolean hasMinimum() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public int getMinimum() { + return minimum_; + } + public Builder setMinimum(int value) { + bitField0_ |= 0x00000001; + minimum_ = value; + onChanged(); + return this; + } + public Builder clearMinimum() { + bitField0_ = (bitField0_ & ~0x00000001); + minimum_ = 0; + onChanged(); + return this; + } + + // optional sint32 maximum = 2; + private int maximum_ ; + public boolean hasMaximum() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public int getMaximum() { + return maximum_; + } + public Builder setMaximum(int value) { + bitField0_ |= 0x00000002; + maximum_ = value; + onChanged(); + return this; + } + public Builder clearMaximum() { + bitField0_ = (bitField0_ & ~0x00000002); + maximum_ = 0; + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hive.ql.io.orc.DateStatistics) + } + + static { + defaultInstance = new DateStatistics(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.ql.io.orc.DateStatistics) + } + public interface ColumnStatisticsOrBuilder extends com.google.protobuf.MessageOrBuilder { @@ -2446,6 +2841,11 @@ void setSum(com.google.protobuf.ByteString value) { boolean hasDecimalStatistics(); org.apache.hadoop.hive.ql.io.orc.OrcProto.DecimalStatistics getDecimalStatistics(); org.apache.hadoop.hive.ql.io.orc.OrcProto.DecimalStatisticsOrBuilder getDecimalStatisticsOrBuilder(); + + // optional .org.apache.hadoop.hive.ql.io.orc.DateStatistics dateStatistics = 7; + boolean hasDateStatistics(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics getDateStatistics(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatisticsOrBuilder getDateStatisticsOrBuilder(); } public static final class ColumnStatistics extends com.google.protobuf.GeneratedMessage @@ -2551,6 +2951,19 @@ public boolean hasDecimalStatistics() { return decimalStatistics_; } + // optional .org.apache.hadoop.hive.ql.io.orc.DateStatistics dateStatistics = 7; + public static final int DATESTATISTICS_FIELD_NUMBER = 7; + private org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics dateStatistics_; + public boolean hasDateStatistics() { + return ((bitField0_ & 0x00000040) == 0x00000040); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics getDateStatistics() { + return dateStatistics_; + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatisticsOrBuilder getDateStatisticsOrBuilder() { + return dateStatistics_; + } + private void initFields() { numberOfValues_ = 0L; intStatistics_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics.getDefaultInstance(); @@ -2558,6 +2971,7 @@ private void initFields() { stringStatistics_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics.getDefaultInstance(); bucketStatistics_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics.getDefaultInstance(); decimalStatistics_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.DecimalStatistics.getDefaultInstance(); + dateStatistics_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics.getDefaultInstance(); } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -2589,6 +3003,9 @@ public void writeTo(com.google.protobuf.CodedOutputStream output) if (((bitField0_ & 0x00000020) == 0x00000020)) { output.writeMessage(6, decimalStatistics_); } + if (((bitField0_ & 0x00000040) == 0x00000040)) { + output.writeMessage(7, dateStatistics_); + } getUnknownFields().writeTo(output); } @@ -2622,6 +3039,10 @@ public int getSerializedSize() { size += com.google.protobuf.CodedOutputStream .computeMessageSize(6, decimalStatistics_); } + if (((bitField0_ & 0x00000040) == 0x00000040)) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(7, dateStatistics_); + } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; @@ -2743,6 +3164,7 @@ private void maybeForceBuilderInitialization() { getStringStatisticsFieldBuilder(); getBucketStatisticsFieldBuilder(); getDecimalStatisticsFieldBuilder(); + getDateStatisticsFieldBuilder(); } } private static Builder create() { @@ -2783,6 +3205,12 @@ public Builder clear() { decimalStatisticsBuilder_.clear(); } bitField0_ = (bitField0_ & ~0x00000020); + if (dateStatisticsBuilder_ == null) { + dateStatistics_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics.getDefaultInstance(); + } else { + dateStatisticsBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000040); return this; } @@ -2865,6 +3293,14 @@ public Builder clone() { } else { result.decimalStatistics_ = decimalStatisticsBuilder_.build(); } + if (((from_bitField0_ & 0x00000040) == 0x00000040)) { + to_bitField0_ |= 0x00000040; + } + if (dateStatisticsBuilder_ == null) { + result.dateStatistics_ = dateStatistics_; + } else { + result.dateStatistics_ = dateStatisticsBuilder_.build(); + } result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -2899,6 +3335,9 @@ public Builder mergeFrom(org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatist if (other.hasDecimalStatistics()) { mergeDecimalStatistics(other.getDecimalStatistics()); } + if (other.hasDateStatistics()) { + mergeDateStatistics(other.getDateStatistics()); + } this.mergeUnknownFields(other.getUnknownFields()); return this; } @@ -2980,6 +3419,15 @@ public Builder mergeFrom( setDecimalStatistics(subBuilder.buildPartial()); break; } + case 58: { + org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics.Builder subBuilder = org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics.newBuilder(); + if (hasDateStatistics()) { + subBuilder.mergeFrom(getDateStatistics()); + } + input.readMessage(subBuilder, extensionRegistry); + setDateStatistics(subBuilder.buildPartial()); + break; + } } } } @@ -3457,6 +3905,96 @@ public Builder clearDecimalStatistics() { return decimalStatisticsBuilder_; } + // optional .org.apache.hadoop.hive.ql.io.orc.DateStatistics dateStatistics = 7; + private org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics dateStatistics_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics.getDefaultInstance(); + private com.google.protobuf.SingleFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatisticsOrBuilder> dateStatisticsBuilder_; + public boolean hasDateStatistics() { + return ((bitField0_ & 0x00000040) == 0x00000040); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics getDateStatistics() { + if (dateStatisticsBuilder_ == null) { + return dateStatistics_; + } else { + return dateStatisticsBuilder_.getMessage(); + } + } + public Builder setDateStatistics(org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics value) { + if (dateStatisticsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + dateStatistics_ = value; + onChanged(); + } else { + dateStatisticsBuilder_.setMessage(value); + } + bitField0_ |= 0x00000040; + return this; + } + public Builder setDateStatistics( + org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics.Builder builderForValue) { + if (dateStatisticsBuilder_ == null) { + dateStatistics_ = builderForValue.build(); + onChanged(); + } else { + dateStatisticsBuilder_.setMessage(builderForValue.build()); + } + bitField0_ |= 0x00000040; + return this; + } + public Builder mergeDateStatistics(org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics value) { + if (dateStatisticsBuilder_ == null) { + if (((bitField0_ & 0x00000040) == 0x00000040) && + dateStatistics_ != org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics.getDefaultInstance()) { + dateStatistics_ = + org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics.newBuilder(dateStatistics_).mergeFrom(value).buildPartial(); + } else { + dateStatistics_ = value; + } + onChanged(); + } else { + dateStatisticsBuilder_.mergeFrom(value); + } + bitField0_ |= 0x00000040; + return this; + } + public Builder clearDateStatistics() { + if (dateStatisticsBuilder_ == null) { + dateStatistics_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics.getDefaultInstance(); + onChanged(); + } else { + dateStatisticsBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000040); + return this; + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics.Builder getDateStatisticsBuilder() { + bitField0_ |= 0x00000040; + onChanged(); + return getDateStatisticsFieldBuilder().getBuilder(); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatisticsOrBuilder getDateStatisticsOrBuilder() { + if (dateStatisticsBuilder_ != null) { + return dateStatisticsBuilder_.getMessageOrBuilder(); + } else { + return dateStatistics_; + } + } + private com.google.protobuf.SingleFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatisticsOrBuilder> + getDateStatisticsFieldBuilder() { + if (dateStatisticsBuilder_ == null) { + dateStatisticsBuilder_ = new com.google.protobuf.SingleFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatisticsOrBuilder>( + dateStatistics_, + getParentForChildren(), + isClean()); + dateStatistics_ = null; + } + return dateStatisticsBuilder_; + } + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hive.ql.io.orc.ColumnStatistics) } @@ -6505,6 +7043,7 @@ public Type getDefaultInstanceForType() { STRUCT(12, 12), UNION(13, 13), DECIMAL(14, 14), + DATE(15, 15), ; public static final int BOOLEAN_VALUE = 0; @@ -6522,6 +7061,7 @@ public Type getDefaultInstanceForType() { public static final int STRUCT_VALUE = 12; public static final int UNION_VALUE = 13; public static final int DECIMAL_VALUE = 14; + public static final int DATE_VALUE = 15; public final int getNumber() { return value; } @@ -6543,6 +7083,7 @@ public static Kind valueOf(int value) { case 12: return STRUCT; case 13: return UNION; case 14: return DECIMAL; + case 15: return DATE; default: return null; } } @@ -6573,7 +7114,7 @@ public Kind findValueByNumber(int number) { } private static final Kind[] VALUES = { - BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, BINARY, TIMESTAMP, LIST, MAP, STRUCT, UNION, DECIMAL, + BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, BINARY, TIMESTAMP, LIST, MAP, STRUCT, UNION, DECIMAL, DATE, }; public static Kind valueOf( @@ -10476,6 +11017,11 @@ void setMagic(com.google.protobuf.ByteString value) { com.google.protobuf.GeneratedMessage.FieldAccessorTable internal_static_org_apache_hadoop_hive_ql_io_orc_DecimalStatistics_fieldAccessorTable; private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hive_ql_io_orc_DateStatistics_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hive_ql_io_orc_DateStatistics_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor internal_static_org_apache_hadoop_hive_ql_io_orc_ColumnStatistics_descriptor; private static com.google.protobuf.GeneratedMessage.FieldAccessorTable @@ -10547,62 +11093,66 @@ void setMagic(com.google.protobuf.ByteString value) { "istics\022\017\n\007minimum\030\001 \001(\t\022\017\n\007maximum\030\002 \001(\t" + "\"%\n\020BucketStatistics\022\021\n\005count\030\001 \003(\004B\002\020\001\"" + "B\n\021DecimalStatistics\022\017\n\007minimum\030\001 \001(\t\022\017\n" + - "\007maximum\030\002 \001(\t\022\013\n\003sum\030\003 \001(\t\"\260\003\n\020ColumnSt" + - "atistics\022\026\n\016numberOfValues\030\001 \001(\004\022J\n\rintS", - "tatistics\030\002 \001(\01323.org.apache.hadoop.hive" + - ".ql.io.orc.IntegerStatistics\022L\n\020doubleSt" + - "atistics\030\003 \001(\01322.org.apache.hadoop.hive." + - "ql.io.orc.DoubleStatistics\022L\n\020stringStat" + - "istics\030\004 \001(\01322.org.apache.hadoop.hive.ql" + - ".io.orc.StringStatistics\022L\n\020bucketStatis" + - "tics\030\005 \001(\01322.org.apache.hadoop.hive.ql.i" + - "o.orc.BucketStatistics\022N\n\021decimalStatist" + - "ics\030\006 \001(\01323.org.apache.hadoop.hive.ql.io" + - ".orc.DecimalStatistics\"n\n\rRowIndexEntry\022", - "\025\n\tpositions\030\001 \003(\004B\002\020\001\022F\n\nstatistics\030\002 \001" + - "(\01322.org.apache.hadoop.hive.ql.io.orc.Co" + - "lumnStatistics\"J\n\010RowIndex\022>\n\005entry\030\001 \003(" + - "\0132/.org.apache.hadoop.hive.ql.io.orc.Row" + - "IndexEntry\"\331\001\n\006Stream\022;\n\004kind\030\001 \002(\0162-.or" + - "g.apache.hadoop.hive.ql.io.orc.Stream.Ki" + - "nd\022\016\n\006column\030\002 \001(\r\022\016\n\006length\030\003 \001(\004\"r\n\004Ki" + - "nd\022\013\n\007PRESENT\020\000\022\010\n\004DATA\020\001\022\n\n\006LENGTH\020\002\022\023\n" + - "\017DICTIONARY_DATA\020\003\022\024\n\020DICTIONARY_COUNT\020\004" + - "\022\r\n\tSECONDARY\020\005\022\r\n\tROW_INDEX\020\006\"\221\001\n\016Colum", - "nEncoding\022C\n\004kind\030\001 \002(\01625.org.apache.had" + - "oop.hive.ql.io.orc.ColumnEncoding.Kind\022\026" + - "\n\016dictionarySize\030\002 \001(\r\"\"\n\004Kind\022\n\n\006DIRECT" + - "\020\000\022\016\n\nDICTIONARY\020\001\"\214\001\n\014StripeFooter\0229\n\007s" + - "treams\030\001 \003(\0132(.org.apache.hadoop.hive.ql" + - ".io.orc.Stream\022A\n\007columns\030\002 \003(\01320.org.ap" + - "ache.hadoop.hive.ql.io.orc.ColumnEncodin" + - "g\"\236\002\n\004Type\0229\n\004kind\030\001 \002(\0162+.org.apache.ha" + - "doop.hive.ql.io.orc.Type.Kind\022\024\n\010subtype" + - "s\030\002 \003(\rB\002\020\001\022\022\n\nfieldNames\030\003 \003(\t\"\260\001\n\004Kind", - "\022\013\n\007BOOLEAN\020\000\022\010\n\004BYTE\020\001\022\t\n\005SHORT\020\002\022\007\n\003IN" + - "T\020\003\022\010\n\004LONG\020\004\022\t\n\005FLOAT\020\005\022\n\n\006DOUBLE\020\006\022\n\n\006" + - "STRING\020\007\022\n\n\006BINARY\020\010\022\r\n\tTIMESTAMP\020\t\022\010\n\004L" + - "IST\020\n\022\007\n\003MAP\020\013\022\n\n\006STRUCT\020\014\022\t\n\005UNION\020\r\022\013\n" + - "\007DECIMAL\020\016\"x\n\021StripeInformation\022\016\n\006offse" + - "t\030\001 \001(\004\022\023\n\013indexLength\030\002 \001(\004\022\022\n\ndataLeng" + - "th\030\003 \001(\004\022\024\n\014footerLength\030\004 \001(\004\022\024\n\014number" + - "OfRows\030\005 \001(\004\"/\n\020UserMetadataItem\022\014\n\004name" + - "\030\001 \002(\t\022\r\n\005value\030\002 \002(\014\"\356\002\n\006Footer\022\024\n\014head" + - "erLength\030\001 \001(\004\022\025\n\rcontentLength\030\002 \001(\004\022D\n", - "\007stripes\030\003 \003(\01323.org.apache.hadoop.hive." + - "ql.io.orc.StripeInformation\0225\n\005types\030\004 \003" + - "(\0132&.org.apache.hadoop.hive.ql.io.orc.Ty" + - "pe\022D\n\010metadata\030\005 \003(\01322.org.apache.hadoop" + - ".hive.ql.io.orc.UserMetadataItem\022\024\n\014numb" + - "erOfRows\030\006 \001(\004\022F\n\nstatistics\030\007 \003(\01322.org" + - ".apache.hadoop.hive.ql.io.orc.ColumnStat" + - "istics\022\026\n\016rowIndexStride\030\010 \001(\r\"\255\001\n\nPostS" + - "cript\022\024\n\014footerLength\030\001 \001(\004\022F\n\013compressi" + - "on\030\002 \001(\01621.org.apache.hadoop.hive.ql.io.", - "orc.CompressionKind\022\034\n\024compressionBlockS" + - "ize\030\003 \001(\004\022\023\n\007version\030\004 \003(\rB\002\020\001\022\016\n\005magic\030" + - "\300> \001(\t*:\n\017CompressionKind\022\010\n\004NONE\020\000\022\010\n\004Z" + - "LIB\020\001\022\n\n\006SNAPPY\020\002\022\007\n\003LZO\020\003" + "\007maximum\030\002 \001(\t\022\013\n\003sum\030\003 \001(\t\"2\n\016DateStati" + + "stics\022\017\n\007minimum\030\001 \001(\021\022\017\n\007maximum\030\002 \001(\021\"", + "\372\003\n\020ColumnStatistics\022\026\n\016numberOfValues\030\001" + + " \001(\004\022J\n\rintStatistics\030\002 \001(\01323.org.apache" + + ".hadoop.hive.ql.io.orc.IntegerStatistics" + + "\022L\n\020doubleStatistics\030\003 \001(\01322.org.apache." + + "hadoop.hive.ql.io.orc.DoubleStatistics\022L" + + "\n\020stringStatistics\030\004 \001(\01322.org.apache.ha" + + "doop.hive.ql.io.orc.StringStatistics\022L\n\020" + + "bucketStatistics\030\005 \001(\01322.org.apache.hado" + + "op.hive.ql.io.orc.BucketStatistics\022N\n\021de" + + "cimalStatistics\030\006 \001(\01323.org.apache.hadoo", + "p.hive.ql.io.orc.DecimalStatistics\022H\n\016da" + + "teStatistics\030\007 \001(\01320.org.apache.hadoop.h" + + "ive.ql.io.orc.DateStatistics\"n\n\rRowIndex" + + "Entry\022\025\n\tpositions\030\001 \003(\004B\002\020\001\022F\n\nstatisti" + + "cs\030\002 \001(\01322.org.apache.hadoop.hive.ql.io." + + "orc.ColumnStatistics\"J\n\010RowIndex\022>\n\005entr" + + "y\030\001 \003(\0132/.org.apache.hadoop.hive.ql.io.o" + + "rc.RowIndexEntry\"\331\001\n\006Stream\022;\n\004kind\030\001 \002(" + + "\0162-.org.apache.hadoop.hive.ql.io.orc.Str" + + "eam.Kind\022\016\n\006column\030\002 \001(\r\022\016\n\006length\030\003 \001(\004", + "\"r\n\004Kind\022\013\n\007PRESENT\020\000\022\010\n\004DATA\020\001\022\n\n\006LENGT" + + "H\020\002\022\023\n\017DICTIONARY_DATA\020\003\022\024\n\020DICTIONARY_C" + + "OUNT\020\004\022\r\n\tSECONDARY\020\005\022\r\n\tROW_INDEX\020\006\"\221\001\n" + + "\016ColumnEncoding\022C\n\004kind\030\001 \002(\01625.org.apac" + + "he.hadoop.hive.ql.io.orc.ColumnEncoding." + + "Kind\022\026\n\016dictionarySize\030\002 \001(\r\"\"\n\004Kind\022\n\n\006" + + "DIRECT\020\000\022\016\n\nDICTIONARY\020\001\"\214\001\n\014StripeFoote" + + "r\0229\n\007streams\030\001 \003(\0132(.org.apache.hadoop.h" + + "ive.ql.io.orc.Stream\022A\n\007columns\030\002 \003(\01320." + + "org.apache.hadoop.hive.ql.io.orc.ColumnE", + "ncoding\"\250\002\n\004Type\0229\n\004kind\030\001 \002(\0162+.org.apa" + + "che.hadoop.hive.ql.io.orc.Type.Kind\022\024\n\010s" + + "ubtypes\030\002 \003(\rB\002\020\001\022\022\n\nfieldNames\030\003 \003(\t\"\272\001" + + "\n\004Kind\022\013\n\007BOOLEAN\020\000\022\010\n\004BYTE\020\001\022\t\n\005SHORT\020\002" + + "\022\007\n\003INT\020\003\022\010\n\004LONG\020\004\022\t\n\005FLOAT\020\005\022\n\n\006DOUBLE" + + "\020\006\022\n\n\006STRING\020\007\022\n\n\006BINARY\020\010\022\r\n\tTIMESTAMP\020" + + "\t\022\010\n\004LIST\020\n\022\007\n\003MAP\020\013\022\n\n\006STRUCT\020\014\022\t\n\005UNIO" + + "N\020\r\022\013\n\007DECIMAL\020\016\022\010\n\004DATE\020\017\"x\n\021StripeInfo" + + "rmation\022\016\n\006offset\030\001 \001(\004\022\023\n\013indexLength\030\002" + + " \001(\004\022\022\n\ndataLength\030\003 \001(\004\022\024\n\014footerLength", + "\030\004 \001(\004\022\024\n\014numberOfRows\030\005 \001(\004\"/\n\020UserMeta" + + "dataItem\022\014\n\004name\030\001 \002(\t\022\r\n\005value\030\002 \002(\014\"\356\002" + + "\n\006Footer\022\024\n\014headerLength\030\001 \001(\004\022\025\n\rconten" + + "tLength\030\002 \001(\004\022D\n\007stripes\030\003 \003(\01323.org.apa" + + "che.hadoop.hive.ql.io.orc.StripeInformat" + + "ion\0225\n\005types\030\004 \003(\0132&.org.apache.hadoop.h" + + "ive.ql.io.orc.Type\022D\n\010metadata\030\005 \003(\01322.o" + + "rg.apache.hadoop.hive.ql.io.orc.UserMeta" + + "dataItem\022\024\n\014numberOfRows\030\006 \001(\004\022F\n\nstatis" + + "tics\030\007 \003(\01322.org.apache.hadoop.hive.ql.i", + "o.orc.ColumnStatistics\022\026\n\016rowIndexStride" + + "\030\010 \001(\r\"\255\001\n\nPostScript\022\024\n\014footerLength\030\001 " + + "\001(\004\022F\n\013compression\030\002 \001(\01621.org.apache.ha" + + "doop.hive.ql.io.orc.CompressionKind\022\034\n\024c" + + "ompressionBlockSize\030\003 \001(\004\022\023\n\007version\030\004 \003" + + "(\rB\002\020\001\022\016\n\005magic\030\300> \001(\t*:\n\017CompressionKin" + + "d\022\010\n\004NONE\020\000\022\010\n\004ZLIB\020\001\022\n\n\006SNAPPY\020\002\022\007\n\003LZO" + + "\020\003" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -10649,16 +11199,24 @@ void setMagic(com.google.protobuf.ByteString value) { new java.lang.String[] { "Minimum", "Maximum", "Sum", }, org.apache.hadoop.hive.ql.io.orc.OrcProto.DecimalStatistics.class, org.apache.hadoop.hive.ql.io.orc.OrcProto.DecimalStatistics.Builder.class); - internal_static_org_apache_hadoop_hive_ql_io_orc_ColumnStatistics_descriptor = + internal_static_org_apache_hadoop_hive_ql_io_orc_DateStatistics_descriptor = getDescriptor().getMessageTypes().get(5); + internal_static_org_apache_hadoop_hive_ql_io_orc_DateStatistics_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hive_ql_io_orc_DateStatistics_descriptor, + new java.lang.String[] { "Minimum", "Maximum", }, + org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics.class, + org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics.Builder.class); + internal_static_org_apache_hadoop_hive_ql_io_orc_ColumnStatistics_descriptor = + getDescriptor().getMessageTypes().get(6); internal_static_org_apache_hadoop_hive_ql_io_orc_ColumnStatistics_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_org_apache_hadoop_hive_ql_io_orc_ColumnStatistics_descriptor, - new java.lang.String[] { "NumberOfValues", "IntStatistics", "DoubleStatistics", "StringStatistics", "BucketStatistics", "DecimalStatistics", }, + new java.lang.String[] { "NumberOfValues", "IntStatistics", "DoubleStatistics", "StringStatistics", "BucketStatistics", "DecimalStatistics", "DateStatistics", }, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.class, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder.class); internal_static_org_apache_hadoop_hive_ql_io_orc_RowIndexEntry_descriptor = - getDescriptor().getMessageTypes().get(6); + getDescriptor().getMessageTypes().get(7); internal_static_org_apache_hadoop_hive_ql_io_orc_RowIndexEntry_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_org_apache_hadoop_hive_ql_io_orc_RowIndexEntry_descriptor, @@ -10666,7 +11224,7 @@ void setMagic(com.google.protobuf.ByteString value) { org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry.class, org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry.Builder.class); internal_static_org_apache_hadoop_hive_ql_io_orc_RowIndex_descriptor = - getDescriptor().getMessageTypes().get(7); + getDescriptor().getMessageTypes().get(8); internal_static_org_apache_hadoop_hive_ql_io_orc_RowIndex_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_org_apache_hadoop_hive_ql_io_orc_RowIndex_descriptor, @@ -10674,7 +11232,7 @@ void setMagic(com.google.protobuf.ByteString value) { org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex.class, org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex.Builder.class); internal_static_org_apache_hadoop_hive_ql_io_orc_Stream_descriptor = - getDescriptor().getMessageTypes().get(8); + getDescriptor().getMessageTypes().get(9); internal_static_org_apache_hadoop_hive_ql_io_orc_Stream_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_org_apache_hadoop_hive_ql_io_orc_Stream_descriptor, @@ -10682,7 +11240,7 @@ void setMagic(com.google.protobuf.ByteString value) { org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.class, org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.Builder.class); internal_static_org_apache_hadoop_hive_ql_io_orc_ColumnEncoding_descriptor = - getDescriptor().getMessageTypes().get(9); + getDescriptor().getMessageTypes().get(10); internal_static_org_apache_hadoop_hive_ql_io_orc_ColumnEncoding_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_org_apache_hadoop_hive_ql_io_orc_ColumnEncoding_descriptor, @@ -10690,7 +11248,7 @@ void setMagic(com.google.protobuf.ByteString value) { org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnEncoding.class, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnEncoding.Builder.class); internal_static_org_apache_hadoop_hive_ql_io_orc_StripeFooter_descriptor = - getDescriptor().getMessageTypes().get(10); + getDescriptor().getMessageTypes().get(11); internal_static_org_apache_hadoop_hive_ql_io_orc_StripeFooter_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_org_apache_hadoop_hive_ql_io_orc_StripeFooter_descriptor, @@ -10698,7 +11256,7 @@ void setMagic(com.google.protobuf.ByteString value) { org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter.class, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter.Builder.class); internal_static_org_apache_hadoop_hive_ql_io_orc_Type_descriptor = - getDescriptor().getMessageTypes().get(11); + getDescriptor().getMessageTypes().get(12); internal_static_org_apache_hadoop_hive_ql_io_orc_Type_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_org_apache_hadoop_hive_ql_io_orc_Type_descriptor, @@ -10706,7 +11264,7 @@ void setMagic(com.google.protobuf.ByteString value) { org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.class, org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.Builder.class); internal_static_org_apache_hadoop_hive_ql_io_orc_StripeInformation_descriptor = - getDescriptor().getMessageTypes().get(12); + getDescriptor().getMessageTypes().get(13); internal_static_org_apache_hadoop_hive_ql_io_orc_StripeInformation_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_org_apache_hadoop_hive_ql_io_orc_StripeInformation_descriptor, @@ -10714,7 +11272,7 @@ void setMagic(com.google.protobuf.ByteString value) { org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation.class, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation.Builder.class); internal_static_org_apache_hadoop_hive_ql_io_orc_UserMetadataItem_descriptor = - getDescriptor().getMessageTypes().get(13); + getDescriptor().getMessageTypes().get(14); internal_static_org_apache_hadoop_hive_ql_io_orc_UserMetadataItem_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_org_apache_hadoop_hive_ql_io_orc_UserMetadataItem_descriptor, @@ -10722,7 +11280,7 @@ void setMagic(com.google.protobuf.ByteString value) { org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem.class, org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem.Builder.class); internal_static_org_apache_hadoop_hive_ql_io_orc_Footer_descriptor = - getDescriptor().getMessageTypes().get(14); + getDescriptor().getMessageTypes().get(15); internal_static_org_apache_hadoop_hive_ql_io_orc_Footer_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_org_apache_hadoop_hive_ql_io_orc_Footer_descriptor, @@ -10730,7 +11288,7 @@ void setMagic(com.google.protobuf.ByteString value) { org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer.class, org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer.Builder.class); internal_static_org_apache_hadoop_hive_ql_io_orc_PostScript_descriptor = - getDescriptor().getMessageTypes().get(15); + getDescriptor().getMessageTypes().get(16); internal_static_org_apache_hadoop_hive_ql_io_orc_PostScript_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_org_apache_hadoop_hive_ql_io_orc_PostScript_descriptor, diff --git ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 8330f65..12e9334 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -172,8 +172,8 @@ DYNAMIC_PARTITION_STRICT_MODE(10096, "Dynamic partition strict mode requires at least one " + "static partition column. To turn this off set hive.exec.dynamic.partition.mode=nonstrict"), NONEXISTPARTCOL(10098, "Non-Partition column appears in the partition specification: "), - UNSUPPORTED_TYPE(10099, "DATE and DATETIME types aren't supported yet. Please use " - + "TIMESTAMP instead"), + UNSUPPORTED_TYPE(10099, "DATETIME type isn't supported yet. Please use " + + "DATE or TIMESTAMP instead"), CREATE_NON_NATIVE_AS(10100, "CREATE TABLE AS SELECT cannot be used for a non-native table"), LOAD_INTO_NON_NATIVE(10101, "A non-native table cannot be used as target for LOAD"), LOCKMGR_NOT_SPECIFIED(10102, "Lock manager not specified correctly, set hive.lock.manager"), diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java index b4b2c90..8e12f28 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java @@ -343,6 +343,8 @@ registerUDF(serdeConstants.STRING_TYPE_NAME, UDFToString.class, false, UDFToString.class.getSimpleName()); + registerGenericUDF(serdeConstants.DATE_TYPE_NAME, + GenericUDFToDate.class); registerGenericUDF(serdeConstants.TIMESTAMP_TYPE_NAME, GenericUDFTimestamp.class); registerGenericUDF(serdeConstants.BINARY_TYPE_NAME, @@ -707,6 +709,11 @@ public static boolean implicitConvertable(TypeInfo from, TypeInfo to) { if (from.equals(TypeInfoFactory.voidTypeInfo)) { return true; } + // Allow implicit String to Date conversion + if (from.equals(TypeInfoFactory.dateTypeInfo) + && to.equals(TypeInfoFactory.stringTypeInfo)) { + return true; + } if (from.equals(TypeInfoFactory.timestampTypeInfo) && to.equals(TypeInfoFactory.stringTypeInfo)) { @@ -1267,7 +1274,8 @@ private static boolean isOpCast(ExprNodeDesc desc) { udfClass == UDFToDouble.class || udfClass == UDFToFloat.class || udfClass == UDFToInteger.class || udfClass == UDFToLong.class || udfClass == UDFToShort.class || udfClass == UDFToString.class || - udfClass == GenericUDFTimestamp.class || udfClass == GenericUDFToBinary.class; + udfClass == GenericUDFTimestamp.class || udfClass == GenericUDFToBinary.class || + udfClass == GenericUDFToDate.class; } /** diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java index f91b175..7e976f5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.io.orc; import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; @@ -543,6 +544,97 @@ public String toString() { } } + private static final class DateStatisticsImpl extends ColumnStatisticsImpl + implements DateColumnStatistics { + private DateWritable minimum = null; + private DateWritable maximum = null; + + DateStatisticsImpl() { + } + + DateStatisticsImpl(OrcProto.ColumnStatistics stats) { + super(stats); + OrcProto.DateStatistics dateStats = stats.getDateStatistics(); + // min,max values serialized/deserialized as int (days since epoch) + if (dateStats.hasMaximum()) { + maximum = new DateWritable(dateStats.getMaximum()); + } + if (dateStats.hasMinimum()) { + minimum = new DateWritable(dateStats.getMinimum()); + } + } + + @Override + void reset() { + super.reset(); + minimum = null; + maximum = null; + } + + @Override + void updateDate(DateWritable value) { + if (minimum == null) { + minimum = value; + maximum = value; + } else if (minimum.compareTo(value) > 0) { + minimum = value; + } else if (maximum.compareTo(value) < 0) { + maximum = value; + } + } + + @Override + void merge(ColumnStatisticsImpl other) { + super.merge(other); + DateStatisticsImpl dateStats = (DateStatisticsImpl) other; + if (minimum == null) { + minimum = dateStats.minimum; + maximum = dateStats.maximum; + } else if (dateStats.minimum != null) { + if (minimum.compareTo(dateStats.minimum) > 0) { + minimum = dateStats.minimum; + } else if (maximum.compareTo(dateStats.maximum) < 0) { + maximum = dateStats.maximum; + } + } + } + + @Override + OrcProto.ColumnStatistics.Builder serialize() { + OrcProto.ColumnStatistics.Builder result = super.serialize(); + OrcProto.DateStatistics.Builder dateStats = + OrcProto.DateStatistics.newBuilder(); + if (getNumberOfValues() != 0) { + dateStats.setMinimum(minimum.getDays()); + dateStats.setMaximum(maximum.getDays()); + } + result.setDateStatistics(dateStats); + return result; + } + + @Override + public DateWritable getMinimum() { + return minimum; + } + + @Override + public DateWritable getMaximum() { + return maximum; + } + + @Override + public String toString() { + StringBuilder buf = new StringBuilder(super.toString()); + if (getNumberOfValues() != 0) { + buf.append(" min: "); + buf.append(minimum); + buf.append(" max: "); + buf.append(maximum); + } + return buf.toString(); + } + } + private long count = 0; ColumnStatisticsImpl(OrcProto.ColumnStatistics stats) { @@ -578,6 +670,10 @@ void updateDecimal(HiveDecimal value) { throw new UnsupportedOperationException("Can't update decimal"); } + void updateDate(DateWritable value) { + throw new UnsupportedOperationException("Can't update date"); + } + void merge(ColumnStatisticsImpl stats) { count += stats.count; } @@ -621,6 +717,8 @@ static ColumnStatisticsImpl create(ObjectInspector inspector) { return new StringStatisticsImpl(); case DECIMAL: return new DecimalStatisticsImpl(); + case DATE: + return new DateStatisticsImpl(); default: return new ColumnStatisticsImpl(); } @@ -640,6 +738,8 @@ static ColumnStatisticsImpl deserialize(OrcProto.ColumnStatistics stats) { return new StringStatisticsImpl(stats); } else if (stats.hasDecimalStatistics()) { return new DecimalStatisticsImpl(stats); + } else if (stats.hasDateStatistics()) { + return new DateStatisticsImpl(stats); } else { return new ColumnStatisticsImpl(stats); } diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/DateColumnStatistics.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/DateColumnStatistics.java new file mode 100644 index 0000000..03cdeef --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/DateColumnStatistics.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +import org.apache.hadoop.hive.serde2.io.DateWritable; + +/** + * Statistics for DATE columns. + */ +public interface DateColumnStatistics extends ColumnStatistics { + /** + * Get the minimum value for the column. + * @return minimum value + */ + DateWritable getMinimum(); + + /** + * Get the maximum value for the column. + * @return maximum value + */ + DateWritable getMaximum(); +} diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java index 3b9397c..2b1f0d8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java @@ -475,6 +475,8 @@ static ObjectInspector createObjectInspector(TypeInfo info) { return PrimitiveObjectInspectorFactory.writableStringObjectInspector; case TIMESTAMP: return PrimitiveObjectInspectorFactory.javaTimestampObjectInspector; + case DATE: + return PrimitiveObjectInspectorFactory.javaDateObjectInspector; case DECIMAL: return PrimitiveObjectInspectorFactory.javaHiveDecimalObjectInspector; default: @@ -519,6 +521,8 @@ static ObjectInspector createObjectInspector(int columnId, return PrimitiveObjectInspectorFactory.writableStringObjectInspector; case TIMESTAMP: return PrimitiveObjectInspectorFactory.javaTimestampObjectInspector; + case DATE: + return PrimitiveObjectInspectorFactory.javaDateObjectInspector; case DECIMAL: return PrimitiveObjectInspectorFactory.javaHiveDecimalObjectInspector; case STRUCT: diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java index 06103e3..92413e1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java @@ -20,6 +20,7 @@ import java.io.EOFException; import java.io.IOException; import java.nio.ByteBuffer; +import java.sql.Date; import java.sql.Timestamp; import java.util.ArrayList; import java.util.HashMap; @@ -31,6 +32,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.serde2.io.ByteWritable; +import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.io.BooleanWritable; @@ -621,6 +623,50 @@ void skipRows(long items) throws IOException { } } + private static class DateTreeReader extends TreeReader{ + private RunLengthIntegerReader reader = null; + + DateTreeReader(Path path, int columnId) { + super(path, columnId); + } + + @Override + void startStripe(Map streams, + List encodings + ) throws IOException { + super.startStripe(streams, encodings); + StreamName name = new StreamName(columnId, + OrcProto.Stream.Kind.DATA); + reader = new RunLengthIntegerReader(streams.get(name), true); + } + + @Override + void seek(PositionProvider[] index) throws IOException { + super.seek(index); + reader.seek(index[columnId]); + } + + @Override + Object next(Object previous) throws IOException { + super.next(previous); + Date result = null; + if (valuePresent) { + if (previous == null) { + result = new Date(0); + } else { + result = (Date) previous; + } + result.setTime(DateWritable.daysToMillis((int) reader.next())); + } + return result; + } + + @Override + void skipRows(long items) throws IOException { + reader.skip(countNonNulls(items)); + } + } + private static class DecimalTreeReader extends TreeReader{ private InStream valueStream; private RunLengthIntegerReader scaleStream; @@ -1099,6 +1145,8 @@ private static TreeReader createTreeReader(Path path, return new BinaryTreeReader(path, columnId); case TIMESTAMP: return new TimestampTreeReader(path, columnId); + case DATE: + return new DateTreeReader(path, columnId); case DECIMAL: return new DecimalTreeReader(path, columnId); case STRUCT: diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java index 52defb9..9f314d9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java @@ -34,6 +34,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry; +import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; @@ -41,12 +42,13 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector; @@ -988,6 +990,46 @@ void recordPosition(PositionRecorder recorder) throws IOException { } } + private static class DateTreeWriter extends TreeWriter { + private final RunLengthIntegerWriter writer; + + DateTreeWriter(int columnId, + ObjectInspector inspector, + StreamFactory writer, + boolean nullable) throws IOException { + super(columnId, inspector, writer, nullable); + PositionedOutputStream out = writer.createStream(id, + OrcProto.Stream.Kind.DATA); + this.writer = new RunLengthIntegerWriter(out, true); + recordPosition(rowIndexPosition); + } + + @Override + void write(Object obj) throws IOException { + super.write(obj); + if (obj != null) { + // Using the Writable here as it's used directly for writing as well as for stats. + DateWritable val = ((DateObjectInspector) inspector).getPrimitiveWritableObject(obj); + indexStatistics.updateDate(val); + writer.write(val.getDays()); + } + } + + @Override + void writeStripe(OrcProto.StripeFooter.Builder builder, + int requiredIndexEntries) throws IOException { + super.writeStripe(builder, requiredIndexEntries); + writer.flush(); + recordPosition(rowIndexPosition); + } + + @Override + void recordPosition(PositionRecorder recorder) throws IOException { + super.recordPosition(recorder); + writer.getPosition(recorder); + } + } + private static class DecimalTreeWriter extends TreeWriter { private final PositionedOutputStream valueStream; private final RunLengthIntegerWriter scaleStream; @@ -1262,6 +1304,9 @@ private static TreeWriter createTreeWriter(ObjectInspector inspector, case TIMESTAMP: return new TimestampTreeWriter(streamFactory.getNextColumnId(), inspector, streamFactory, nullable); + case DATE: + return new DateTreeWriter(streamFactory.getNextColumnId(), + inspector, streamFactory, nullable); case DECIMAL: return new DecimalTreeWriter(streamFactory.getNextColumnId(), inspector, streamFactory, nullable); @@ -1324,6 +1369,9 @@ private static void writeTypes(OrcProto.Footer.Builder builder, case TIMESTAMP: type.setKind(OrcProto.Type.Kind.TIMESTAMP); break; + case DATE: + type.setKind(OrcProto.Type.Kind.DATE); + break; case DECIMAL: type.setKind(OrcProto.Type.Kind.DECIMAL); break; diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index b2da758..932ec66 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -156,8 +156,8 @@ } public static String getTypeName(int token) throws SemanticException { - // date and datetime types aren't currently supported - if (token == HiveParser.TOK_DATE || token == HiveParser.TOK_DATETIME) { + // datetime type isn't currently supported + if (token == HiveParser.TOK_DATETIME) { throw new SemanticException(ErrorMsg.UNSUPPORTED_TYPE.getMsg()); } return TokenToTypeName.get(token); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 2d7a594..36d62a6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -106,6 +106,7 @@ TOK_BOOLEAN; TOK_FLOAT; TOK_DOUBLE; TOK_DATE; +TOK_DATELITERAL; TOK_DATETIME; TOK_TIMESTAMP; TOK_STRING; diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g index b62dc88..c898858 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g @@ -222,6 +222,7 @@ constant @after { gParent.msgs.pop(); } : Number + | dateLiteral | StringLiteral | stringLiteralSequence | BigintLiteral @@ -244,6 +245,11 @@ charSetStringLiteral csName=CharSetName csLiteral=CharSetLiteral -> ^(TOK_CHARSETLITERAL $csName $csLiteral) ; +dateLiteral + : + KW_DATE StringLiteral -> ^(TOK_DATELITERAL StringLiteral) + ; + expression @init { gParent.msgs.push("expression specification"); } @after { gParent.msgs.pop(); } @@ -254,6 +260,7 @@ expression atomExpression : KW_NULL -> TOK_NULL + | dateLiteral | constant | function | castExpression diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java index 9968635..59572ac 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.parse; +import java.sql.Date; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -152,7 +153,8 @@ public static ExprNodeDesc processGByExpr(Node nd, Object procCtx) getStrExprProcessor()); opRules.put(new RuleRegExp("R4", HiveParser.KW_TRUE + "%|" + HiveParser.KW_FALSE + "%"), getBoolExprProcessor()); - opRules.put(new RuleRegExp("R5", HiveParser.TOK_TABLE_OR_COL + "%"), + opRules.put(new RuleRegExp("R5", HiveParser.TOK_DATELITERAL + "%"), getDateExprProcessor()); + opRules.put(new RuleRegExp("R6", HiveParser.TOK_TABLE_OR_COL + "%"), getColumnExprProcessor()); // The dispatcher fires the processor corresponding to the closest matching @@ -388,6 +390,51 @@ public static BoolExprProcessor getBoolExprProcessor() { } /** + * Processor for date constants. + */ + public static class DateExprProcessor implements NodeProcessor { + + @Override + public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, + Object... nodeOutputs) throws SemanticException { + + TypeCheckCtx ctx = (TypeCheckCtx) procCtx; + if (ctx.getError() != null) { + return null; + } + + ExprNodeDesc desc = TypeCheckProcFactory.processGByExpr(nd, procCtx); + if (desc != null) { + return desc; + } + + ASTNode expr = (ASTNode) nd; + + // Get the string value and convert to a Date value. + if (expr.getChildCount() == 1) { + try { + String dateString = BaseSemanticAnalyzer.stripQuotes(expr.getChild(0).getText()); + Date date = Date.valueOf(dateString); + return new ExprNodeConstantDesc(TypeInfoFactory.dateTypeInfo, date); + } catch (IllegalArgumentException err) { + throw new SemanticException("Unable to convert date literal string to date value.", err); + } + } else { + throw new SemanticException("Expected date string after DATE keyword"); + } + } + } + + /** + * Factory method to get DateExprProcessor. + * + * @return DateExprProcessor. + */ + public static DateExprProcessor getDateExprProcessor() { + return new DateExprProcessor(); + } + + /** * Processor for table columns. */ public static class ColumnExprProcessor implements NodeProcessor { @@ -519,6 +566,8 @@ public static ColumnExprProcessor getColumnExprProcessor() { serdeConstants.STRING_TYPE_NAME); conversionFunctionTextHashMap.put(HiveParser.TOK_BINARY, serdeConstants.BINARY_TYPE_NAME); + conversionFunctionTextHashMap.put(HiveParser.TOK_DATE, + serdeConstants.DATE_TYPE_NAME); conversionFunctionTextHashMap.put(HiveParser.TOK_TIMESTAMP, serdeConstants.TIMESTAMP_TYPE_NAME); conversionFunctionTextHashMap.put(HiveParser.TOK_DECIMAL, diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDate.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDate.java index 2386496..3df453c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDate.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDate.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.io.Text; @@ -67,6 +68,15 @@ public Text evaluate(Text dateString) { } } + public Text evaluate(DateWritable d) { + if (d == null) { + return null; + } + + t.set(formatter.format(d.get())); + return t; + } + public Text evaluate(TimestampWritable i) { if (i == null) { return null; diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateAdd.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateAdd.java index da826a2..b1b0bf2 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateAdd.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateAdd.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; @@ -82,6 +83,17 @@ public Text evaluate(Text dateString1, IntWritable days) { } } + public Text evaluate(DateWritable d, IntWritable days) { + if (d == null || days == null) { + return null; + } + calendar.setTime(d.get()); + calendar.add(Calendar.DAY_OF_MONTH, days.get()); + Date newDate = calendar.getTime(); + result.set(formatter.format(newDate)); + return result; + } + public Text evaluate(TimestampWritable t, IntWritable days) { if (t == null || days == null) { return null; diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateDiff.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateDiff.java index b39e374..da14c4f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateDiff.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateDiff.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; @@ -79,6 +80,26 @@ public IntWritable evaluate(Text dateString, TimestampWritable t) { return evaluate(toDate(dateString), toDate(t)); } + public IntWritable evaluate(Text dateString, DateWritable d) { + return evaluate(toDate(dateString), d.get()); + } + + public IntWritable evaluate(TimestampWritable t, DateWritable d) { + return evaluate(toDate(t), d.get()); + } + + public IntWritable evaluate(DateWritable d1, DateWritable d2) { + return evaluate(d1.get(), d2.get()); + } + + public IntWritable evaluate(DateWritable d, Text dateString) { + return evaluate(d.get(), toDate(dateString)); + } + + public IntWritable evaluate(DateWritable d, TimestampWritable t) { + return evaluate(d.get(), toDate(t)); + } + private IntWritable evaluate(Date date, Date date2) { if (date == null || date2 == null) { return null; diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateSub.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateSub.java index 28784b6..c8a1d1f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateSub.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateSub.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; @@ -82,6 +83,17 @@ public Text evaluate(Text dateString1, IntWritable days) { } } + public Text evaluate(DateWritable d, IntWritable days) { + if (d == null || days == null) { + return null; + } + calendar.setTime(d.get()); + calendar.add(Calendar.DAY_OF_MONTH, -days.get()); + Date newDate = calendar.getTime(); + result.set(formatter.format(newDate)); + return result; + } + public Text evaluate(TimestampWritable t, IntWritable days) { if (t == null || days == null) { return null; diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDayOfMonth.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDayOfMonth.java index fc193eb..c78ddf2 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDayOfMonth.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDayOfMonth.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; @@ -73,6 +74,16 @@ public IntWritable evaluate(Text dateString) { } } + public IntWritable evaluate(DateWritable d) { + if (d == null) { + return null; + } + + calendar.setTime(d.get()); + result.set(calendar.get(Calendar.DAY_OF_MONTH)); + return result; + } + public IntWritable evaluate(TimestampWritable t) { if (t == null) { return null; diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMonth.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMonth.java index a82ec30..71d9513 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMonth.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMonth.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; @@ -69,6 +70,16 @@ public IntWritable evaluate(Text dateString) { } } + public IntWritable evaluate(DateWritable d) { + if (d == null) { + return null; + } + + calendar.setTime(d.get()); + result.set(1 + calendar.get(Calendar.MONTH)); + return result; + } + public IntWritable evaluate(TimestampWritable t) { if (t == null) { return null; diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java index e96791c..b0921cd 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java @@ -21,9 +21,10 @@ import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.ql.exec.UDF; -import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; +import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.io.BooleanWritable; @@ -166,6 +167,11 @@ public BooleanWritable evaluate(Text i) { } } + public BooleanWritable evaluate(DateWritable d) { + // date value to boolean doesn't make any sense. + return null; + } + public BooleanWritable evaluate(TimestampWritable i) { if (i == null) { return null; diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java index d0bb667..11408b3 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java @@ -19,9 +19,9 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; -import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.lazy.LazyByte; diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java index 8a9e662..bc1be48 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java @@ -19,9 +19,9 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; -import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.io.BooleanWritable; diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java index aea11fc..9d3ff0b 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java @@ -19,9 +19,9 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; -import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.io.BooleanWritable; diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java index c34e6b6..76794b6 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java @@ -19,9 +19,9 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; -import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.lazy.LazyInteger; diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java index b5925fe..87c2288 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java @@ -19,9 +19,9 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; -import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.lazy.LazyLong; diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java index 1f63ce9..5a96723 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java @@ -19,9 +19,9 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; -import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.lazy.LazyShort; diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java index d6c7a30..17a79f0 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; +import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.lazy.LazyInteger; import org.apache.hadoop.hive.serde2.lazy.LazyLong; @@ -133,6 +134,15 @@ public Text evaluate(Text i) { return i; } + public Text evaluate(DateWritable d) { + if (d == null) { + return null; + } else { + t.set(d.toString()); + return t; + } + } + public Text evaluate(TimestampWritable i) { if (i == null) { return null; diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFUnixTimeStamp.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFUnixTimeStamp.java index 031120a..aeda61e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFUnixTimeStamp.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFUnixTimeStamp.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; @@ -106,6 +107,15 @@ public LongWritable evaluate(Text dateText, Text patternText) { return evaluate(dateText); } + public LongWritable evaluate(DateWritable d) { + if (d == null) { + return null; + } else { + result.set(d.getTimeInSeconds()); + return result; + } + } + public LongWritable evaluate(TimestampWritable i) { if (i == null) { return null; diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFWeekOfYear.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFWeekOfYear.java index 0db41f1..7f087e0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFWeekOfYear.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFWeekOfYear.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; @@ -74,6 +75,16 @@ public IntWritable evaluate(Text dateString) { } } + public IntWritable evaluate(DateWritable d) { + if (d == null) { + return null; + } + + calendar.setTime(d.get()); + result.set(calendar.get(Calendar.WEEK_OF_YEAR)); + return result; + } + public IntWritable evaluate(TimestampWritable t) { if (t == null) { return null; diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFYear.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFYear.java index ca54d01..0150cbd 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFYear.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFYear.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; @@ -73,6 +74,16 @@ public IntWritable evaluate(Text dateString) { } } + public IntWritable evaluate(DateWritable d) { + if (d == null) { + return null; + } + + calendar.setTime(d.get()); + result.set(calendar.get(Calendar.YEAR)); + return result; + } + public IntWritable evaluate(TimestampWritable t) { if (t == null) { return null; diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFAverage.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFAverage.java index 07ad6d9..6ec279f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFAverage.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFAverage.java @@ -80,6 +80,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) case DECIMAL: return new GenericUDAFAverageEvaluatorDecimal(); case BOOLEAN: + case DATE: default: throw new UDFArgumentTypeException(0, "Only numeric or string type arguments are accepted but " diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFContextNGrams.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFContextNGrams.java index 7246f0d..0758e3f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFContextNGrams.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFContextNGrams.java @@ -19,10 +19,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.Iterator; -import java.util.Set; -import java.util.Map; -import java.util.Collections; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -30,22 +26,15 @@ import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.StandardMapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.StructField; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDoubleObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.io.Text; /** @@ -75,7 +64,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE throw new UDFArgumentTypeException(parameters.length-1, "Please specify either three or four arguments."); } - + // Validate the first parameter, which is the expression to compute over. This should be an // array of strings type, or an array of arrays of strings. PrimitiveTypeInfo pti; @@ -105,7 +94,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE } if(pti.getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.STRING) { throw new UDFArgumentTypeException(0, - "Only array or array> is allowed, but " + "Only array or array> is allowed, but " + parameters[0].getTypeName() + " was passed as parameter 1."); } @@ -115,7 +104,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE ObjectInspector.Category.PRIMITIVE) { throw new UDFArgumentTypeException(1, "Only arrays of strings are accepted but " + parameters[1].getTypeName() + " was passed as parameter 2."); - } + } if(((PrimitiveTypeInfo) ((ListTypeInfo)parameters[1]).getListElementTypeInfo()). getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.STRING) { throw new UDFArgumentTypeException(1, "Only arrays of strings are accepted but " @@ -126,7 +115,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE if(parameters[2].getCategory() != ObjectInspector.Category.PRIMITIVE) { throw new UDFArgumentTypeException(2, "Only integers are accepted but " + parameters[2].getTypeName() + " was passed as parameter 3."); - } + } switch(((PrimitiveTypeInfo) parameters[2]).getPrimitiveCategory()) { case BYTE: case SHORT: @@ -146,7 +135,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE if(parameters[3].getCategory() != ObjectInspector.Category.PRIMITIVE) { throw new UDFArgumentTypeException(3, "Only integers are accepted but " + parameters[3].getTypeName() + " was passed as parameter 4."); - } + } switch(((PrimitiveTypeInfo) parameters[3]).getPrimitiveCategory()) { case BYTE: case SHORT: @@ -177,7 +166,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE private PrimitiveObjectInspector kOI; private PrimitiveObjectInspector pOI; - // For PARTIAL2 and FINAL: ObjectInspectors for partial aggregations + // For PARTIAL2 and FINAL: ObjectInspectors for partial aggregations private StandardListObjectInspector loi; @Override @@ -213,8 +202,8 @@ public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveExc // Init output object inspectors. // // The return type for a partial aggregation is still a list of strings. - // - // The return type for FINAL and COMPLETE is a full aggregation result, which is + // + // The return type for FINAL and COMPLETE is a full aggregation result, which is // an array of structures containing the n-gram and its estimated frequency. if (m == Mode.PARTIAL1 || m == Mode.PARTIAL2) { return ObjectInspectorFactory.getStandardListObjectInspector( @@ -228,7 +217,7 @@ public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveExc foi.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector); ArrayList fname = new ArrayList(); fname.add("ngram"); - fname.add("estfrequency"); + fname.add("estfrequency"); return ObjectInspectorFactory.getStandardListObjectInspector( ObjectInspectorFactory.getStandardStructObjectInspector(fname, foi) ); } @@ -236,7 +225,7 @@ public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveExc @Override public void merge(AggregationBuffer agg, Object obj) throws HiveException { - if(obj == null) { + if(obj == null) { return; } NGramAggBuf myagg = (NGramAggBuf) agg; @@ -258,7 +247,7 @@ public void merge(AggregationBuffer agg, Object obj) throws HiveException { myagg.context.add( null ); } else { myagg.context.add( word ); - } + } } partial.subList(partial.size()-contextSize, partial.size()).clear(); myagg.nge.merge(partial); @@ -320,7 +309,7 @@ public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveExcep return; } NGramAggBuf myagg = (NGramAggBuf) agg; - + // Parse out the context and 'k' if we haven't already done so, and while we're at it, // also parse out the precision factor 'pf' if the user has supplied one. if(!myagg.nge.isInitialized()) { @@ -378,7 +367,7 @@ public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveExcep // parse out n-grams, update frequency counts processNgrams(myagg, words); - } + } } else { // we're dealing with an array of strings ArrayList words = new ArrayList(); @@ -399,7 +388,7 @@ public Object terminate(AggregationBuffer agg) throws HiveException { } - // Aggregation buffer methods. + // Aggregation buffer methods. static class NGramAggBuf extends AbstractAggregationBuffer { ArrayList context; NGramEstimator nge; diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCorrelation.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCorrelation.java index 23dc66d..5da5d52 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCorrelation.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCorrelation.java @@ -116,6 +116,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE return new GenericUDAFCorrelationEvaluator(); case STRING: case BOOLEAN: + case DATE: default: throw new UDFArgumentTypeException(1, "Only numeric type arguments are accepted but " @@ -123,6 +124,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE } case STRING: case BOOLEAN: + case DATE: default: throw new UDFArgumentTypeException(0, "Only numeric type arguments are accepted but " diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovariance.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovariance.java index 8fb8a51..f82ef14 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovariance.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovariance.java @@ -106,6 +106,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE return new GenericUDAFCovarianceEvaluator(); case STRING: case BOOLEAN: + case DATE: default: throw new UDFArgumentTypeException(1, "Only numeric or string type arguments are accepted but " @@ -113,6 +114,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE } case STRING: case BOOLEAN: + case DATE: default: throw new UDFArgumentTypeException(0, "Only numeric or string type arguments are accepted but " diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovarianceSample.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovarianceSample.java index ecf7151..de0f153 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovarianceSample.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovarianceSample.java @@ -80,6 +80,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) return new GenericUDAFCovarianceSampleEvaluator(); case STRING: case BOOLEAN: + case DATE: default: throw new UDFArgumentTypeException(1, "Only numeric or string type arguments are accepted but " @@ -87,6 +88,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) } case STRING: case BOOLEAN: + case DATE: default: throw new UDFArgumentTypeException(0, "Only numeric or string type arguments are accepted but " diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java index 9587552..32c6cb5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java @@ -85,6 +85,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE break; case STRING: case BOOLEAN: + case DATE: default: throw new UDFArgumentTypeException(0, "Only numeric type arguments are accepted but " diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java index 71a91e1..fbc5db1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java @@ -90,6 +90,7 @@ public GenericUDAFEvaluator getEvaluator(GenericUDAFParameterInfo info) throws S case TIMESTAMP: case DECIMAL: break; + case DATE: default: throw new UDFArgumentTypeException(0, "Only numeric type arguments are accepted but " diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStd.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStd.java index 5d0aa50..159a2fe 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStd.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStd.java @@ -59,6 +59,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) case DECIMAL: return new GenericUDAFStdEvaluator(); case BOOLEAN: + case DATE: default: throw new UDFArgumentTypeException(0, "Only numeric or string type arguments are accepted but " diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStdSample.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStdSample.java index cde947c..e85046c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStdSample.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStdSample.java @@ -58,6 +58,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE case DECIMAL: return new GenericUDAFStdSampleEvaluator(); case BOOLEAN: + case DATE: default: throw new UDFArgumentTypeException(0, "Only numeric or string type arguments are accepted but " diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java index 2e8cc79..d44df6f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java @@ -72,6 +72,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) case DECIMAL: return new GenericUDAFSumHiveDecimal(); case BOOLEAN: + case DATE: default: throw new UDFArgumentTypeException(0, "Only numeric or string type arguments are accepted but " diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java index abf4008..82fce3f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java @@ -76,6 +76,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE case DECIMAL: return new GenericUDAFVarianceEvaluator(); case BOOLEAN: + case DATE: default: throw new UDFArgumentTypeException(0, "Only numeric or string type arguments are accepted but " diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVarianceSample.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVarianceSample.java index fa549e1..ab863be 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVarianceSample.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVarianceSample.java @@ -59,6 +59,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) case DECIMAL: return new GenericUDAFVarianceSampleEvaluator(); case BOOLEAN: + case DATE: default: throw new UDFArgumentTypeException(0, "Only numeric or string type arguments are accepted but " diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java index 1ff9d35..4f28628 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java @@ -26,22 +26,15 @@ import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.StandardMapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.StructField; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDoubleObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.io.Text; /** @@ -69,7 +62,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE throw new UDFArgumentTypeException(parameters.length-1, "Please specify either three or four arguments."); } - + // Validate the first parameter, which is the expression to compute over. This should be an // array of strings type, or an array of arrays of strings. PrimitiveTypeInfo pti; @@ -99,7 +92,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE } if(pti.getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.STRING) { throw new UDFArgumentTypeException(0, - "Only array or array> is allowed, but " + "Only array or array> is allowed, but " + parameters[0].getTypeName() + " was passed as parameter 1."); } @@ -107,7 +100,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE if(parameters[1].getCategory() != ObjectInspector.Category.PRIMITIVE) { throw new UDFArgumentTypeException(1, "Only integers are accepted but " + parameters[1].getTypeName() + " was passed as parameter 2."); - } + } switch(((PrimitiveTypeInfo) parameters[1]).getPrimitiveCategory()) { case BYTE: case SHORT: @@ -125,7 +118,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE if(parameters[2].getCategory() != ObjectInspector.Category.PRIMITIVE) { throw new UDFArgumentTypeException(2, "Only integers are accepted but " + parameters[2].getTypeName() + " was passed as parameter 3."); - } + } switch(((PrimitiveTypeInfo) parameters[2]).getPrimitiveCategory()) { case BYTE: case SHORT: @@ -144,7 +137,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE if(parameters[3].getCategory() != ObjectInspector.Category.PRIMITIVE) { throw new UDFArgumentTypeException(3, "Only integers are accepted but " + parameters[3].getTypeName() + " was passed as parameter 4."); - } + } switch(((PrimitiveTypeInfo) parameters[3]).getPrimitiveCategory()) { case BYTE: case SHORT: @@ -174,7 +167,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE private PrimitiveObjectInspector kOI; private PrimitiveObjectInspector pOI; - // For PARTIAL2 and FINAL: ObjectInspectors for partial aggregations + // For PARTIAL2 and FINAL: ObjectInspectors for partial aggregations private StandardListObjectInspector loi; @Override @@ -209,8 +202,8 @@ public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveExc // Init output object inspectors. // // The return type for a partial aggregation is still a list of strings. - // - // The return type for FINAL and COMPLETE is a full aggregation result, which is + // + // The return type for FINAL and COMPLETE is a full aggregation result, which is // an array of structures containing the n-gram and its estimated frequency. if (m == Mode.PARTIAL1 || m == Mode.PARTIAL2) { return ObjectInspectorFactory.getStandardListObjectInspector( @@ -224,7 +217,7 @@ public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveExc foi.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector); ArrayList fname = new ArrayList(); fname.add("ngram"); - fname.add("estfrequency"); + fname.add("estfrequency"); return ObjectInspectorFactory.getStandardListObjectInspector( ObjectInspectorFactory.getStandardStructObjectInspector(fname, foi) ); } @@ -232,14 +225,14 @@ public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveExc @Override public void merge(AggregationBuffer agg, Object partial) throws HiveException { - if(partial == null) { + if(partial == null) { return; } NGramAggBuf myagg = (NGramAggBuf) agg; List partialNGrams = (List) loi.getList(partial); int n = Integer.parseInt(partialNGrams.get(partialNGrams.size()-1).toString()); if(myagg.n > 0 && myagg.n != n) { - throw new HiveException(getClass().getSimpleName() + ": mismatch in value for 'n'" + throw new HiveException(getClass().getSimpleName() + ": mismatch in value for 'n'" + ", which usually is caused by a non-constant expression. Found '"+n+"' and '" + myagg.n + "'."); } @@ -273,7 +266,7 @@ public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveExcep return; } NGramAggBuf myagg = (NGramAggBuf) agg; - + // Parse out 'n' and 'k' if we haven't already done so, and while we're at it, // also parse out the precision factor 'pf' if the user has supplied one. if(!myagg.nge.isInitialized()) { @@ -317,7 +310,7 @@ public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveExcep // parse out n-grams, update frequency counts processNgrams(myagg, words); - } + } } else { // we're dealing with an array of strings ArrayList words = new ArrayList(); @@ -337,7 +330,7 @@ public Object terminate(AggregationBuffer agg) throws HiveException { return myagg.nge.getNGrams(); } - // Aggregation buffer methods. + // Aggregation buffer methods. static class NGramAggBuf extends AbstractAggregationBuffer { NGramEstimator nge; int n; diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseCompare.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseCompare.java index 3fb3879..c17a2a6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseCompare.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseCompare.java @@ -136,10 +136,18 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen if (oiTypeInfo0 != oiTypeInfo1) { compareType = CompareType.NEED_CONVERT; - // If either argument is a string, we convert to a double because a number - // in string form should always be convertible into a double - if (oiTypeInfo0.equals(TypeInfoFactory.stringTypeInfo) + if ((oiTypeInfo0.equals(TypeInfoFactory.stringTypeInfo) + && oiTypeInfo1.equals(TypeInfoFactory.dateTypeInfo)) + || (oiTypeInfo0.equals(TypeInfoFactory.dateTypeInfo) + && oiTypeInfo1.equals(TypeInfoFactory.stringTypeInfo))) { + // Date should be comparable with string + compareOI = TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo( + TypeInfoFactory.stringTypeInfo); + + } else if (oiTypeInfo0.equals(TypeInfoFactory.stringTypeInfo) || oiTypeInfo1.equals(TypeInfoFactory.stringTypeInfo)) { + // If either argument is a string, we convert to a double because a number + // in string form should always be convertible into a double compareOI = TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo( TypeInfoFactory.doubleTypeInfo); } else { diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDate.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDate.java new file mode 100644 index 0000000..087f9b1 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDate.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.udf.generic; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter.DateConverter; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; + +/** + * GenericUDFToDate + */ +@Description(name = "date", + value = "CAST( as DATE) - Returns the date represented by the date string.", + extended = "date_string is a string in the format 'yyyy-MM-dd.'" + + "Example:\n " + + " > SELECT CAST('2009-01-01' AS DATE) FROM src LIMIT 1;\n" + + " '2009-01-01'") +public class GenericUDFToDate extends GenericUDF { + + private PrimitiveObjectInspector argumentOI; + private DateConverter dc; + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + if (arguments.length < 1) { + throw new UDFArgumentLengthException( + "The function CAST as DATE requires at least one argument, got " + + arguments.length); + } + try { + argumentOI = (PrimitiveObjectInspector) arguments[0]; + switch (argumentOI.getPrimitiveCategory()) { + case DATE: + case STRING: + case TIMESTAMP: + break; + default: + throw new UDFArgumentException( + "CAST as DATE only allows date,string, or timestamp types"); + } + } catch (ClassCastException e) { + throw new UDFArgumentException( + "The function CAST as DATE takes only primitive types"); + } + + dc = new DateConverter(argumentOI, + PrimitiveObjectInspectorFactory.writableDateObjectInspector); + return PrimitiveObjectInspectorFactory.writableDateObjectInspector; + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + Object o0 = arguments[0].get(); + if (o0 == null) { + return null; + } + + return dc.convert(o0); + } + + @Override + public String getDisplayString(String[] children) { + assert (children.length == 1); + StringBuilder sb = new StringBuilder(); + sb.append("CAST( "); + sb.append(children[0]); + sb.append(" AS DATE)"); + return sb.toString(); + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUnixTimeStamp.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUnixTimeStamp.java index e887552..ce1055e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUnixTimeStamp.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUnixTimeStamp.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector; @@ -42,6 +43,7 @@ public class GenericUDFToUnixTimeStamp extends GenericUDF { private StringObjectInspector intputTextOI; + private DateObjectInspector inputDateOI; private TimestampObjectInspector inputTimestampOI; private StringObjectInspector patternOI; @@ -69,6 +71,8 @@ protected void initializeInput(ObjectInspector[] arguments) throws UDFArgumentEx } patternOI = (StringObjectInspector) arguments[1]; } + } else if (arguments[0] instanceof DateObjectInspector) { + inputDateOI = (DateObjectInspector) arguments[0]; } else if (arguments[0] instanceof TimestampObjectInspector) { inputTimestampOI = (TimestampObjectInspector) arguments[0]; } else { @@ -106,7 +110,11 @@ public Object evaluate(DeferredObject[] arguments) throws HiveException { } catch (ParseException e) { return null; } - } + } else if (inputDateOI != null) { + retValue.set(inputDateOI.getPrimitiveWritableObject(arguments[0].get()) + .getTimeInSeconds()); + return retValue; + } Timestamp timestamp = inputTimestampOI.getPrimitiveJavaObject(arguments[0].get()); retValue.set(timestamp.getTime() / 1000); return retValue; diff --git ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto index d19bc06..417c37a 100644 --- ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto +++ ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto @@ -27,6 +27,12 @@ message DecimalStatistics { optional string sum = 3; } +message DateStatistics { + // min,max values saved as days since epoch + optional sint32 minimum = 1; + optional sint32 maximum = 2; +} + message ColumnStatistics { optional uint64 numberOfValues = 1; optional IntegerStatistics intStatistics = 2; @@ -34,6 +40,7 @@ message ColumnStatistics { optional StringStatistics stringStatistics = 4; optional BucketStatistics bucketStatistics = 5; optional DecimalStatistics decimalStatistics = 6; + optional DateStatistics dateStatistics = 7; } message RowIndexEntry { @@ -93,6 +100,7 @@ message Type { STRUCT = 12; UNION = 13; DECIMAL = 14; + DATE = 15; } required Kind kind = 1; repeated uint32 subtypes = 2 [packed=true]; diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/TestFunctionRegistry.java ql/src/test/org/apache/hadoop/hive/ql/exec/TestFunctionRegistry.java index 4eff65a..9775f38 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/TestFunctionRegistry.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/TestFunctionRegistry.java @@ -18,20 +18,20 @@ package org.apache.hadoop.hive.ql.exec; -import java.util.List; -import java.util.LinkedList; import java.lang.reflect.Method; +import java.util.LinkedList; +import java.util.List; import junit.framework.TestCase; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.IntWritable; public class TestFunctionRegistry extends TestCase { @@ -41,10 +41,11 @@ public void same(HiveDecimalWritable x, HiveDecimalWritable y) {} public void one(IntWritable x, HiveDecimalWritable y) {} public void one(IntWritable x, DoubleWritable y) {} public void one(IntWritable x, IntWritable y) {} + public void mismatch(DateWritable x, HiveDecimalWritable y) {} public void mismatch(TimestampWritable x, HiveDecimalWritable y) {} public void mismatch(BytesWritable x, DoubleWritable y) {} } - + @Override protected void setUp() { } @@ -58,17 +59,18 @@ public void testImplicitConversion() { implicit(TypeInfoFactory.floatTypeInfo, TypeInfoFactory.decimalTypeInfo, true); implicit(TypeInfoFactory.doubleTypeInfo, TypeInfoFactory.decimalTypeInfo, true); implicit(TypeInfoFactory.stringTypeInfo, TypeInfoFactory.decimalTypeInfo, true); + implicit(TypeInfoFactory.dateTypeInfo, TypeInfoFactory.decimalTypeInfo, false); implicit(TypeInfoFactory.timestampTypeInfo, TypeInfoFactory.decimalTypeInfo, false); } - private void verify(Class udf, String name, TypeInfo ta, TypeInfo tb, + private void verify(Class udf, String name, TypeInfo ta, TypeInfo tb, Class a, Class b, boolean throwException) { List args = new LinkedList(); args.add(ta); args.add(tb); Method result = null; - + try { result = FunctionRegistry.getMethodInternal(udf, name, false, args); } catch (UDFArgumentException e) { @@ -116,13 +118,13 @@ private void common(TypeInfo a, TypeInfo b, TypeInfo result) { } public void testCommonClass() { - common(TypeInfoFactory.intTypeInfo, TypeInfoFactory.decimalTypeInfo, + common(TypeInfoFactory.intTypeInfo, TypeInfoFactory.decimalTypeInfo, TypeInfoFactory.decimalTypeInfo); - common(TypeInfoFactory.stringTypeInfo, TypeInfoFactory.decimalTypeInfo, + common(TypeInfoFactory.stringTypeInfo, TypeInfoFactory.decimalTypeInfo, TypeInfoFactory.stringTypeInfo); - common(TypeInfoFactory.doubleTypeInfo, TypeInfoFactory.decimalTypeInfo, + common(TypeInfoFactory.doubleTypeInfo, TypeInfoFactory.decimalTypeInfo, TypeInfoFactory.decimalTypeInfo); - common(TypeInfoFactory.doubleTypeInfo, TypeInfoFactory.stringTypeInfo, + common(TypeInfoFactory.doubleTypeInfo, TypeInfoFactory.stringTypeInfo, TypeInfoFactory.stringTypeInfo); } @@ -131,13 +133,13 @@ private void comparison(TypeInfo a, TypeInfo b, TypeInfo result) { } public void testCommonClassComparison() { - comparison(TypeInfoFactory.intTypeInfo, TypeInfoFactory.decimalTypeInfo, + comparison(TypeInfoFactory.intTypeInfo, TypeInfoFactory.decimalTypeInfo, TypeInfoFactory.decimalTypeInfo); - comparison(TypeInfoFactory.stringTypeInfo, TypeInfoFactory.decimalTypeInfo, + comparison(TypeInfoFactory.stringTypeInfo, TypeInfoFactory.decimalTypeInfo, TypeInfoFactory.decimalTypeInfo); - comparison(TypeInfoFactory.doubleTypeInfo, TypeInfoFactory.decimalTypeInfo, + comparison(TypeInfoFactory.doubleTypeInfo, TypeInfoFactory.decimalTypeInfo, TypeInfoFactory.decimalTypeInfo); - comparison(TypeInfoFactory.doubleTypeInfo, TypeInfoFactory.stringTypeInfo, + comparison(TypeInfoFactory.doubleTypeInfo, TypeInfoFactory.stringTypeInfo, TypeInfoFactory.doubleTypeInfo); } diff --git ql/src/test/queries/clientnegative/date_literal1.q ql/src/test/queries/clientnegative/date_literal1.q new file mode 100644 index 0000000..b7fac0d --- /dev/null +++ ql/src/test/queries/clientnegative/date_literal1.q @@ -0,0 +1,2 @@ +-- Not in YYYY-MM-DD format +SELECT DATE '2001-1-1' FROM src LIMIT 2; diff --git ql/src/test/queries/clientnegative/date_literal2.q ql/src/test/queries/clientnegative/date_literal2.q new file mode 100644 index 0000000..711dc9e --- /dev/null +++ ql/src/test/queries/clientnegative/date_literal2.q @@ -0,0 +1,2 @@ +-- Not in YYYY-MM-DD format +SELECT DATE '2001/01/01' FROM src LIMIT 2; diff --git ql/src/test/queries/clientnegative/date_literal3.q ql/src/test/queries/clientnegative/date_literal3.q new file mode 100644 index 0000000..9483509 --- /dev/null +++ ql/src/test/queries/clientnegative/date_literal3.q @@ -0,0 +1,2 @@ +-- Invalid date value +SELECT DATE '2001-01-32' FROM src; diff --git ql/src/test/queries/clientnegative/invalid_t_alter1.q ql/src/test/queries/clientnegative/invalid_t_alter1.q index 181526c..bb19cff 100644 --- ql/src/test/queries/clientnegative/invalid_t_alter1.q +++ ql/src/test/queries/clientnegative/invalid_t_alter1.q @@ -1,2 +1,2 @@ CREATE TABLE alter_test (d STRING); -ALTER TABLE alter_test CHANGE d d DATE; +ALTER TABLE alter_test CHANGE d d DATETIME; diff --git ql/src/test/queries/clientnegative/invalid_t_alter2.q ql/src/test/queries/clientnegative/invalid_t_alter2.q index a946086..aa01b35 100644 --- ql/src/test/queries/clientnegative/invalid_t_alter2.q +++ ql/src/test/queries/clientnegative/invalid_t_alter2.q @@ -1,2 +1,2 @@ CREATE TABLE alter_test (d STRING); -ALTER TABLE alter_test ADD COLUMNS (ds DATE); +ALTER TABLE alter_test ADD COLUMNS (ds DATETIME); diff --git ql/src/test/queries/clientnegative/invalid_t_create1.q ql/src/test/queries/clientnegative/invalid_t_create1.q deleted file mode 100644 index 777e6cc..0000000 --- ql/src/test/queries/clientnegative/invalid_t_create1.q +++ /dev/null @@ -1 +0,0 @@ -CREATE TABLE date_test (d DATE); diff --git ql/src/test/queries/clientnegative/invalid_t_transform.q ql/src/test/queries/clientnegative/invalid_t_transform.q index 435ea71..dfc4864 100644 --- ql/src/test/queries/clientnegative/invalid_t_transform.q +++ ql/src/test/queries/clientnegative/invalid_t_transform.q @@ -1 +1 @@ -SELECT TRANSFORM(*) USING 'cat' AS (key DATE) FROM src; +SELECT TRANSFORM(*) USING 'cat' AS (key DATETIME) FROM src; diff --git ql/src/test/queries/clientpositive/ctas_date.q ql/src/test/queries/clientpositive/ctas_date.q new file mode 100644 index 0000000..03af69f --- /dev/null +++ ql/src/test/queries/clientpositive/ctas_date.q @@ -0,0 +1,26 @@ +drop table ctas_date_1; +drop table ctas_date_2; +drop view ctas_date_3; +drop view ctas_date_4; + +create table ctas_date_1 (key int, value string, dd date); +insert overwrite table ctas_date_1 + select key, value, date '2012-01-01' from src sort by key, value limit 5; + +-- create table as with date column +create table ctas_date_2 as select key, value, dd, date '1980-12-12' from ctas_date_1; + +-- view with date column +create view ctas_date_3 as select * from ctas_date_2 where dd > date '2000-01-01'; +create view ctas_date_4 as select * from ctas_date_2 where dd < date '2000-01-01'; + +select key, value, dd, date '1980-12-12' from ctas_date_1; +select * from ctas_date_2; +select * from ctas_date_3; +select count(*) from ctas_date_4; + + +drop table ctas_date_1; +drop table ctas_date_2; +drop view ctas_date_3; +drop view ctas_date_4; diff --git ql/src/test/queries/clientpositive/date_1.q ql/src/test/queries/clientpositive/date_1.q new file mode 100644 index 0000000..a2322fc --- /dev/null +++ ql/src/test/queries/clientpositive/date_1.q @@ -0,0 +1,87 @@ +drop table date_1; + +create table date_1 (d date); + +insert overwrite table date_1 + select cast('2011-01-01' as date) from src limit 1; + +select * from date_1 limit 1; +select d, count(d) from date_1 group by d; + +insert overwrite table date_1 + select date '2011-01-01' from src limit 1; + +select * from date_1 limit 1; +select d, count(d) from date_1 group by d; + +insert overwrite table date_1 + select cast(cast('2011-01-01 00:00:00' as timestamp) as date) from src limit 1; + +select * from date_1 limit 1; +select d, count(d) from date_1 group by d; + +-- Valid casts +select + cast('2012-01-01' as string), + cast(d as string), + cast(d as timestamp), + cast(cast(d as timestamp) as date), + cast(d as date) +from date_1 limit 1; + +-- Invalid casts. +select + cast(d as boolean), + cast(d as tinyint), + cast(d as smallint), + cast(d as int), + cast(d as bigint), + cast(d as float), + cast(d as double) +from date_1 limit 1; + +-- These comparisons should all be true +select + date '2011-01-01' = date '2011-01-01', + unix_timestamp(date '2011-01-01') = unix_timestamp(date '2011-01-01'), + unix_timestamp(date '2011-01-01') = unix_timestamp(cast(date '2011-01-01' as timestamp)), + unix_timestamp(date '2011-01-01') = unix_timestamp(cast(cast('2011-01-01 12:13:14' as timestamp) as date)), + unix_timestamp(date '2011-01-01') < unix_timestamp(cast('2011-01-01 00:00:01' as timestamp)), + unix_timestamp(date '2011-01-01') = unix_timestamp(cast('2011-01-01 00:00:00' as timestamp)), + unix_timestamp(date '2011-01-01') > unix_timestamp(cast('2010-12-31 23:59:59' as timestamp)), + date '2011-01-01' = cast(timestamp('2011-01-01 23:24:25') as date), + '2011-01-01' = cast(d as string), + '2011-01-01' = cast(date '2011-01-01' as string) +from date_1 limit 1; + +select + date('2001-01-28'), + date('2001-02-28'), + date('2001-03-28'), + date('2001-04-28'), + date('2001-05-28'), + date('2001-06-28'), + date('2001-07-28'), + date('2001-08-28'), + date('2001-09-28'), + date('2001-10-28'), + date('2001-11-28'), + date('2001-12-28') +from date_1 limit 1; + +select + unix_timestamp(date('2001-01-28')) = unix_timestamp(cast('2001-01-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-02-28')) = unix_timestamp(cast('2001-02-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-03-28')) = unix_timestamp(cast('2001-03-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-04-28')) = unix_timestamp(cast('2001-04-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-05-28')) = unix_timestamp(cast('2001-05-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-06-28')) = unix_timestamp(cast('2001-06-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-07-28')) = unix_timestamp(cast('2001-07-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-08-28')) = unix_timestamp(cast('2001-08-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-09-28')) = unix_timestamp(cast('2001-09-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-10-28')) = unix_timestamp(cast('2001-10-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-11-28')) = unix_timestamp(cast('2001-11-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-12-28')) = unix_timestamp(cast('2001-12-28 0:0:0' as timestamp)) +from date_1 limit 1; + +drop table date_1; diff --git ql/src/test/queries/clientpositive/date_2.q ql/src/test/queries/clientpositive/date_2.q new file mode 100644 index 0000000..0821e01 --- /dev/null +++ ql/src/test/queries/clientpositive/date_2.q @@ -0,0 +1,18 @@ +drop table if exists date_2; + +create table date_2 ( + ORIGIN_CITY_NAME string, + DEST_CITY_NAME string, + FL_DATE date, + ARR_DELAY float, + FL_NUM int +); + +LOAD DATA LOCAL INPATH '../data/files/flights_tiny.txt.1' OVERWRITE INTO TABLE date_2; + +select fl_date, fl_num from date_2 order by fl_date asc, fl_num desc; +select fl_date, fl_num from date_2 order by fl_date desc, fl_num asc; + +select fl_date, count(*) from date_2 group by fl_date; + +drop table date_2; diff --git ql/src/test/queries/clientpositive/date_3.q ql/src/test/queries/clientpositive/date_3.q new file mode 100644 index 0000000..be25148 --- /dev/null +++ ql/src/test/queries/clientpositive/date_3.q @@ -0,0 +1,14 @@ +drop table date_3; + +create table date_3 ( + c1 int +); + +alter table date_3 add columns (c2 date); + +insert overwrite table date_3 + select 1, cast(cast('2011-01-01 00:00:00' as timestamp) as date) from src limit 1; + +select * from date_3; + +drop table date_3; diff --git ql/src/test/queries/clientpositive/date_4.q ql/src/test/queries/clientpositive/date_4.q new file mode 100644 index 0000000..4801a79 --- /dev/null +++ ql/src/test/queries/clientpositive/date_4.q @@ -0,0 +1,11 @@ +drop table date_4; + +create table date_4 (d date); +alter table date_4 set serde 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'; + +-- Test date literal syntax +insert overwrite table date_4 + select date '2011-01-01' from src limit 1; +select d, date '2011-01-01' from date_4 limit 1; + +drop table date_4; diff --git ql/src/test/queries/clientpositive/date_comparison.q ql/src/test/queries/clientpositive/date_comparison.q new file mode 100644 index 0000000..bdcb6c1 --- /dev/null +++ ql/src/test/queries/clientpositive/date_comparison.q @@ -0,0 +1,38 @@ +-- Comparisons against same value +select cast('2011-05-06' as date) > + cast('2011-05-06' as date) from src limit 1; + +select cast('2011-05-06' as date) < + cast('2011-05-06' as date) from src limit 1; + +select cast('2011-05-06' as date) = + cast('2011-05-06' as date) from src limit 1; + +select cast('2011-05-06' as date) <> + cast('2011-05-06' as date) from src limit 1; + +select cast('2011-05-06' as date) >= + cast('2011-05-06' as date) from src limit 1; + +select cast('2011-05-06' as date) <= + cast('2011-05-06' as date) from src limit 1; + +-- Now try with differing values +select cast('2011-05-05' as date) > + cast('2011-05-06' as date) from src limit 1; + +select cast('2011-05-05' as date) < + cast('2011-05-06' as date) from src limit 1; + +select cast('2011-05-05' as date) = + cast('2011-05-06' as date) from src limit 1; + +select cast('2011-05-05' as date) <> + cast('2011-05-06' as date) from src limit 1; + +select cast('2011-05-05' as date) >= + cast('2011-05-06' as date) from src limit 1; + +select cast('2011-05-05' as date) <= + cast('2011-05-06' as date) from src limit 1; + diff --git ql/src/test/queries/clientpositive/date_join1.q ql/src/test/queries/clientpositive/date_join1.q new file mode 100644 index 0000000..a5844b7 --- /dev/null +++ ql/src/test/queries/clientpositive/date_join1.q @@ -0,0 +1,19 @@ +drop table date_join1; + +create table date_join1 ( + ORIGIN_CITY_NAME string, + DEST_CITY_NAME string, + FL_DATE date, + ARR_DELAY float, + FL_NUM int +); + +LOAD DATA LOCAL INPATH '../data/files/flights_join.txt' OVERWRITE INTO TABLE date_join1; + +-- Note that there are 2 rows with date 2000-11-28, so we should expect 4 rows with that date in the join results +select t1.fl_num, t1.fl_date, t2.fl_num, t2.fl_date + from date_join1 t1 + join date_join1 t2 + on (t1.fl_date = t2.fl_date); + +drop table date_join1; diff --git ql/src/test/queries/clientpositive/date_serde.q ql/src/test/queries/clientpositive/date_serde.q new file mode 100644 index 0000000..ffc06d2 --- /dev/null +++ ql/src/test/queries/clientpositive/date_serde.q @@ -0,0 +1,112 @@ +drop table if exists date_serde_regex; +drop table date_serde_lb; +drop table date_serde_ls; +drop table date_serde_c; +drop table date_serde_lbc; +drop table date_serde_orc; + + +-- +-- RegexSerDe +-- +create table date_serde_regex ( + ORIGIN_CITY_NAME string, + DEST_CITY_NAME string, + FL_DATE date, + ARR_DELAY float, + FL_NUM int +) +row format serde 'org.apache.hadoop.hive.serde2.RegexSerDe' +with serdeproperties ( + "input.regex" = "([^]*)([^]*)([^]*)([^]*)([0-9]*)" +) +stored as textfile; + +load data local inpath '../data/files/flights_tiny.txt.1' overwrite into table date_serde_regex; + +select * from date_serde_regex; +select fl_date, count(*) from date_serde_regex group by fl_date; + +-- +-- LazyBinary +-- +create table date_serde_lb ( + c1 date, + c2 int +); +alter table date_serde_lb set serde 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe'; + +insert overwrite table date_serde_lb + select fl_date, fl_num from date_serde_regex limit 1; + +select * from date_serde_lb; +select c1, sum(c2) from date_serde_lb group by c1; + +-- +-- LazySimple +-- +create table date_serde_ls ( + c1 date, + c2 int +); +alter table date_serde_ls set serde 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'; + +insert overwrite table date_serde_ls + select c1, c2 from date_serde_lb limit 1; + +select * from date_serde_ls; +select c1, sum(c2) from date_serde_ls group by c1; + +-- +-- Columnar +-- +create table date_serde_c ( + c1 date, + c2 int +) stored as rcfile; +alter table date_serde_c set serde 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe'; + +insert overwrite table date_serde_c + select c1, c2 from date_serde_ls limit 1; + +select * from date_serde_c; +select c1, sum(c2) from date_serde_c group by c1; + +-- +-- LazyBinaryColumnar +-- +create table date_serde_lbc ( + c1 date, + c2 int +) stored as rcfile; +alter table date_serde_lbc set serde 'org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe'; + +insert overwrite table date_serde_lbc + select c1, c2 from date_serde_c limit 1; + +select * from date_serde_lbc; +select c1, sum(c2) from date_serde_lbc group by c1; + +-- +-- ORC +-- +create table date_serde_orc ( + c1 date, + c2 int +) stored as orc; +alter table date_serde_orc set serde 'org.apache.hadoop.hive.ql.io.orc.OrcSerde'; + +insert overwrite table date_serde_orc + select c1, c2 from date_serde_lbc limit 1; + +select * from date_serde_orc; +select c1, sum(c2) from date_serde_orc group by c1; + + + +drop table date_serde_regex; +drop table date_serde_lb; +drop table date_serde_ls; +drop table date_serde_c; +drop table date_serde_lbc; +drop table date_serde_orc; diff --git ql/src/test/queries/clientpositive/date_udf.q ql/src/test/queries/clientpositive/date_udf.q new file mode 100644 index 0000000..9696320 --- /dev/null +++ ql/src/test/queries/clientpositive/date_udf.q @@ -0,0 +1,69 @@ +drop table date_udf; +drop table date_udf_string; +drop table date_udf_flight; + +create table date_udf (d date); +create table date_udf_string (d string); +from src + insert overwrite table date_udf + select '2011-05-06' limit 1 + insert overwrite table date_udf_string + select '2011-05-06' limit 1; + +create table date_udf_flight ( + ORIGIN_CITY_NAME string, + DEST_CITY_NAME string, + FL_DATE date, + ARR_DELAY float, + FL_NUM int +); +LOAD DATA LOCAL INPATH '../data/files/flights_tiny.txt.1' OVERWRITE INTO TABLE date_udf_flight; + +-- Test UDFs with date input +select unix_timestamp(d), year(d), month(d), day(d), dayofmonth(d), + weekofyear(d), to_date(d) + from date_udf; + +select date_add(d, 5), date_sub(d, 10) + from date_udf; + +select datediff(d, d), datediff(d, '2002-03-21'), datediff('2002-03-21', d), + datediff(cast ('2002-03-21 00:00:00' as timestamp), d), + datediff(d, cast ('2002-03-21 00:00:00' as timestamp)) + from date_udf; + +-- Test UDFs with string input +select unix_timestamp(d), year(d), month(d), day(d), dayofmonth(d), + weekofyear(d), to_date(d) + from date_udf_string; + +select date_add(d, 5), date_sub(d, 10) from date_udf_string; + +select datediff(d, d), datediff(d, '2002-03-21'), datediff('2002-03-21', d), + datediff('2002-03-21 00:00:00', d), + datediff(d, '2002-03-21 00:00:00') + from date_udf_string; + +select + to_utc_timestamp(date '1970-01-01', 'America/Los_Angeles'), + from_utc_timestamp(date '1970-01-01', 'America/Los_Angeles'), + to_utc_timestamp(date '2013-06-19', 'America/Los_Angeles'), + from_utc_timestamp(date '2013-06-19', 'America/Los_Angeles') + from date_udf; + +-- should all be true +select + to_utc_timestamp(date '1970-01-01', 'America/Los_Angeles') = to_utc_timestamp(timestamp('1970-01-01 00:00:00'), 'America/Los_Angeles'), + from_utc_timestamp(date '1970-01-01', 'America/Los_Angeles') = from_utc_timestamp(timestamp('1970-01-01 00:00:00'), 'America/Los_Angeles'), + to_utc_timestamp(date '2013-06-19', 'America/Los_Angeles') = to_utc_timestamp(timestamp('2013-06-19 00:00:00'), 'America/Los_Angeles'), + from_utc_timestamp(date '2013-06-19', 'America/Los_Angeles') = from_utc_timestamp(timestamp('2013-06-19 00:00:00'), 'America/Los_Angeles') + from date_udf; + +-- Aggregation functions (min/max) +select min(fl_date) from date_udf_flight; +select max(fl_date) from date_udf_flight; + + +drop table date_udf; +drop table date_udf_string; +drop table date_udf_flight; diff --git ql/src/test/queries/clientpositive/partition_date.q ql/src/test/queries/clientpositive/partition_date.q new file mode 100644 index 0000000..c788916 --- /dev/null +++ ql/src/test/queries/clientpositive/partition_date.q @@ -0,0 +1,45 @@ +drop table partition_date_1; + +create table partition_date_1 (key string, value string) partitioned by (dt date, region int); + +insert overwrite table partition_date_1 partition(dt='2000-01-01', region=1) + select * from src limit 10; +insert overwrite table partition_date_1 partition(dt='2000-01-01', region=2) + select * from src limit 5; +insert overwrite table partition_date_1 partition(dt='2013-08-08', region=1) + select * from src limit 20; +insert overwrite table partition_date_1 partition(dt='2013-08-08', region=10) + select * from src limit 11; + +select distinct dt from partition_date_1; +select *, cast(dt as timestamp) from partition_date_1 where dt = '2000-01-01' and region = 2; + +-- 15 +select count(*) from partition_date_1 where dt = date '2000-01-01'; +-- 15. Also try with string value in predicate +select count(*) from partition_date_1 where dt = '2000-01-01'; +-- 5 +select count(*) from partition_date_1 where dt = date '2000-01-01' and region = 2; +-- 11 +select count(*) from partition_date_1 where dt = date '2013-08-08' and region = 10; +-- 30 +select count(*) from partition_date_1 where region = 1; +-- 0 +select count(*) from partition_date_1 where dt = date '2000-01-01' and region = 3; +-- 0 +select count(*) from partition_date_1 where dt = date '1999-01-01'; + +-- Try other comparison operations + +-- 20 +select count(*) from partition_date_1 where dt > date '2000-01-01' and region = 1; +-- 10 +select count(*) from partition_date_1 where dt < date '2000-01-02' and region = 1; +-- 20 +select count(*) from partition_date_1 where dt >= date '2000-01-02' and region = 1; +-- 10 +select count(*) from partition_date_1 where dt <= date '2000-01-01' and region = 1; +-- 20 +select count(*) from partition_date_1 where dt <> date '2000-01-01' and region = 1; + +drop table partition_date_1; diff --git ql/src/test/queries/clientpositive/union_date.q ql/src/test/queries/clientpositive/union_date.q new file mode 100644 index 0000000..e332a8a --- /dev/null +++ ql/src/test/queries/clientpositive/union_date.q @@ -0,0 +1,32 @@ +drop table union_date_1; +drop table union_date_2; + +create table union_date_1 ( + ORIGIN_CITY_NAME string, + DEST_CITY_NAME string, + FL_DATE date, + ARR_DELAY float, + FL_NUM int +); + +create table union_date_2 ( + ORIGIN_CITY_NAME string, + DEST_CITY_NAME string, + FL_DATE date, + ARR_DELAY float, + FL_NUM int +); + +LOAD DATA LOCAL INPATH '../data/files/flights_join.txt' OVERWRITE INTO TABLE union_date_1; +LOAD DATA LOCAL INPATH '../data/files/flights_join.txt' OVERWRITE INTO TABLE union_date_2; + +select * from ( + select fl_num, fl_date from union_date_1 + union all + select fl_num, fl_date from union_date_2 +) union_result order by fl_date, fl_num; + +drop table union_date_1; +drop table union_date_2; + + diff --git ql/src/test/results/clientnegative/date_literal1.q.out ql/src/test/results/clientnegative/date_literal1.q.out new file mode 100644 index 0000000..82f6425 --- /dev/null +++ ql/src/test/results/clientnegative/date_literal1.q.out @@ -0,0 +1 @@ +FAILED: SemanticException Unable to convert date literal string to date value. diff --git ql/src/test/results/clientnegative/date_literal2.q.out ql/src/test/results/clientnegative/date_literal2.q.out new file mode 100644 index 0000000..82f6425 --- /dev/null +++ ql/src/test/results/clientnegative/date_literal2.q.out @@ -0,0 +1 @@ +FAILED: SemanticException Unable to convert date literal string to date value. diff --git ql/src/test/results/clientnegative/date_literal3.q.out ql/src/test/results/clientnegative/date_literal3.q.out new file mode 100644 index 0000000..82f6425 --- /dev/null +++ ql/src/test/results/clientnegative/date_literal3.q.out @@ -0,0 +1 @@ +FAILED: SemanticException Unable to convert date literal string to date value. diff --git ql/src/test/results/clientnegative/invalid_create_tbl1.q.out ql/src/test/results/clientnegative/invalid_create_tbl1.q.out index a8839fe..04dfd97 100644 --- ql/src/test/results/clientnegative/invalid_create_tbl1.q.out +++ ql/src/test/results/clientnegative/invalid_create_tbl1.q.out @@ -1 +1 @@ -FAILED: SemanticException [Error 10099]: DATE and DATETIME types aren't supported yet. Please use TIMESTAMP instead +FAILED: SemanticException [Error 10099]: DATETIME type isn't supported yet. Please use DATE or TIMESTAMP instead diff --git ql/src/test/results/clientnegative/invalid_t_alter1.q.out ql/src/test/results/clientnegative/invalid_t_alter1.q.out index 3513175..d769404 100644 --- ql/src/test/results/clientnegative/invalid_t_alter1.q.out +++ ql/src/test/results/clientnegative/invalid_t_alter1.q.out @@ -3,4 +3,4 @@ PREHOOK: type: CREATETABLE POSTHOOK: query: CREATE TABLE alter_test (d STRING) POSTHOOK: type: CREATETABLE POSTHOOK: Output: default@alter_test -FAILED: SemanticException [Error 10099]: DATE and DATETIME types aren't supported yet. Please use TIMESTAMP instead +FAILED: SemanticException [Error 10099]: DATETIME type isn't supported yet. Please use DATE or TIMESTAMP instead diff --git ql/src/test/results/clientnegative/invalid_t_alter2.q.out ql/src/test/results/clientnegative/invalid_t_alter2.q.out index 3513175..d769404 100644 --- ql/src/test/results/clientnegative/invalid_t_alter2.q.out +++ ql/src/test/results/clientnegative/invalid_t_alter2.q.out @@ -3,4 +3,4 @@ PREHOOK: type: CREATETABLE POSTHOOK: query: CREATE TABLE alter_test (d STRING) POSTHOOK: type: CREATETABLE POSTHOOK: Output: default@alter_test -FAILED: SemanticException [Error 10099]: DATE and DATETIME types aren't supported yet. Please use TIMESTAMP instead +FAILED: SemanticException [Error 10099]: DATETIME type isn't supported yet. Please use DATE or TIMESTAMP instead diff --git ql/src/test/results/clientnegative/invalid_t_create1.q.out ql/src/test/results/clientnegative/invalid_t_create1.q.out deleted file mode 100644 index a8839fe..0000000 --- ql/src/test/results/clientnegative/invalid_t_create1.q.out +++ /dev/null @@ -1 +0,0 @@ -FAILED: SemanticException [Error 10099]: DATE and DATETIME types aren't supported yet. Please use TIMESTAMP instead diff --git ql/src/test/results/clientnegative/invalid_t_create2.q.out ql/src/test/results/clientnegative/invalid_t_create2.q.out index a8839fe..04dfd97 100644 --- ql/src/test/results/clientnegative/invalid_t_create2.q.out +++ ql/src/test/results/clientnegative/invalid_t_create2.q.out @@ -1 +1 @@ -FAILED: SemanticException [Error 10099]: DATE and DATETIME types aren't supported yet. Please use TIMESTAMP instead +FAILED: SemanticException [Error 10099]: DATETIME type isn't supported yet. Please use DATE or TIMESTAMP instead diff --git ql/src/test/results/clientnegative/invalid_t_transform.q.out ql/src/test/results/clientnegative/invalid_t_transform.q.out index a8839fe..04dfd97 100644 --- ql/src/test/results/clientnegative/invalid_t_transform.q.out +++ ql/src/test/results/clientnegative/invalid_t_transform.q.out @@ -1 +1 @@ -FAILED: SemanticException [Error 10099]: DATE and DATETIME types aren't supported yet. Please use TIMESTAMP instead +FAILED: SemanticException [Error 10099]: DATETIME type isn't supported yet. Please use DATE or TIMESTAMP instead diff --git ql/src/test/results/clientpositive/ctas_date.q.out ql/src/test/results/clientpositive/ctas_date.q.out new file mode 100644 index 0000000..c3404fe --- /dev/null +++ ql/src/test/results/clientpositive/ctas_date.q.out @@ -0,0 +1,172 @@ +PREHOOK: query: drop table ctas_date_1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table ctas_date_1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table ctas_date_2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table ctas_date_2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop view ctas_date_3 +PREHOOK: type: DROPVIEW +POSTHOOK: query: drop view ctas_date_3 +POSTHOOK: type: DROPVIEW +PREHOOK: query: drop view ctas_date_4 +PREHOOK: type: DROPVIEW +POSTHOOK: query: drop view ctas_date_4 +POSTHOOK: type: DROPVIEW +PREHOOK: query: create table ctas_date_1 (key int, value string, dd date) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table ctas_date_1 (key int, value string, dd date) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@ctas_date_1 +PREHOOK: query: insert overwrite table ctas_date_1 + select key, value, date '2012-01-01' from src sort by key, value limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@ctas_date_1 +POSTHOOK: query: insert overwrite table ctas_date_1 + select key, value, date '2012-01-01' from src sort by key, value limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@ctas_date_1 +POSTHOOK: Lineage: ctas_date_1.dd SIMPLE [] +POSTHOOK: Lineage: ctas_date_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_date_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- create table as with date column +create table ctas_date_2 as select key, value, dd, date '1980-12-12' from ctas_date_1 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@ctas_date_1 +POSTHOOK: query: -- create table as with date column +create table ctas_date_2 as select key, value, dd, date '1980-12-12' from ctas_date_1 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@ctas_date_1 +POSTHOOK: Output: default@ctas_date_2 +POSTHOOK: Lineage: ctas_date_1.dd SIMPLE [] +POSTHOOK: Lineage: ctas_date_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_date_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- view with date column +create view ctas_date_3 as select * from ctas_date_2 where dd > date '2000-01-01' +PREHOOK: type: CREATEVIEW +POSTHOOK: query: -- view with date column +create view ctas_date_3 as select * from ctas_date_2 where dd > date '2000-01-01' +POSTHOOK: type: CREATEVIEW +POSTHOOK: Output: default@ctas_date_3 +POSTHOOK: Lineage: ctas_date_1.dd SIMPLE [] +POSTHOOK: Lineage: ctas_date_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_date_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: create view ctas_date_4 as select * from ctas_date_2 where dd < date '2000-01-01' +PREHOOK: type: CREATEVIEW +POSTHOOK: query: create view ctas_date_4 as select * from ctas_date_2 where dd < date '2000-01-01' +POSTHOOK: type: CREATEVIEW +POSTHOOK: Output: default@ctas_date_4 +POSTHOOK: Lineage: ctas_date_1.dd SIMPLE [] +POSTHOOK: Lineage: ctas_date_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_date_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select key, value, dd, date '1980-12-12' from ctas_date_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@ctas_date_1 +#### A masked pattern was here #### +POSTHOOK: query: select key, value, dd, date '1980-12-12' from ctas_date_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ctas_date_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: ctas_date_1.dd SIMPLE [] +POSTHOOK: Lineage: ctas_date_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_date_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +0 val_0 2012-01-01 1980-12-12 +0 val_0 2012-01-01 1980-12-12 +0 val_0 2012-01-01 1980-12-12 +10 val_10 2012-01-01 1980-12-12 +100 val_100 2012-01-01 1980-12-12 +PREHOOK: query: select * from ctas_date_2 +PREHOOK: type: QUERY +PREHOOK: Input: default@ctas_date_2 +#### A masked pattern was here #### +POSTHOOK: query: select * from ctas_date_2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ctas_date_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: ctas_date_1.dd SIMPLE [] +POSTHOOK: Lineage: ctas_date_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_date_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +0 val_0 2012-01-01 1980-12-12 +0 val_0 2012-01-01 1980-12-12 +0 val_0 2012-01-01 1980-12-12 +10 val_10 2012-01-01 1980-12-12 +100 val_100 2012-01-01 1980-12-12 +PREHOOK: query: select * from ctas_date_3 +PREHOOK: type: QUERY +PREHOOK: Input: default@ctas_date_2 +PREHOOK: Input: default@ctas_date_3 +#### A masked pattern was here #### +POSTHOOK: query: select * from ctas_date_3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ctas_date_2 +POSTHOOK: Input: default@ctas_date_3 +#### A masked pattern was here #### +POSTHOOK: Lineage: ctas_date_1.dd SIMPLE [] +POSTHOOK: Lineage: ctas_date_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_date_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +0 val_0 2012-01-01 1980-12-12 +0 val_0 2012-01-01 1980-12-12 +0 val_0 2012-01-01 1980-12-12 +10 val_10 2012-01-01 1980-12-12 +100 val_100 2012-01-01 1980-12-12 +PREHOOK: query: select count(*) from ctas_date_4 +PREHOOK: type: QUERY +PREHOOK: Input: default@ctas_date_2 +PREHOOK: Input: default@ctas_date_4 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from ctas_date_4 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ctas_date_2 +POSTHOOK: Input: default@ctas_date_4 +#### A masked pattern was here #### +POSTHOOK: Lineage: ctas_date_1.dd SIMPLE [] +POSTHOOK: Lineage: ctas_date_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_date_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +0 +PREHOOK: query: drop table ctas_date_1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ctas_date_1 +PREHOOK: Output: default@ctas_date_1 +POSTHOOK: query: drop table ctas_date_1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ctas_date_1 +POSTHOOK: Output: default@ctas_date_1 +POSTHOOK: Lineage: ctas_date_1.dd SIMPLE [] +POSTHOOK: Lineage: ctas_date_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_date_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: drop table ctas_date_2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ctas_date_2 +PREHOOK: Output: default@ctas_date_2 +POSTHOOK: query: drop table ctas_date_2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ctas_date_2 +POSTHOOK: Output: default@ctas_date_2 +POSTHOOK: Lineage: ctas_date_1.dd SIMPLE [] +POSTHOOK: Lineage: ctas_date_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_date_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: drop view ctas_date_3 +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@ctas_date_3 +PREHOOK: Output: default@ctas_date_3 +POSTHOOK: query: drop view ctas_date_3 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@ctas_date_3 +POSTHOOK: Output: default@ctas_date_3 +POSTHOOK: Lineage: ctas_date_1.dd SIMPLE [] +POSTHOOK: Lineage: ctas_date_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_date_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: drop view ctas_date_4 +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@ctas_date_4 +PREHOOK: Output: default@ctas_date_4 +POSTHOOK: query: drop view ctas_date_4 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@ctas_date_4 +POSTHOOK: Output: default@ctas_date_4 +POSTHOOK: Lineage: ctas_date_1.dd SIMPLE [] +POSTHOOK: Lineage: ctas_date_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_date_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] diff --git ql/src/test/results/clientpositive/date_1.q.out ql/src/test/results/clientpositive/date_1.q.out new file mode 100644 index 0000000..354681b --- /dev/null +++ ql/src/test/results/clientpositive/date_1.q.out @@ -0,0 +1,290 @@ +PREHOOK: query: drop table date_1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table date_1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table date_1 (d date) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table date_1 (d date) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@date_1 +PREHOOK: query: insert overwrite table date_1 + select cast('2011-01-01' as date) from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@date_1 +POSTHOOK: query: insert overwrite table date_1 + select cast('2011-01-01' as date) from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@date_1 +POSTHOOK: Lineage: date_1.d EXPRESSION [] +PREHOOK: query: select * from date_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@date_1 +#### A masked pattern was here #### +POSTHOOK: query: select * from date_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: date_1.d EXPRESSION [] +2011-01-01 +PREHOOK: query: select d, count(d) from date_1 group by d +PREHOOK: type: QUERY +PREHOOK: Input: default@date_1 +#### A masked pattern was here #### +POSTHOOK: query: select d, count(d) from date_1 group by d +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: date_1.d EXPRESSION [] +2011-01-01 1 +PREHOOK: query: insert overwrite table date_1 + select date '2011-01-01' from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@date_1 +POSTHOOK: query: insert overwrite table date_1 + select date '2011-01-01' from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@date_1 +POSTHOOK: Lineage: date_1.d EXPRESSION [] +POSTHOOK: Lineage: date_1.d SIMPLE [] +PREHOOK: query: select * from date_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@date_1 +#### A masked pattern was here #### +POSTHOOK: query: select * from date_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: date_1.d EXPRESSION [] +POSTHOOK: Lineage: date_1.d SIMPLE [] +2011-01-01 +PREHOOK: query: select d, count(d) from date_1 group by d +PREHOOK: type: QUERY +PREHOOK: Input: default@date_1 +#### A masked pattern was here #### +POSTHOOK: query: select d, count(d) from date_1 group by d +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: date_1.d EXPRESSION [] +POSTHOOK: Lineage: date_1.d SIMPLE [] +2011-01-01 1 +PREHOOK: query: insert overwrite table date_1 + select cast(cast('2011-01-01 00:00:00' as timestamp) as date) from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@date_1 +POSTHOOK: query: insert overwrite table date_1 + select cast(cast('2011-01-01 00:00:00' as timestamp) as date) from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@date_1 +POSTHOOK: Lineage: date_1.d EXPRESSION [] +POSTHOOK: Lineage: date_1.d SIMPLE [] +POSTHOOK: Lineage: date_1.d EXPRESSION [] +PREHOOK: query: select * from date_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@date_1 +#### A masked pattern was here #### +POSTHOOK: query: select * from date_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: date_1.d EXPRESSION [] +POSTHOOK: Lineage: date_1.d SIMPLE [] +POSTHOOK: Lineage: date_1.d EXPRESSION [] +2011-01-01 +PREHOOK: query: select d, count(d) from date_1 group by d +PREHOOK: type: QUERY +PREHOOK: Input: default@date_1 +#### A masked pattern was here #### +POSTHOOK: query: select d, count(d) from date_1 group by d +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: date_1.d EXPRESSION [] +POSTHOOK: Lineage: date_1.d SIMPLE [] +POSTHOOK: Lineage: date_1.d EXPRESSION [] +2011-01-01 1 +PREHOOK: query: -- Valid casts +select + cast('2012-01-01' as string), + cast(d as string), + cast(d as timestamp), + cast(cast(d as timestamp) as date), + cast(d as date) +from date_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@date_1 +#### A masked pattern was here #### +POSTHOOK: query: -- Valid casts +select + cast('2012-01-01' as string), + cast(d as string), + cast(d as timestamp), + cast(cast(d as timestamp) as date), + cast(d as date) +from date_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: date_1.d EXPRESSION [] +POSTHOOK: Lineage: date_1.d SIMPLE [] +POSTHOOK: Lineage: date_1.d EXPRESSION [] +2012-01-01 2011-01-01 2011-01-01 00:00:00 2011-01-01 2011-01-01 +PREHOOK: query: -- Invalid casts. +select + cast(d as boolean), + cast(d as tinyint), + cast(d as smallint), + cast(d as int), + cast(d as bigint), + cast(d as float), + cast(d as double) +from date_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@date_1 +#### A masked pattern was here #### +POSTHOOK: query: -- Invalid casts. +select + cast(d as boolean), + cast(d as tinyint), + cast(d as smallint), + cast(d as int), + cast(d as bigint), + cast(d as float), + cast(d as double) +from date_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: date_1.d EXPRESSION [] +POSTHOOK: Lineage: date_1.d SIMPLE [] +POSTHOOK: Lineage: date_1.d EXPRESSION [] +NULL NULL NULL NULL NULL NULL NULL +PREHOOK: query: -- These comparisons should all be true +select + date '2011-01-01' = date '2011-01-01', + unix_timestamp(date '2011-01-01') = unix_timestamp(date '2011-01-01'), + unix_timestamp(date '2011-01-01') = unix_timestamp(cast(date '2011-01-01' as timestamp)), + unix_timestamp(date '2011-01-01') = unix_timestamp(cast(cast('2011-01-01 12:13:14' as timestamp) as date)), + unix_timestamp(date '2011-01-01') < unix_timestamp(cast('2011-01-01 00:00:01' as timestamp)), + unix_timestamp(date '2011-01-01') = unix_timestamp(cast('2011-01-01 00:00:00' as timestamp)), + unix_timestamp(date '2011-01-01') > unix_timestamp(cast('2010-12-31 23:59:59' as timestamp)), + date '2011-01-01' = cast(timestamp('2011-01-01 23:24:25') as date), + '2011-01-01' = cast(d as string), + '2011-01-01' = cast(date '2011-01-01' as string) +from date_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@date_1 +#### A masked pattern was here #### +POSTHOOK: query: -- These comparisons should all be true +select + date '2011-01-01' = date '2011-01-01', + unix_timestamp(date '2011-01-01') = unix_timestamp(date '2011-01-01'), + unix_timestamp(date '2011-01-01') = unix_timestamp(cast(date '2011-01-01' as timestamp)), + unix_timestamp(date '2011-01-01') = unix_timestamp(cast(cast('2011-01-01 12:13:14' as timestamp) as date)), + unix_timestamp(date '2011-01-01') < unix_timestamp(cast('2011-01-01 00:00:01' as timestamp)), + unix_timestamp(date '2011-01-01') = unix_timestamp(cast('2011-01-01 00:00:00' as timestamp)), + unix_timestamp(date '2011-01-01') > unix_timestamp(cast('2010-12-31 23:59:59' as timestamp)), + date '2011-01-01' = cast(timestamp('2011-01-01 23:24:25') as date), + '2011-01-01' = cast(d as string), + '2011-01-01' = cast(date '2011-01-01' as string) +from date_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: date_1.d EXPRESSION [] +POSTHOOK: Lineage: date_1.d SIMPLE [] +POSTHOOK: Lineage: date_1.d EXPRESSION [] +true true true true true true true true true true +PREHOOK: query: select + date('2001-01-28'), + date('2001-02-28'), + date('2001-03-28'), + date('2001-04-28'), + date('2001-05-28'), + date('2001-06-28'), + date('2001-07-28'), + date('2001-08-28'), + date('2001-09-28'), + date('2001-10-28'), + date('2001-11-28'), + date('2001-12-28') +from date_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@date_1 +#### A masked pattern was here #### +POSTHOOK: query: select + date('2001-01-28'), + date('2001-02-28'), + date('2001-03-28'), + date('2001-04-28'), + date('2001-05-28'), + date('2001-06-28'), + date('2001-07-28'), + date('2001-08-28'), + date('2001-09-28'), + date('2001-10-28'), + date('2001-11-28'), + date('2001-12-28') +from date_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: date_1.d EXPRESSION [] +POSTHOOK: Lineage: date_1.d SIMPLE [] +POSTHOOK: Lineage: date_1.d EXPRESSION [] +2001-01-28 2001-02-28 2001-03-28 2001-04-28 2001-05-28 2001-06-28 2001-07-28 2001-08-28 2001-09-28 2001-10-28 2001-11-28 2001-12-28 +PREHOOK: query: select + unix_timestamp(date('2001-01-28')) = unix_timestamp(cast('2001-01-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-02-28')) = unix_timestamp(cast('2001-02-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-03-28')) = unix_timestamp(cast('2001-03-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-04-28')) = unix_timestamp(cast('2001-04-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-05-28')) = unix_timestamp(cast('2001-05-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-06-28')) = unix_timestamp(cast('2001-06-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-07-28')) = unix_timestamp(cast('2001-07-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-08-28')) = unix_timestamp(cast('2001-08-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-09-28')) = unix_timestamp(cast('2001-09-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-10-28')) = unix_timestamp(cast('2001-10-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-11-28')) = unix_timestamp(cast('2001-11-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-12-28')) = unix_timestamp(cast('2001-12-28 0:0:0' as timestamp)) +from date_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@date_1 +#### A masked pattern was here #### +POSTHOOK: query: select + unix_timestamp(date('2001-01-28')) = unix_timestamp(cast('2001-01-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-02-28')) = unix_timestamp(cast('2001-02-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-03-28')) = unix_timestamp(cast('2001-03-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-04-28')) = unix_timestamp(cast('2001-04-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-05-28')) = unix_timestamp(cast('2001-05-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-06-28')) = unix_timestamp(cast('2001-06-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-07-28')) = unix_timestamp(cast('2001-07-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-08-28')) = unix_timestamp(cast('2001-08-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-09-28')) = unix_timestamp(cast('2001-09-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-10-28')) = unix_timestamp(cast('2001-10-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-11-28')) = unix_timestamp(cast('2001-11-28 0:0:0' as timestamp)), + unix_timestamp(date('2001-12-28')) = unix_timestamp(cast('2001-12-28 0:0:0' as timestamp)) +from date_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: date_1.d EXPRESSION [] +POSTHOOK: Lineage: date_1.d SIMPLE [] +POSTHOOK: Lineage: date_1.d EXPRESSION [] +true true true true true true true true true true true true +PREHOOK: query: drop table date_1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@date_1 +PREHOOK: Output: default@date_1 +POSTHOOK: query: drop table date_1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@date_1 +POSTHOOK: Output: default@date_1 +POSTHOOK: Lineage: date_1.d EXPRESSION [] +POSTHOOK: Lineage: date_1.d SIMPLE [] +POSTHOOK: Lineage: date_1.d EXPRESSION [] diff --git ql/src/test/results/clientpositive/date_2.q.out ql/src/test/results/clientpositive/date_2.q.out new file mode 100644 index 0000000..eee4ef1 --- /dev/null +++ ql/src/test/results/clientpositive/date_2.q.out @@ -0,0 +1,345 @@ +PREHOOK: query: drop table if exists date_2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists date_2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table date_2 ( + ORIGIN_CITY_NAME string, + DEST_CITY_NAME string, + FL_DATE date, + ARR_DELAY float, + FL_NUM int +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table date_2 ( + ORIGIN_CITY_NAME string, + DEST_CITY_NAME string, + FL_DATE date, + ARR_DELAY float, + FL_NUM int +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@date_2 +PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/flights_tiny.txt.1' OVERWRITE INTO TABLE date_2 +PREHOOK: type: LOAD +PREHOOK: Output: default@date_2 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/flights_tiny.txt.1' OVERWRITE INTO TABLE date_2 +POSTHOOK: type: LOAD +POSTHOOK: Output: default@date_2 +PREHOOK: query: select fl_date, fl_num from date_2 order by fl_date asc, fl_num desc +PREHOOK: type: QUERY +PREHOOK: Input: default@date_2 +#### A masked pattern was here #### +POSTHOOK: query: select fl_date, fl_num from date_2 order by fl_date asc, fl_num desc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_2 +#### A masked pattern was here #### +2010-10-20 7291 +2010-10-20 3198 +2010-10-20 3014 +2010-10-20 2630 +2010-10-20 1610 +2010-10-20 1599 +2010-10-20 1531 +2010-10-20 1142 +2010-10-20 1064 +2010-10-20 897 +2010-10-20 361 +2010-10-21 7291 +2010-10-21 3198 +2010-10-21 3014 +2010-10-21 2646 +2010-10-21 2630 +2010-10-21 1610 +2010-10-21 1599 +2010-10-21 1531 +2010-10-21 1142 +2010-10-21 1064 +2010-10-21 897 +2010-10-21 361 +2010-10-22 3198 +2010-10-22 3014 +2010-10-22 2646 +2010-10-22 2630 +2010-10-22 1610 +2010-10-22 1599 +2010-10-22 1531 +2010-10-22 1142 +2010-10-22 1064 +2010-10-22 897 +2010-10-22 361 +2010-10-23 7274 +2010-10-23 5917 +2010-10-23 5904 +2010-10-23 5832 +2010-10-23 3171 +2010-10-23 3085 +2010-10-23 2932 +2010-10-23 1805 +2010-10-23 650 +2010-10-23 426 +2010-10-23 384 +2010-10-23 272 +2010-10-24 7282 +2010-10-24 3198 +2010-10-24 3014 +2010-10-24 2646 +2010-10-24 2630 +2010-10-24 2571 +2010-10-24 2254 +2010-10-24 1610 +2010-10-24 1599 +2010-10-24 1531 +2010-10-24 897 +2010-10-24 361 +2010-10-25 7291 +2010-10-25 3198 +2010-10-25 3014 +2010-10-25 2646 +2010-10-25 2630 +2010-10-25 1610 +2010-10-25 1599 +2010-10-25 1531 +2010-10-25 1142 +2010-10-25 1064 +2010-10-25 897 +2010-10-25 361 +2010-10-26 7291 +2010-10-26 3198 +2010-10-26 3014 +2010-10-26 2662 +2010-10-26 2646 +2010-10-26 2630 +2010-10-26 1610 +2010-10-26 1599 +2010-10-26 1531 +2010-10-26 1142 +2010-10-26 1064 +2010-10-26 897 +2010-10-26 361 +2010-10-27 7291 +2010-10-27 3198 +2010-10-27 3014 +2010-10-27 2630 +2010-10-27 1610 +2010-10-27 1599 +2010-10-27 1531 +2010-10-27 1142 +2010-10-27 1064 +2010-10-27 897 +2010-10-27 361 +2010-10-28 7291 +2010-10-28 3198 +2010-10-28 3014 +2010-10-28 2646 +2010-10-28 2630 +2010-10-28 1610 +2010-10-28 1599 +2010-10-28 1531 +2010-10-28 1142 +2010-10-28 1064 +2010-10-28 897 +2010-10-28 361 +2010-10-29 7291 +2010-10-29 3198 +2010-10-29 3014 +2010-10-29 2646 +2010-10-29 2630 +2010-10-29 1610 +2010-10-29 1599 +2010-10-29 1531 +2010-10-29 1142 +2010-10-29 1064 +2010-10-29 897 +2010-10-29 361 +2010-10-30 5917 +2010-10-30 5904 +2010-10-30 3171 +2010-10-30 3085 +2010-10-30 2932 +2010-10-30 2018 +2010-10-30 1805 +2010-10-30 650 +2010-10-30 426 +2010-10-30 384 +2010-10-30 272 +2010-10-31 7282 +2010-10-31 3198 +2010-10-31 2571 +2010-10-31 1610 +2010-10-31 1599 +2010-10-31 1531 +2010-10-31 897 +2010-10-31 361 +PREHOOK: query: select fl_date, fl_num from date_2 order by fl_date desc, fl_num asc +PREHOOK: type: QUERY +PREHOOK: Input: default@date_2 +#### A masked pattern was here #### +POSTHOOK: query: select fl_date, fl_num from date_2 order by fl_date desc, fl_num asc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_2 +#### A masked pattern was here #### +2010-10-31 361 +2010-10-31 897 +2010-10-31 1531 +2010-10-31 1599 +2010-10-31 1610 +2010-10-31 2571 +2010-10-31 3198 +2010-10-31 7282 +2010-10-30 272 +2010-10-30 384 +2010-10-30 426 +2010-10-30 650 +2010-10-30 1805 +2010-10-30 2018 +2010-10-30 2932 +2010-10-30 3085 +2010-10-30 3171 +2010-10-30 5904 +2010-10-30 5917 +2010-10-29 361 +2010-10-29 897 +2010-10-29 1064 +2010-10-29 1142 +2010-10-29 1531 +2010-10-29 1599 +2010-10-29 1610 +2010-10-29 2630 +2010-10-29 2646 +2010-10-29 3014 +2010-10-29 3198 +2010-10-29 7291 +2010-10-28 361 +2010-10-28 897 +2010-10-28 1064 +2010-10-28 1142 +2010-10-28 1531 +2010-10-28 1599 +2010-10-28 1610 +2010-10-28 2630 +2010-10-28 2646 +2010-10-28 3014 +2010-10-28 3198 +2010-10-28 7291 +2010-10-27 361 +2010-10-27 897 +2010-10-27 1064 +2010-10-27 1142 +2010-10-27 1531 +2010-10-27 1599 +2010-10-27 1610 +2010-10-27 2630 +2010-10-27 3014 +2010-10-27 3198 +2010-10-27 7291 +2010-10-26 361 +2010-10-26 897 +2010-10-26 1064 +2010-10-26 1142 +2010-10-26 1531 +2010-10-26 1599 +2010-10-26 1610 +2010-10-26 2630 +2010-10-26 2646 +2010-10-26 2662 +2010-10-26 3014 +2010-10-26 3198 +2010-10-26 7291 +2010-10-25 361 +2010-10-25 897 +2010-10-25 1064 +2010-10-25 1142 +2010-10-25 1531 +2010-10-25 1599 +2010-10-25 1610 +2010-10-25 2630 +2010-10-25 2646 +2010-10-25 3014 +2010-10-25 3198 +2010-10-25 7291 +2010-10-24 361 +2010-10-24 897 +2010-10-24 1531 +2010-10-24 1599 +2010-10-24 1610 +2010-10-24 2254 +2010-10-24 2571 +2010-10-24 2630 +2010-10-24 2646 +2010-10-24 3014 +2010-10-24 3198 +2010-10-24 7282 +2010-10-23 272 +2010-10-23 384 +2010-10-23 426 +2010-10-23 650 +2010-10-23 1805 +2010-10-23 2932 +2010-10-23 3085 +2010-10-23 3171 +2010-10-23 5832 +2010-10-23 5904 +2010-10-23 5917 +2010-10-23 7274 +2010-10-22 361 +2010-10-22 897 +2010-10-22 1064 +2010-10-22 1142 +2010-10-22 1531 +2010-10-22 1599 +2010-10-22 1610 +2010-10-22 2630 +2010-10-22 2646 +2010-10-22 3014 +2010-10-22 3198 +2010-10-21 361 +2010-10-21 897 +2010-10-21 1064 +2010-10-21 1142 +2010-10-21 1531 +2010-10-21 1599 +2010-10-21 1610 +2010-10-21 2630 +2010-10-21 2646 +2010-10-21 3014 +2010-10-21 3198 +2010-10-21 7291 +2010-10-20 361 +2010-10-20 897 +2010-10-20 1064 +2010-10-20 1142 +2010-10-20 1531 +2010-10-20 1599 +2010-10-20 1610 +2010-10-20 2630 +2010-10-20 3014 +2010-10-20 3198 +2010-10-20 7291 +PREHOOK: query: select fl_date, count(*) from date_2 group by fl_date +PREHOOK: type: QUERY +PREHOOK: Input: default@date_2 +#### A masked pattern was here #### +POSTHOOK: query: select fl_date, count(*) from date_2 group by fl_date +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_2 +#### A masked pattern was here #### +2010-10-20 11 +2010-10-21 12 +2010-10-22 11 +2010-10-23 12 +2010-10-24 12 +2010-10-25 12 +2010-10-26 13 +2010-10-27 11 +2010-10-28 12 +2010-10-29 12 +2010-10-30 11 +2010-10-31 8 +PREHOOK: query: drop table date_2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@date_2 +PREHOOK: Output: default@date_2 +POSTHOOK: query: drop table date_2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@date_2 +POSTHOOK: Output: default@date_2 diff --git ql/src/test/results/clientpositive/date_3.q.out ql/src/test/results/clientpositive/date_3.q.out new file mode 100644 index 0000000..230539e --- /dev/null +++ ql/src/test/results/clientpositive/date_3.q.out @@ -0,0 +1,54 @@ +PREHOOK: query: drop table date_3 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table date_3 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table date_3 ( + c1 int +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table date_3 ( + c1 int +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@date_3 +PREHOOK: query: alter table date_3 add columns (c2 date) +PREHOOK: type: ALTERTABLE_ADDCOLS +PREHOOK: Input: default@date_3 +PREHOOK: Output: default@date_3 +POSTHOOK: query: alter table date_3 add columns (c2 date) +POSTHOOK: type: ALTERTABLE_ADDCOLS +POSTHOOK: Input: default@date_3 +POSTHOOK: Output: default@date_3 +PREHOOK: query: insert overwrite table date_3 + select 1, cast(cast('2011-01-01 00:00:00' as timestamp) as date) from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@date_3 +POSTHOOK: query: insert overwrite table date_3 + select 1, cast(cast('2011-01-01 00:00:00' as timestamp) as date) from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@date_3 +POSTHOOK: Lineage: date_3.c1 SIMPLE [] +POSTHOOK: Lineage: date_3.c2 EXPRESSION [] +PREHOOK: query: select * from date_3 +PREHOOK: type: QUERY +PREHOOK: Input: default@date_3 +#### A masked pattern was here #### +POSTHOOK: query: select * from date_3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_3 +#### A masked pattern was here #### +POSTHOOK: Lineage: date_3.c1 SIMPLE [] +POSTHOOK: Lineage: date_3.c2 EXPRESSION [] +1 2011-01-01 +PREHOOK: query: drop table date_3 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@date_3 +PREHOOK: Output: default@date_3 +POSTHOOK: query: drop table date_3 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@date_3 +POSTHOOK: Output: default@date_3 +POSTHOOK: Lineage: date_3.c1 SIMPLE [] +POSTHOOK: Lineage: date_3.c2 EXPRESSION [] diff --git ql/src/test/results/clientpositive/date_4.q.out ql/src/test/results/clientpositive/date_4.q.out new file mode 100644 index 0000000..a97d522 --- /dev/null +++ ql/src/test/results/clientpositive/date_4.q.out @@ -0,0 +1,49 @@ +PREHOOK: query: drop table date_4 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table date_4 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table date_4 (d date) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table date_4 (d date) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@date_4 +PREHOOK: query: alter table date_4 set serde 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +PREHOOK: type: ALTERTABLE_SERIALIZER +PREHOOK: Input: default@date_4 +PREHOOK: Output: default@date_4 +POSTHOOK: query: alter table date_4 set serde 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +POSTHOOK: type: ALTERTABLE_SERIALIZER +POSTHOOK: Input: default@date_4 +POSTHOOK: Output: default@date_4 +PREHOOK: query: -- Test date literal syntax +insert overwrite table date_4 + select date '2011-01-01' from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@date_4 +POSTHOOK: query: -- Test date literal syntax +insert overwrite table date_4 + select date '2011-01-01' from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@date_4 +POSTHOOK: Lineage: date_4.d SIMPLE [] +PREHOOK: query: select d, date '2011-01-01' from date_4 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@date_4 +#### A masked pattern was here #### +POSTHOOK: query: select d, date '2011-01-01' from date_4 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_4 +#### A masked pattern was here #### +POSTHOOK: Lineage: date_4.d SIMPLE [] +2011-01-01 2011-01-01 +PREHOOK: query: drop table date_4 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@date_4 +PREHOOK: Output: default@date_4 +POSTHOOK: query: drop table date_4 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@date_4 +POSTHOOK: Output: default@date_4 +POSTHOOK: Lineage: date_4.d SIMPLE [] diff --git ql/src/test/results/clientpositive/date_comparison.q.out ql/src/test/results/clientpositive/date_comparison.q.out new file mode 100644 index 0000000..73e8560 --- /dev/null +++ ql/src/test/results/clientpositive/date_comparison.q.out @@ -0,0 +1,136 @@ +PREHOOK: query: -- Comparisons against same value +select cast('2011-05-06' as date) > + cast('2011-05-06' as date) from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: -- Comparisons against same value +select cast('2011-05-06' as date) > + cast('2011-05-06' as date) from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +false +PREHOOK: query: select cast('2011-05-06' as date) < + cast('2011-05-06' as date) from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select cast('2011-05-06' as date) < + cast('2011-05-06' as date) from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +false +PREHOOK: query: select cast('2011-05-06' as date) = + cast('2011-05-06' as date) from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select cast('2011-05-06' as date) = + cast('2011-05-06' as date) from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +true +PREHOOK: query: select cast('2011-05-06' as date) <> + cast('2011-05-06' as date) from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select cast('2011-05-06' as date) <> + cast('2011-05-06' as date) from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +false +PREHOOK: query: select cast('2011-05-06' as date) >= + cast('2011-05-06' as date) from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select cast('2011-05-06' as date) >= + cast('2011-05-06' as date) from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +true +PREHOOK: query: select cast('2011-05-06' as date) <= + cast('2011-05-06' as date) from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select cast('2011-05-06' as date) <= + cast('2011-05-06' as date) from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +true +PREHOOK: query: -- Now try with differing values +select cast('2011-05-05' as date) > + cast('2011-05-06' as date) from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: -- Now try with differing values +select cast('2011-05-05' as date) > + cast('2011-05-06' as date) from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +false +PREHOOK: query: select cast('2011-05-05' as date) < + cast('2011-05-06' as date) from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select cast('2011-05-05' as date) < + cast('2011-05-06' as date) from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +true +PREHOOK: query: select cast('2011-05-05' as date) = + cast('2011-05-06' as date) from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select cast('2011-05-05' as date) = + cast('2011-05-06' as date) from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +false +PREHOOK: query: select cast('2011-05-05' as date) <> + cast('2011-05-06' as date) from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select cast('2011-05-05' as date) <> + cast('2011-05-06' as date) from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +true +PREHOOK: query: select cast('2011-05-05' as date) >= + cast('2011-05-06' as date) from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select cast('2011-05-05' as date) >= + cast('2011-05-06' as date) from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +false +PREHOOK: query: select cast('2011-05-05' as date) <= + cast('2011-05-06' as date) from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select cast('2011-05-05' as date) <= + cast('2011-05-06' as date) from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +true diff --git ql/src/test/results/clientpositive/date_join1.q.out ql/src/test/results/clientpositive/date_join1.q.out new file mode 100644 index 0000000..0ba749c --- /dev/null +++ ql/src/test/results/clientpositive/date_join1.q.out @@ -0,0 +1,73 @@ +PREHOOK: query: drop table date_join1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table date_join1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table date_join1 ( + ORIGIN_CITY_NAME string, + DEST_CITY_NAME string, + FL_DATE date, + ARR_DELAY float, + FL_NUM int +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table date_join1 ( + ORIGIN_CITY_NAME string, + DEST_CITY_NAME string, + FL_DATE date, + ARR_DELAY float, + FL_NUM int +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@date_join1 +PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/flights_join.txt' OVERWRITE INTO TABLE date_join1 +PREHOOK: type: LOAD +PREHOOK: Output: default@date_join1 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/flights_join.txt' OVERWRITE INTO TABLE date_join1 +POSTHOOK: type: LOAD +POSTHOOK: Output: default@date_join1 +PREHOOK: query: -- Note that there are 2 rows with date 2000-11-28, so we should expect 4 rows with that date in the join results +select t1.fl_num, t1.fl_date, t2.fl_num, t2.fl_date + from date_join1 t1 + join date_join1 t2 + on (t1.fl_date = t2.fl_date) +PREHOOK: type: QUERY +PREHOOK: Input: default@date_join1 +#### A masked pattern was here #### +POSTHOOK: query: -- Note that there are 2 rows with date 2000-11-28, so we should expect 4 rows with that date in the join results +select t1.fl_num, t1.fl_date, t2.fl_num, t2.fl_date + from date_join1 t1 + join date_join1 t2 + on (t1.fl_date = t2.fl_date) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_join1 +#### A masked pattern was here #### +1064 2000-11-20 1064 2000-11-20 +1142 2000-11-21 1142 2000-11-21 +1599 2000-11-22 1599 2000-11-22 +361 2000-11-23 361 2000-11-23 +897 2000-11-24 897 2000-11-24 +1531 2000-11-25 1531 2000-11-25 +1610 2000-11-26 1610 2000-11-26 +3198 2000-11-27 3198 2000-11-27 +1142 2000-11-28 1064 2000-11-28 +1142 2000-11-28 1142 2000-11-28 +1064 2000-11-28 1064 2000-11-28 +1064 2000-11-28 1142 2000-11-28 +1064 2010-10-20 1064 2010-10-20 +1142 2010-10-21 1142 2010-10-21 +1599 2010-10-22 1599 2010-10-22 +361 2010-10-23 361 2010-10-23 +897 2010-10-24 897 2010-10-24 +1531 2010-10-25 1531 2010-10-25 +1610 2010-10-26 1610 2010-10-26 +3198 2010-10-27 3198 2010-10-27 +1064 2010-10-28 1064 2010-10-28 +1142 2010-10-29 1142 2010-10-29 +PREHOOK: query: drop table date_join1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@date_join1 +PREHOOK: Output: default@date_join1 +POSTHOOK: query: drop table date_join1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@date_join1 +POSTHOOK: Output: default@date_join1 diff --git ql/src/test/results/clientpositive/date_serde.q.out ql/src/test/results/clientpositive/date_serde.q.out new file mode 100644 index 0000000..1624e5e --- /dev/null +++ ql/src/test/results/clientpositive/date_serde.q.out @@ -0,0 +1,731 @@ +PREHOOK: query: drop table if exists date_serde_regex +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists date_serde_regex +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table date_serde_lb +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table date_serde_lb +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table date_serde_ls +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table date_serde_ls +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table date_serde_c +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table date_serde_c +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table date_serde_lbc +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table date_serde_lbc +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table date_serde_orc +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table date_serde_orc +POSTHOOK: type: DROPTABLE +PREHOOK: query: -- +-- RegexSerDe +-- +create table date_serde_regex ( + ORIGIN_CITY_NAME string, + DEST_CITY_NAME string, + FL_DATE date, + ARR_DELAY float, + FL_NUM int +) +row format serde 'org.apache.hadoop.hive.serde2.RegexSerDe' +with serdeproperties ( + "input.regex" = "([^]*)([^]*)([^]*)([^]*)([0-9]*)" +) +stored as textfile +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- +-- RegexSerDe +-- +create table date_serde_regex ( + ORIGIN_CITY_NAME string, + DEST_CITY_NAME string, + FL_DATE date, + ARR_DELAY float, + FL_NUM int +) +row format serde 'org.apache.hadoop.hive.serde2.RegexSerDe' +with serdeproperties ( + "input.regex" = "([^]*)([^]*)([^]*)([^]*)([0-9]*)" +) +stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@date_serde_regex +PREHOOK: query: load data local inpath '../data/files/flights_tiny.txt.1' overwrite into table date_serde_regex +PREHOOK: type: LOAD +PREHOOK: Output: default@date_serde_regex +POSTHOOK: query: load data local inpath '../data/files/flights_tiny.txt.1' overwrite into table date_serde_regex +POSTHOOK: type: LOAD +POSTHOOK: Output: default@date_serde_regex +PREHOOK: query: select * from date_serde_regex +PREHOOK: type: QUERY +PREHOOK: Input: default@date_serde_regex +#### A masked pattern was here #### +POSTHOOK: query: select * from date_serde_regex +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_serde_regex +#### A masked pattern was here #### +Baltimore New York 2010-10-20 -30.0 1064 +Baltimore New York 2010-10-20 23.0 1142 +Baltimore New York 2010-10-20 6.0 1599 +Chicago New York 2010-10-20 42.0 361 +Chicago New York 2010-10-20 24.0 897 +Chicago New York 2010-10-20 15.0 1531 +Chicago New York 2010-10-20 -6.0 1610 +Chicago New York 2010-10-20 -2.0 3198 +Baltimore New York 2010-10-21 17.0 1064 +Baltimore New York 2010-10-21 105.0 1142 +Baltimore New York 2010-10-21 28.0 1599 +Chicago New York 2010-10-21 142.0 361 +Chicago New York 2010-10-21 77.0 897 +Chicago New York 2010-10-21 53.0 1531 +Chicago New York 2010-10-21 -5.0 1610 +Chicago New York 2010-10-21 51.0 3198 +Baltimore New York 2010-10-22 -12.0 1064 +Baltimore New York 2010-10-22 54.0 1142 +Baltimore New York 2010-10-22 18.0 1599 +Chicago New York 2010-10-22 2.0 361 +Chicago New York 2010-10-22 24.0 897 +Chicago New York 2010-10-22 16.0 1531 +Chicago New York 2010-10-22 -6.0 1610 +Chicago New York 2010-10-22 -11.0 3198 +Baltimore New York 2010-10-23 18.0 272 +Baltimore New York 2010-10-23 -10.0 1805 +Baltimore New York 2010-10-23 6.0 3171 +Chicago New York 2010-10-23 3.0 384 +Chicago New York 2010-10-23 32.0 426 +Chicago New York 2010-10-23 1.0 650 +Chicago New York 2010-10-23 11.0 3085 +Baltimore New York 2010-10-24 12.0 1599 +Baltimore New York 2010-10-24 20.0 2571 +Chicago New York 2010-10-24 10.0 361 +Chicago New York 2010-10-24 113.0 897 +Chicago New York 2010-10-24 -5.0 1531 +Chicago New York 2010-10-24 -17.0 1610 +Chicago New York 2010-10-24 -3.0 3198 +Baltimore New York 2010-10-25 -25.0 1064 +Baltimore New York 2010-10-25 92.0 1142 +Baltimore New York 2010-10-25 106.0 1599 +Chicago New York 2010-10-25 31.0 361 +Chicago New York 2010-10-25 -1.0 897 +Chicago New York 2010-10-25 43.0 1531 +Chicago New York 2010-10-25 6.0 1610 +Chicago New York 2010-10-25 -16.0 3198 +Baltimore New York 2010-10-26 -22.0 1064 +Baltimore New York 2010-10-26 123.0 1142 +Baltimore New York 2010-10-26 90.0 1599 +Chicago New York 2010-10-26 12.0 361 +Chicago New York 2010-10-26 0.0 897 +Chicago New York 2010-10-26 29.0 1531 +Chicago New York 2010-10-26 -17.0 1610 +Chicago New York 2010-10-26 6.0 3198 +Baltimore New York 2010-10-27 -18.0 1064 +Baltimore New York 2010-10-27 49.0 1142 +Baltimore New York 2010-10-27 92.0 1599 +Chicago New York 2010-10-27 148.0 361 +Chicago New York 2010-10-27 -11.0 897 +Chicago New York 2010-10-27 70.0 1531 +Chicago New York 2010-10-27 8.0 1610 +Chicago New York 2010-10-27 21.0 3198 +Baltimore New York 2010-10-28 -4.0 1064 +Baltimore New York 2010-10-28 -14.0 1142 +Baltimore New York 2010-10-28 -14.0 1599 +Chicago New York 2010-10-28 2.0 361 +Chicago New York 2010-10-28 2.0 897 +Chicago New York 2010-10-28 -11.0 1531 +Chicago New York 2010-10-28 3.0 1610 +Chicago New York 2010-10-28 -18.0 3198 +Baltimore New York 2010-10-29 -24.0 1064 +Baltimore New York 2010-10-29 21.0 1142 +Baltimore New York 2010-10-29 -2.0 1599 +Chicago New York 2010-10-29 -12.0 361 +Chicago New York 2010-10-29 -11.0 897 +Chicago New York 2010-10-29 15.0 1531 +Chicago New York 2010-10-29 -18.0 1610 +Chicago New York 2010-10-29 -4.0 3198 +Baltimore New York 2010-10-30 14.0 272 +Baltimore New York 2010-10-30 -1.0 1805 +Baltimore New York 2010-10-30 5.0 3171 +Chicago New York 2010-10-30 -6.0 384 +Chicago New York 2010-10-30 -10.0 426 +Chicago New York 2010-10-30 -5.0 650 +Chicago New York 2010-10-30 -5.0 3085 +Baltimore New York 2010-10-31 -1.0 1599 +Baltimore New York 2010-10-31 -14.0 2571 +Chicago New York 2010-10-31 -25.0 361 +Chicago New York 2010-10-31 -18.0 897 +Chicago New York 2010-10-31 -4.0 1531 +Chicago New York 2010-10-31 -22.0 1610 +Chicago New York 2010-10-31 -15.0 3198 +Cleveland New York 2010-10-30 -23.0 2018 +Cleveland New York 2010-10-30 -12.0 2932 +Cleveland New York 2010-10-29 -4.0 2630 +Cleveland New York 2010-10-29 -19.0 2646 +Cleveland New York 2010-10-29 -12.0 3014 +Cleveland New York 2010-10-28 3.0 2630 +Cleveland New York 2010-10-28 -6.0 2646 +Cleveland New York 2010-10-28 1.0 3014 +Cleveland New York 2010-10-27 16.0 2630 +Cleveland New York 2010-10-27 27.0 3014 +Cleveland New York 2010-10-26 4.0 2630 +Cleveland New York 2010-10-26 -27.0 2646 +Cleveland New York 2010-10-26 -11.0 2662 +Cleveland New York 2010-10-26 13.0 3014 +Cleveland New York 2010-10-25 -4.0 2630 +Cleveland New York 2010-10-25 81.0 2646 +Cleveland New York 2010-10-25 42.0 3014 +Cleveland New York 2010-10-24 5.0 2254 +Cleveland New York 2010-10-24 -11.0 2630 +Cleveland New York 2010-10-24 -20.0 2646 +Cleveland New York 2010-10-24 -9.0 3014 +Cleveland New York 2010-10-23 -21.0 2932 +Cleveland New York 2010-10-22 1.0 2630 +Cleveland New York 2010-10-22 -25.0 2646 +Cleveland New York 2010-10-22 -3.0 3014 +Cleveland New York 2010-10-21 3.0 2630 +Cleveland New York 2010-10-21 29.0 2646 +Cleveland New York 2010-10-21 72.0 3014 +Cleveland New York 2010-10-20 -8.0 2630 +Cleveland New York 2010-10-20 -15.0 3014 +Washington New York 2010-10-23 -25.0 5832 +Washington New York 2010-10-23 -21.0 5904 +Washington New York 2010-10-23 -18.0 5917 +Washington New York 2010-10-30 -27.0 5904 +Washington New York 2010-10-30 -16.0 5917 +Washington New York 2010-10-20 -2.0 7291 +Washington New York 2010-10-21 22.0 7291 +Washington New York 2010-10-23 -16.0 7274 +Washington New York 2010-10-24 -26.0 7282 +Washington New York 2010-10-25 9.0 7291 +Washington New York 2010-10-26 4.0 7291 +Washington New York 2010-10-27 26.0 7291 +Washington New York 2010-10-28 45.0 7291 +Washington New York 2010-10-29 1.0 7291 +Washington New York 2010-10-31 -18.0 7282 +PREHOOK: query: select fl_date, count(*) from date_serde_regex group by fl_date +PREHOOK: type: QUERY +PREHOOK: Input: default@date_serde_regex +#### A masked pattern was here #### +POSTHOOK: query: select fl_date, count(*) from date_serde_regex group by fl_date +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_serde_regex +#### A masked pattern was here #### +2010-10-20 11 +2010-10-21 12 +2010-10-22 11 +2010-10-23 12 +2010-10-24 12 +2010-10-25 12 +2010-10-26 13 +2010-10-27 11 +2010-10-28 12 +2010-10-29 12 +2010-10-30 11 +2010-10-31 8 +PREHOOK: query: -- +-- LazyBinary +-- +create table date_serde_lb ( + c1 date, + c2 int +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- +-- LazyBinary +-- +create table date_serde_lb ( + c1 date, + c2 int +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@date_serde_lb +PREHOOK: query: alter table date_serde_lb set serde 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe' +PREHOOK: type: ALTERTABLE_SERIALIZER +PREHOOK: Input: default@date_serde_lb +PREHOOK: Output: default@date_serde_lb +POSTHOOK: query: alter table date_serde_lb set serde 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe' +POSTHOOK: type: ALTERTABLE_SERIALIZER +POSTHOOK: Input: default@date_serde_lb +POSTHOOK: Output: default@date_serde_lb +PREHOOK: query: insert overwrite table date_serde_lb + select fl_date, fl_num from date_serde_regex limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@date_serde_regex +PREHOOK: Output: default@date_serde_lb +POSTHOOK: query: insert overwrite table date_serde_lb + select fl_date, fl_num from date_serde_regex limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_serde_regex +POSTHOOK: Output: default@date_serde_lb +POSTHOOK: Lineage: date_serde_lb.c1 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_date, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c2 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_num, type:int, comment:from deserializer), ] +PREHOOK: query: select * from date_serde_lb +PREHOOK: type: QUERY +PREHOOK: Input: default@date_serde_lb +#### A masked pattern was here #### +POSTHOOK: query: select * from date_serde_lb +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_serde_lb +#### A masked pattern was here #### +POSTHOOK: Lineage: date_serde_lb.c1 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_date, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c2 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_num, type:int, comment:from deserializer), ] +2010-10-20 1064 +PREHOOK: query: select c1, sum(c2) from date_serde_lb group by c1 +PREHOOK: type: QUERY +PREHOOK: Input: default@date_serde_lb +#### A masked pattern was here #### +POSTHOOK: query: select c1, sum(c2) from date_serde_lb group by c1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_serde_lb +#### A masked pattern was here #### +POSTHOOK: Lineage: date_serde_lb.c1 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_date, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c2 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_num, type:int, comment:from deserializer), ] +2010-10-20 1064 +PREHOOK: query: -- +-- LazySimple +-- +create table date_serde_ls ( + c1 date, + c2 int +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- +-- LazySimple +-- +create table date_serde_ls ( + c1 date, + c2 int +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@date_serde_ls +POSTHOOK: Lineage: date_serde_lb.c1 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_date, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c2 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_num, type:int, comment:from deserializer), ] +PREHOOK: query: alter table date_serde_ls set serde 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +PREHOOK: type: ALTERTABLE_SERIALIZER +PREHOOK: Input: default@date_serde_ls +PREHOOK: Output: default@date_serde_ls +POSTHOOK: query: alter table date_serde_ls set serde 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +POSTHOOK: type: ALTERTABLE_SERIALIZER +POSTHOOK: Input: default@date_serde_ls +POSTHOOK: Output: default@date_serde_ls +POSTHOOK: Lineage: date_serde_lb.c1 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_date, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c2 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_num, type:int, comment:from deserializer), ] +PREHOOK: query: insert overwrite table date_serde_ls + select c1, c2 from date_serde_lb limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@date_serde_lb +PREHOOK: Output: default@date_serde_ls +POSTHOOK: query: insert overwrite table date_serde_ls + select c1, c2 from date_serde_lb limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_serde_lb +POSTHOOK: Output: default@date_serde_ls +POSTHOOK: Lineage: date_serde_lb.c1 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_date, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c2 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_num, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c1 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c2 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c2, type:int, comment:from deserializer), ] +PREHOOK: query: select * from date_serde_ls +PREHOOK: type: QUERY +PREHOOK: Input: default@date_serde_ls +#### A masked pattern was here #### +POSTHOOK: query: select * from date_serde_ls +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_serde_ls +#### A masked pattern was here #### +POSTHOOK: Lineage: date_serde_lb.c1 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_date, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c2 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_num, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c1 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c2 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c2, type:int, comment:from deserializer), ] +2010-10-20 1064 +PREHOOK: query: select c1, sum(c2) from date_serde_ls group by c1 +PREHOOK: type: QUERY +PREHOOK: Input: default@date_serde_ls +#### A masked pattern was here #### +POSTHOOK: query: select c1, sum(c2) from date_serde_ls group by c1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_serde_ls +#### A masked pattern was here #### +POSTHOOK: Lineage: date_serde_lb.c1 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_date, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c2 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_num, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c1 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c2 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c2, type:int, comment:from deserializer), ] +2010-10-20 1064 +PREHOOK: query: -- +-- Columnar +-- +create table date_serde_c ( + c1 date, + c2 int +) stored as rcfile +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- +-- Columnar +-- +create table date_serde_c ( + c1 date, + c2 int +) stored as rcfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@date_serde_c +POSTHOOK: Lineage: date_serde_lb.c1 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_date, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c2 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_num, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c1 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c2 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c2, type:int, comment:from deserializer), ] +PREHOOK: query: alter table date_serde_c set serde 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe' +PREHOOK: type: ALTERTABLE_SERIALIZER +PREHOOK: Input: default@date_serde_c +PREHOOK: Output: default@date_serde_c +POSTHOOK: query: alter table date_serde_c set serde 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe' +POSTHOOK: type: ALTERTABLE_SERIALIZER +POSTHOOK: Input: default@date_serde_c +POSTHOOK: Output: default@date_serde_c +POSTHOOK: Lineage: date_serde_lb.c1 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_date, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c2 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_num, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c1 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c2 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c2, type:int, comment:from deserializer), ] +PREHOOK: query: insert overwrite table date_serde_c + select c1, c2 from date_serde_ls limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@date_serde_ls +PREHOOK: Output: default@date_serde_c +POSTHOOK: query: insert overwrite table date_serde_c + select c1, c2 from date_serde_ls limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_serde_ls +POSTHOOK: Output: default@date_serde_c +POSTHOOK: Lineage: date_serde_c.c1 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_c.c2 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c1 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_date, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c2 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_num, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c1 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c2 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c2, type:int, comment:from deserializer), ] +PREHOOK: query: select * from date_serde_c +PREHOOK: type: QUERY +PREHOOK: Input: default@date_serde_c +#### A masked pattern was here #### +POSTHOOK: query: select * from date_serde_c +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_serde_c +#### A masked pattern was here #### +POSTHOOK: Lineage: date_serde_c.c1 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_c.c2 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c1 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_date, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c2 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_num, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c1 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c2 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c2, type:int, comment:from deserializer), ] +2010-10-20 1064 +PREHOOK: query: select c1, sum(c2) from date_serde_c group by c1 +PREHOOK: type: QUERY +PREHOOK: Input: default@date_serde_c +#### A masked pattern was here #### +POSTHOOK: query: select c1, sum(c2) from date_serde_c group by c1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_serde_c +#### A masked pattern was here #### +POSTHOOK: Lineage: date_serde_c.c1 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_c.c2 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c1 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_date, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c2 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_num, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c1 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c2 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c2, type:int, comment:from deserializer), ] +2010-10-20 1064 +PREHOOK: query: -- +-- LazyBinaryColumnar +-- +create table date_serde_lbc ( + c1 date, + c2 int +) stored as rcfile +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- +-- LazyBinaryColumnar +-- +create table date_serde_lbc ( + c1 date, + c2 int +) stored as rcfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@date_serde_lbc +POSTHOOK: Lineage: date_serde_c.c1 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_c.c2 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c1 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_date, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c2 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_num, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c1 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c2 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c2, type:int, comment:from deserializer), ] +PREHOOK: query: alter table date_serde_lbc set serde 'org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe' +PREHOOK: type: ALTERTABLE_SERIALIZER +PREHOOK: Input: default@date_serde_lbc +PREHOOK: Output: default@date_serde_lbc +POSTHOOK: query: alter table date_serde_lbc set serde 'org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe' +POSTHOOK: type: ALTERTABLE_SERIALIZER +POSTHOOK: Input: default@date_serde_lbc +POSTHOOK: Output: default@date_serde_lbc +POSTHOOK: Lineage: date_serde_c.c1 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_c.c2 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c1 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_date, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c2 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_num, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c1 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c2 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c2, type:int, comment:from deserializer), ] +PREHOOK: query: insert overwrite table date_serde_lbc + select c1, c2 from date_serde_c limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@date_serde_c +PREHOOK: Output: default@date_serde_lbc +POSTHOOK: query: insert overwrite table date_serde_lbc + select c1, c2 from date_serde_c limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_serde_c +POSTHOOK: Output: default@date_serde_lbc +POSTHOOK: Lineage: date_serde_c.c1 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_c.c2 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c1 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_date, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c2 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_num, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lbc.c1 SIMPLE [(date_serde_c)date_serde_c.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lbc.c2 SIMPLE [(date_serde_c)date_serde_c.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c1 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c2 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c2, type:int, comment:from deserializer), ] +PREHOOK: query: select * from date_serde_lbc +PREHOOK: type: QUERY +PREHOOK: Input: default@date_serde_lbc +#### A masked pattern was here #### +POSTHOOK: query: select * from date_serde_lbc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_serde_lbc +#### A masked pattern was here #### +POSTHOOK: Lineage: date_serde_c.c1 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_c.c2 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c1 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_date, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c2 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_num, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lbc.c1 SIMPLE [(date_serde_c)date_serde_c.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lbc.c2 SIMPLE [(date_serde_c)date_serde_c.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c1 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c2 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c2, type:int, comment:from deserializer), ] +2010-10-20 1064 +PREHOOK: query: select c1, sum(c2) from date_serde_lbc group by c1 +PREHOOK: type: QUERY +PREHOOK: Input: default@date_serde_lbc +#### A masked pattern was here #### +POSTHOOK: query: select c1, sum(c2) from date_serde_lbc group by c1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_serde_lbc +#### A masked pattern was here #### +POSTHOOK: Lineage: date_serde_c.c1 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_c.c2 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c1 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_date, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c2 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_num, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lbc.c1 SIMPLE [(date_serde_c)date_serde_c.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lbc.c2 SIMPLE [(date_serde_c)date_serde_c.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c1 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c2 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c2, type:int, comment:from deserializer), ] +2010-10-20 1064 +PREHOOK: query: -- +-- ORC +-- +create table date_serde_orc ( + c1 date, + c2 int +) stored as orc +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- +-- ORC +-- +create table date_serde_orc ( + c1 date, + c2 int +) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@date_serde_orc +POSTHOOK: Lineage: date_serde_c.c1 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_c.c2 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c1 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_date, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c2 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_num, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lbc.c1 SIMPLE [(date_serde_c)date_serde_c.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lbc.c2 SIMPLE [(date_serde_c)date_serde_c.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c1 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c2 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c2, type:int, comment:from deserializer), ] +PREHOOK: query: alter table date_serde_orc set serde 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' +PREHOOK: type: ALTERTABLE_SERIALIZER +PREHOOK: Input: default@date_serde_orc +PREHOOK: Output: default@date_serde_orc +POSTHOOK: query: alter table date_serde_orc set serde 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' +POSTHOOK: type: ALTERTABLE_SERIALIZER +POSTHOOK: Input: default@date_serde_orc +POSTHOOK: Output: default@date_serde_orc +POSTHOOK: Lineage: date_serde_c.c1 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_c.c2 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c1 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_date, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c2 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_num, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lbc.c1 SIMPLE [(date_serde_c)date_serde_c.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lbc.c2 SIMPLE [(date_serde_c)date_serde_c.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c1 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c2 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c2, type:int, comment:from deserializer), ] +PREHOOK: query: insert overwrite table date_serde_orc + select c1, c2 from date_serde_lbc limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@date_serde_lbc +PREHOOK: Output: default@date_serde_orc +POSTHOOK: query: insert overwrite table date_serde_orc + select c1, c2 from date_serde_lbc limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_serde_lbc +POSTHOOK: Output: default@date_serde_orc +POSTHOOK: Lineage: date_serde_c.c1 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_c.c2 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c1 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_date, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c2 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_num, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lbc.c1 SIMPLE [(date_serde_c)date_serde_c.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lbc.c2 SIMPLE [(date_serde_c)date_serde_c.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c1 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c2 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_orc.c1 SIMPLE [(date_serde_lbc)date_serde_lbc.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_orc.c2 SIMPLE [(date_serde_lbc)date_serde_lbc.FieldSchema(name:c2, type:int, comment:from deserializer), ] +PREHOOK: query: select * from date_serde_orc +PREHOOK: type: QUERY +PREHOOK: Input: default@date_serde_orc +#### A masked pattern was here #### +POSTHOOK: query: select * from date_serde_orc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_serde_orc +#### A masked pattern was here #### +POSTHOOK: Lineage: date_serde_c.c1 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_c.c2 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c1 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_date, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c2 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_num, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lbc.c1 SIMPLE [(date_serde_c)date_serde_c.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lbc.c2 SIMPLE [(date_serde_c)date_serde_c.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c1 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c2 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_orc.c1 SIMPLE [(date_serde_lbc)date_serde_lbc.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_orc.c2 SIMPLE [(date_serde_lbc)date_serde_lbc.FieldSchema(name:c2, type:int, comment:from deserializer), ] +2010-10-20 1064 +PREHOOK: query: select c1, sum(c2) from date_serde_orc group by c1 +PREHOOK: type: QUERY +PREHOOK: Input: default@date_serde_orc +#### A masked pattern was here #### +POSTHOOK: query: select c1, sum(c2) from date_serde_orc group by c1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_serde_orc +#### A masked pattern was here #### +POSTHOOK: Lineage: date_serde_c.c1 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_c.c2 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c1 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_date, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c2 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_num, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lbc.c1 SIMPLE [(date_serde_c)date_serde_c.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lbc.c2 SIMPLE [(date_serde_c)date_serde_c.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c1 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c2 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_orc.c1 SIMPLE [(date_serde_lbc)date_serde_lbc.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_orc.c2 SIMPLE [(date_serde_lbc)date_serde_lbc.FieldSchema(name:c2, type:int, comment:from deserializer), ] +2010-10-20 1064 +PREHOOK: query: drop table date_serde_regex +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@date_serde_regex +PREHOOK: Output: default@date_serde_regex +POSTHOOK: query: drop table date_serde_regex +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@date_serde_regex +POSTHOOK: Output: default@date_serde_regex +POSTHOOK: Lineage: date_serde_c.c1 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_c.c2 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c1 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_date, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c2 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_num, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lbc.c1 SIMPLE [(date_serde_c)date_serde_c.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lbc.c2 SIMPLE [(date_serde_c)date_serde_c.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c1 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c2 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_orc.c1 SIMPLE [(date_serde_lbc)date_serde_lbc.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_orc.c2 SIMPLE [(date_serde_lbc)date_serde_lbc.FieldSchema(name:c2, type:int, comment:from deserializer), ] +PREHOOK: query: drop table date_serde_lb +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@date_serde_lb +PREHOOK: Output: default@date_serde_lb +POSTHOOK: query: drop table date_serde_lb +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@date_serde_lb +POSTHOOK: Output: default@date_serde_lb +POSTHOOK: Lineage: date_serde_c.c1 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_c.c2 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c1 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_date, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c2 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_num, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lbc.c1 SIMPLE [(date_serde_c)date_serde_c.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lbc.c2 SIMPLE [(date_serde_c)date_serde_c.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c1 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c2 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_orc.c1 SIMPLE [(date_serde_lbc)date_serde_lbc.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_orc.c2 SIMPLE [(date_serde_lbc)date_serde_lbc.FieldSchema(name:c2, type:int, comment:from deserializer), ] +PREHOOK: query: drop table date_serde_ls +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@date_serde_ls +PREHOOK: Output: default@date_serde_ls +POSTHOOK: query: drop table date_serde_ls +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@date_serde_ls +POSTHOOK: Output: default@date_serde_ls +POSTHOOK: Lineage: date_serde_c.c1 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_c.c2 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c1 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_date, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c2 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_num, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lbc.c1 SIMPLE [(date_serde_c)date_serde_c.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lbc.c2 SIMPLE [(date_serde_c)date_serde_c.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c1 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c2 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_orc.c1 SIMPLE [(date_serde_lbc)date_serde_lbc.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_orc.c2 SIMPLE [(date_serde_lbc)date_serde_lbc.FieldSchema(name:c2, type:int, comment:from deserializer), ] +PREHOOK: query: drop table date_serde_c +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@date_serde_c +PREHOOK: Output: default@date_serde_c +POSTHOOK: query: drop table date_serde_c +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@date_serde_c +POSTHOOK: Output: default@date_serde_c +POSTHOOK: Lineage: date_serde_c.c1 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_c.c2 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c1 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_date, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c2 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_num, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lbc.c1 SIMPLE [(date_serde_c)date_serde_c.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lbc.c2 SIMPLE [(date_serde_c)date_serde_c.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c1 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c2 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_orc.c1 SIMPLE [(date_serde_lbc)date_serde_lbc.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_orc.c2 SIMPLE [(date_serde_lbc)date_serde_lbc.FieldSchema(name:c2, type:int, comment:from deserializer), ] +PREHOOK: query: drop table date_serde_lbc +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@date_serde_lbc +PREHOOK: Output: default@date_serde_lbc +POSTHOOK: query: drop table date_serde_lbc +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@date_serde_lbc +POSTHOOK: Output: default@date_serde_lbc +POSTHOOK: Lineage: date_serde_c.c1 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_c.c2 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c1 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_date, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c2 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_num, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lbc.c1 SIMPLE [(date_serde_c)date_serde_c.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lbc.c2 SIMPLE [(date_serde_c)date_serde_c.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c1 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c2 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_orc.c1 SIMPLE [(date_serde_lbc)date_serde_lbc.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_orc.c2 SIMPLE [(date_serde_lbc)date_serde_lbc.FieldSchema(name:c2, type:int, comment:from deserializer), ] +PREHOOK: query: drop table date_serde_orc +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@date_serde_orc +PREHOOK: Output: default@date_serde_orc +POSTHOOK: query: drop table date_serde_orc +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@date_serde_orc +POSTHOOK: Output: default@date_serde_orc +POSTHOOK: Lineage: date_serde_c.c1 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_c.c2 SIMPLE [(date_serde_ls)date_serde_ls.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c1 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_date, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lb.c2 SIMPLE [(date_serde_regex)date_serde_regex.FieldSchema(name:fl_num, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lbc.c1 SIMPLE [(date_serde_c)date_serde_c.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_lbc.c2 SIMPLE [(date_serde_c)date_serde_c.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c1 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_ls.c2 SIMPLE [(date_serde_lb)date_serde_lb.FieldSchema(name:c2, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_orc.c1 SIMPLE [(date_serde_lbc)date_serde_lbc.FieldSchema(name:c1, type:date, comment:from deserializer), ] +POSTHOOK: Lineage: date_serde_orc.c2 SIMPLE [(date_serde_lbc)date_serde_lbc.FieldSchema(name:c2, type:int, comment:from deserializer), ] diff --git ql/src/test/results/clientpositive/date_udf.q.out ql/src/test/results/clientpositive/date_udf.q.out new file mode 100644 index 0000000..33af984 --- /dev/null +++ ql/src/test/results/clientpositive/date_udf.q.out @@ -0,0 +1,259 @@ +PREHOOK: query: drop table date_udf +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table date_udf +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table date_udf_string +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table date_udf_string +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table date_udf_flight +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table date_udf_flight +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table date_udf (d date) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table date_udf (d date) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@date_udf +PREHOOK: query: create table date_udf_string (d string) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table date_udf_string (d string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@date_udf_string +PREHOOK: query: from src + insert overwrite table date_udf + select '2011-05-06' limit 1 + insert overwrite table date_udf_string + select '2011-05-06' limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@date_udf +PREHOOK: Output: default@date_udf_string +POSTHOOK: query: from src + insert overwrite table date_udf + select '2011-05-06' limit 1 + insert overwrite table date_udf_string + select '2011-05-06' limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@date_udf +POSTHOOK: Output: default@date_udf_string +POSTHOOK: Lineage: date_udf.d EXPRESSION [] +POSTHOOK: Lineage: date_udf_string.d SIMPLE [] +PREHOOK: query: create table date_udf_flight ( + ORIGIN_CITY_NAME string, + DEST_CITY_NAME string, + FL_DATE date, + ARR_DELAY float, + FL_NUM int +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table date_udf_flight ( + ORIGIN_CITY_NAME string, + DEST_CITY_NAME string, + FL_DATE date, + ARR_DELAY float, + FL_NUM int +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@date_udf_flight +POSTHOOK: Lineage: date_udf.d EXPRESSION [] +POSTHOOK: Lineage: date_udf_string.d SIMPLE [] +PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/flights_tiny.txt.1' OVERWRITE INTO TABLE date_udf_flight +PREHOOK: type: LOAD +PREHOOK: Output: default@date_udf_flight +POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/flights_tiny.txt.1' OVERWRITE INTO TABLE date_udf_flight +POSTHOOK: type: LOAD +POSTHOOK: Output: default@date_udf_flight +POSTHOOK: Lineage: date_udf.d EXPRESSION [] +POSTHOOK: Lineage: date_udf_string.d SIMPLE [] +PREHOOK: query: -- Test UDFs with date input +select unix_timestamp(d), year(d), month(d), day(d), dayofmonth(d), + weekofyear(d), to_date(d) + from date_udf +PREHOOK: type: QUERY +PREHOOK: Input: default@date_udf +#### A masked pattern was here #### +POSTHOOK: query: -- Test UDFs with date input +select unix_timestamp(d), year(d), month(d), day(d), dayofmonth(d), + weekofyear(d), to_date(d) + from date_udf +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_udf +#### A masked pattern was here #### +POSTHOOK: Lineage: date_udf.d EXPRESSION [] +POSTHOOK: Lineage: date_udf_string.d SIMPLE [] +1304665200 2011 5 6 6 18 2011-05-06 +PREHOOK: query: select date_add(d, 5), date_sub(d, 10) + from date_udf +PREHOOK: type: QUERY +PREHOOK: Input: default@date_udf +#### A masked pattern was here #### +POSTHOOK: query: select date_add(d, 5), date_sub(d, 10) + from date_udf +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_udf +#### A masked pattern was here #### +POSTHOOK: Lineage: date_udf.d EXPRESSION [] +POSTHOOK: Lineage: date_udf_string.d SIMPLE [] +2011-05-11 2011-04-26 +PREHOOK: query: select datediff(d, d), datediff(d, '2002-03-21'), datediff('2002-03-21', d), + datediff(cast ('2002-03-21 00:00:00' as timestamp), d), + datediff(d, cast ('2002-03-21 00:00:00' as timestamp)) + from date_udf +PREHOOK: type: QUERY +PREHOOK: Input: default@date_udf +#### A masked pattern was here #### +POSTHOOK: query: select datediff(d, d), datediff(d, '2002-03-21'), datediff('2002-03-21', d), + datediff(cast ('2002-03-21 00:00:00' as timestamp), d), + datediff(d, cast ('2002-03-21 00:00:00' as timestamp)) + from date_udf +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_udf +#### A masked pattern was here #### +POSTHOOK: Lineage: date_udf.d EXPRESSION [] +POSTHOOK: Lineage: date_udf_string.d SIMPLE [] +0 3333 -3333 -3332 3332 +PREHOOK: query: -- Test UDFs with string input +select unix_timestamp(d), year(d), month(d), day(d), dayofmonth(d), + weekofyear(d), to_date(d) + from date_udf_string +PREHOOK: type: QUERY +PREHOOK: Input: default@date_udf_string +#### A masked pattern was here #### +POSTHOOK: query: -- Test UDFs with string input +select unix_timestamp(d), year(d), month(d), day(d), dayofmonth(d), + weekofyear(d), to_date(d) + from date_udf_string +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_udf_string +#### A masked pattern was here #### +POSTHOOK: Lineage: date_udf.d EXPRESSION [] +POSTHOOK: Lineage: date_udf_string.d SIMPLE [] +NULL 2011 5 6 6 18 2011-05-06 +PREHOOK: query: select date_add(d, 5), date_sub(d, 10) from date_udf_string +PREHOOK: type: QUERY +PREHOOK: Input: default@date_udf_string +#### A masked pattern was here #### +POSTHOOK: query: select date_add(d, 5), date_sub(d, 10) from date_udf_string +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_udf_string +#### A masked pattern was here #### +POSTHOOK: Lineage: date_udf.d EXPRESSION [] +POSTHOOK: Lineage: date_udf_string.d SIMPLE [] +2011-05-11 2011-04-26 +PREHOOK: query: select datediff(d, d), datediff(d, '2002-03-21'), datediff('2002-03-21', d), + datediff('2002-03-21 00:00:00', d), + datediff(d, '2002-03-21 00:00:00') + from date_udf_string +PREHOOK: type: QUERY +PREHOOK: Input: default@date_udf_string +#### A masked pattern was here #### +POSTHOOK: query: select datediff(d, d), datediff(d, '2002-03-21'), datediff('2002-03-21', d), + datediff('2002-03-21 00:00:00', d), + datediff(d, '2002-03-21 00:00:00') + from date_udf_string +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_udf_string +#### A masked pattern was here #### +POSTHOOK: Lineage: date_udf.d EXPRESSION [] +POSTHOOK: Lineage: date_udf_string.d SIMPLE [] +0 3333 -3333 -3333 3333 +PREHOOK: query: select + to_utc_timestamp(date '1970-01-01', 'America/Los_Angeles'), + from_utc_timestamp(date '1970-01-01', 'America/Los_Angeles'), + to_utc_timestamp(date '2013-06-19', 'America/Los_Angeles'), + from_utc_timestamp(date '2013-06-19', 'America/Los_Angeles') + from date_udf +PREHOOK: type: QUERY +PREHOOK: Input: default@date_udf +#### A masked pattern was here #### +POSTHOOK: query: select + to_utc_timestamp(date '1970-01-01', 'America/Los_Angeles'), + from_utc_timestamp(date '1970-01-01', 'America/Los_Angeles'), + to_utc_timestamp(date '2013-06-19', 'America/Los_Angeles'), + from_utc_timestamp(date '2013-06-19', 'America/Los_Angeles') + from date_udf +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_udf +#### A masked pattern was here #### +POSTHOOK: Lineage: date_udf.d EXPRESSION [] +POSTHOOK: Lineage: date_udf_string.d SIMPLE [] +1970-01-01 08:00:00 1969-12-31 16:00:00 2013-06-19 07:00:00 2013-06-18 17:00:00 +PREHOOK: query: -- should all be true +select + to_utc_timestamp(date '1970-01-01', 'America/Los_Angeles') = to_utc_timestamp(timestamp('1970-01-01 00:00:00'), 'America/Los_Angeles'), + from_utc_timestamp(date '1970-01-01', 'America/Los_Angeles') = from_utc_timestamp(timestamp('1970-01-01 00:00:00'), 'America/Los_Angeles'), + to_utc_timestamp(date '2013-06-19', 'America/Los_Angeles') = to_utc_timestamp(timestamp('2013-06-19 00:00:00'), 'America/Los_Angeles'), + from_utc_timestamp(date '2013-06-19', 'America/Los_Angeles') = from_utc_timestamp(timestamp('2013-06-19 00:00:00'), 'America/Los_Angeles') + from date_udf +PREHOOK: type: QUERY +PREHOOK: Input: default@date_udf +#### A masked pattern was here #### +POSTHOOK: query: -- should all be true +select + to_utc_timestamp(date '1970-01-01', 'America/Los_Angeles') = to_utc_timestamp(timestamp('1970-01-01 00:00:00'), 'America/Los_Angeles'), + from_utc_timestamp(date '1970-01-01', 'America/Los_Angeles') = from_utc_timestamp(timestamp('1970-01-01 00:00:00'), 'America/Los_Angeles'), + to_utc_timestamp(date '2013-06-19', 'America/Los_Angeles') = to_utc_timestamp(timestamp('2013-06-19 00:00:00'), 'America/Los_Angeles'), + from_utc_timestamp(date '2013-06-19', 'America/Los_Angeles') = from_utc_timestamp(timestamp('2013-06-19 00:00:00'), 'America/Los_Angeles') + from date_udf +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_udf +#### A masked pattern was here #### +POSTHOOK: Lineage: date_udf.d EXPRESSION [] +POSTHOOK: Lineage: date_udf_string.d SIMPLE [] +true true true true +PREHOOK: query: -- Aggregation functions (min/max) +select min(fl_date) from date_udf_flight +PREHOOK: type: QUERY +PREHOOK: Input: default@date_udf_flight +#### A masked pattern was here #### +POSTHOOK: query: -- Aggregation functions (min/max) +select min(fl_date) from date_udf_flight +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_udf_flight +#### A masked pattern was here #### +POSTHOOK: Lineage: date_udf.d EXPRESSION [] +POSTHOOK: Lineage: date_udf_string.d SIMPLE [] +2010-10-20 +PREHOOK: query: select max(fl_date) from date_udf_flight +PREHOOK: type: QUERY +PREHOOK: Input: default@date_udf_flight +#### A masked pattern was here #### +POSTHOOK: query: select max(fl_date) from date_udf_flight +POSTHOOK: type: QUERY +POSTHOOK: Input: default@date_udf_flight +#### A masked pattern was here #### +POSTHOOK: Lineage: date_udf.d EXPRESSION [] +POSTHOOK: Lineage: date_udf_string.d SIMPLE [] +2010-10-31 +PREHOOK: query: drop table date_udf +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@date_udf +PREHOOK: Output: default@date_udf +POSTHOOK: query: drop table date_udf +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@date_udf +POSTHOOK: Output: default@date_udf +POSTHOOK: Lineage: date_udf.d EXPRESSION [] +POSTHOOK: Lineage: date_udf_string.d SIMPLE [] +PREHOOK: query: drop table date_udf_string +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@date_udf_string +PREHOOK: Output: default@date_udf_string +POSTHOOK: query: drop table date_udf_string +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@date_udf_string +POSTHOOK: Output: default@date_udf_string +POSTHOOK: Lineage: date_udf.d EXPRESSION [] +POSTHOOK: Lineage: date_udf_string.d SIMPLE [] +PREHOOK: query: drop table date_udf_flight +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@date_udf_flight +PREHOOK: Output: default@date_udf_flight +POSTHOOK: query: drop table date_udf_flight +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@date_udf_flight +POSTHOOK: Output: default@date_udf_flight +POSTHOOK: Lineage: date_udf.d EXPRESSION [] +POSTHOOK: Lineage: date_udf_string.d SIMPLE [] diff --git ql/src/test/results/clientpositive/partition_date.q.out ql/src/test/results/clientpositive/partition_date.q.out new file mode 100644 index 0000000..53fd0b1 --- /dev/null +++ ql/src/test/results/clientpositive/partition_date.q.out @@ -0,0 +1,392 @@ +PREHOOK: query: drop table partition_date_1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table partition_date_1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table partition_date_1 (key string, value string) partitioned by (dt date, region int) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table partition_date_1 (key string, value string) partitioned by (dt date, region int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@partition_date_1 +PREHOOK: query: insert overwrite table partition_date_1 partition(dt='2000-01-01', region=1) + select * from src limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@partition_date_1@dt=2000-01-01/region=1 +POSTHOOK: query: insert overwrite table partition_date_1 partition(dt='2000-01-01', region=1) + select * from src limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@partition_date_1@dt=2000-01-01/region=1 +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table partition_date_1 partition(dt='2000-01-01', region=2) + select * from src limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@partition_date_1@dt=2000-01-01/region=2 +POSTHOOK: query: insert overwrite table partition_date_1 partition(dt='2000-01-01', region=2) + select * from src limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@partition_date_1@dt=2000-01-01/region=2 +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table partition_date_1 partition(dt='2013-08-08', region=1) + select * from src limit 20 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@partition_date_1@dt=2013-08-08/region=1 +POSTHOOK: query: insert overwrite table partition_date_1 partition(dt='2013-08-08', region=1) + select * from src limit 20 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@partition_date_1@dt=2013-08-08/region=1 +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table partition_date_1 partition(dt='2013-08-08', region=10) + select * from src limit 11 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@partition_date_1@dt=2013-08-08/region=10 +POSTHOOK: query: insert overwrite table partition_date_1 partition(dt='2013-08-08', region=10) + select * from src limit 11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@partition_date_1@dt=2013-08-08/region=10 +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select distinct dt from partition_date_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_date_1 +PREHOOK: Input: default@partition_date_1@dt=2000-01-01/region=1 +PREHOOK: Input: default@partition_date_1@dt=2000-01-01/region=2 +PREHOOK: Input: default@partition_date_1@dt=2013-08-08/region=1 +PREHOOK: Input: default@partition_date_1@dt=2013-08-08/region=10 +#### A masked pattern was here #### +POSTHOOK: query: select distinct dt from partition_date_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_date_1 +POSTHOOK: Input: default@partition_date_1@dt=2000-01-01/region=1 +POSTHOOK: Input: default@partition_date_1@dt=2000-01-01/region=2 +POSTHOOK: Input: default@partition_date_1@dt=2013-08-08/region=1 +POSTHOOK: Input: default@partition_date_1@dt=2013-08-08/region=10 +#### A masked pattern was here #### +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +2000-01-01 +2013-08-08 +PREHOOK: query: select *, cast(dt as timestamp) from partition_date_1 where dt = '2000-01-01' and region = 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_date_1 +PREHOOK: Input: default@partition_date_1@dt=2000-01-01/region=2 +#### A masked pattern was here #### +POSTHOOK: query: select *, cast(dt as timestamp) from partition_date_1 where dt = '2000-01-01' and region = 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_date_1 +POSTHOOK: Input: default@partition_date_1@dt=2000-01-01/region=2 +#### A masked pattern was here #### +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +238 val_238 2000-01-01 2 1969-12-31 16:00:00 +86 val_86 2000-01-01 2 1969-12-31 16:00:00 +311 val_311 2000-01-01 2 1969-12-31 16:00:00 +27 val_27 2000-01-01 2 1969-12-31 16:00:00 +165 val_165 2000-01-01 2 1969-12-31 16:00:00 +PREHOOK: query: -- 15 +select count(*) from partition_date_1 where dt = date '2000-01-01' +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_date_1 +PREHOOK: Input: default@partition_date_1@dt=2000-01-01/region=1 +PREHOOK: Input: default@partition_date_1@dt=2000-01-01/region=2 +#### A masked pattern was here #### +POSTHOOK: query: -- 15 +select count(*) from partition_date_1 where dt = date '2000-01-01' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_date_1 +POSTHOOK: Input: default@partition_date_1@dt=2000-01-01/region=1 +POSTHOOK: Input: default@partition_date_1@dt=2000-01-01/region=2 +#### A masked pattern was here #### +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +15 +PREHOOK: query: -- 15. Also try with string value in predicate +select count(*) from partition_date_1 where dt = '2000-01-01' +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_date_1 +PREHOOK: Input: default@partition_date_1@dt=2000-01-01/region=1 +PREHOOK: Input: default@partition_date_1@dt=2000-01-01/region=2 +#### A masked pattern was here #### +POSTHOOK: query: -- 15. Also try with string value in predicate +select count(*) from partition_date_1 where dt = '2000-01-01' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_date_1 +POSTHOOK: Input: default@partition_date_1@dt=2000-01-01/region=1 +POSTHOOK: Input: default@partition_date_1@dt=2000-01-01/region=2 +#### A masked pattern was here #### +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +15 +PREHOOK: query: -- 5 +select count(*) from partition_date_1 where dt = date '2000-01-01' and region = 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_date_1 +PREHOOK: Input: default@partition_date_1@dt=2000-01-01/region=2 +#### A masked pattern was here #### +POSTHOOK: query: -- 5 +select count(*) from partition_date_1 where dt = date '2000-01-01' and region = 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_date_1 +POSTHOOK: Input: default@partition_date_1@dt=2000-01-01/region=2 +#### A masked pattern was here #### +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +5 +PREHOOK: query: -- 11 +select count(*) from partition_date_1 where dt = date '2013-08-08' and region = 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_date_1 +PREHOOK: Input: default@partition_date_1@dt=2013-08-08/region=10 +#### A masked pattern was here #### +POSTHOOK: query: -- 11 +select count(*) from partition_date_1 where dt = date '2013-08-08' and region = 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_date_1 +POSTHOOK: Input: default@partition_date_1@dt=2013-08-08/region=10 +#### A masked pattern was here #### +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +11 +PREHOOK: query: -- 30 +select count(*) from partition_date_1 where region = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_date_1 +PREHOOK: Input: default@partition_date_1@dt=2000-01-01/region=1 +PREHOOK: Input: default@partition_date_1@dt=2013-08-08/region=1 +#### A masked pattern was here #### +POSTHOOK: query: -- 30 +select count(*) from partition_date_1 where region = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_date_1 +POSTHOOK: Input: default@partition_date_1@dt=2000-01-01/region=1 +POSTHOOK: Input: default@partition_date_1@dt=2013-08-08/region=1 +#### A masked pattern was here #### +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +30 +PREHOOK: query: -- 0 +select count(*) from partition_date_1 where dt = date '2000-01-01' and region = 3 +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_date_1 +#### A masked pattern was here #### +POSTHOOK: query: -- 0 +select count(*) from partition_date_1 where dt = date '2000-01-01' and region = 3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_date_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +0 +PREHOOK: query: -- 0 +select count(*) from partition_date_1 where dt = date '1999-01-01' +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_date_1 +#### A masked pattern was here #### +POSTHOOK: query: -- 0 +select count(*) from partition_date_1 where dt = date '1999-01-01' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_date_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +0 +PREHOOK: query: -- Try other comparison operations + +-- 20 +select count(*) from partition_date_1 where dt > date '2000-01-01' and region = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_date_1 +PREHOOK: Input: default@partition_date_1@dt=2013-08-08/region=1 +#### A masked pattern was here #### +POSTHOOK: query: -- Try other comparison operations + +-- 20 +select count(*) from partition_date_1 where dt > date '2000-01-01' and region = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_date_1 +POSTHOOK: Input: default@partition_date_1@dt=2013-08-08/region=1 +#### A masked pattern was here #### +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +20 +PREHOOK: query: -- 10 +select count(*) from partition_date_1 where dt < date '2000-01-02' and region = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_date_1 +PREHOOK: Input: default@partition_date_1@dt=2000-01-01/region=1 +#### A masked pattern was here #### +POSTHOOK: query: -- 10 +select count(*) from partition_date_1 where dt < date '2000-01-02' and region = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_date_1 +POSTHOOK: Input: default@partition_date_1@dt=2000-01-01/region=1 +#### A masked pattern was here #### +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +10 +PREHOOK: query: -- 20 +select count(*) from partition_date_1 where dt >= date '2000-01-02' and region = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_date_1 +PREHOOK: Input: default@partition_date_1@dt=2013-08-08/region=1 +#### A masked pattern was here #### +POSTHOOK: query: -- 20 +select count(*) from partition_date_1 where dt >= date '2000-01-02' and region = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_date_1 +POSTHOOK: Input: default@partition_date_1@dt=2013-08-08/region=1 +#### A masked pattern was here #### +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +20 +PREHOOK: query: -- 10 +select count(*) from partition_date_1 where dt <= date '2000-01-01' and region = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_date_1 +PREHOOK: Input: default@partition_date_1@dt=2000-01-01/region=1 +#### A masked pattern was here #### +POSTHOOK: query: -- 10 +select count(*) from partition_date_1 where dt <= date '2000-01-01' and region = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_date_1 +POSTHOOK: Input: default@partition_date_1@dt=2000-01-01/region=1 +#### A masked pattern was here #### +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +10 +PREHOOK: query: -- 20 +select count(*) from partition_date_1 where dt <> date '2000-01-01' and region = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_date_1 +PREHOOK: Input: default@partition_date_1@dt=2013-08-08/region=1 +#### A masked pattern was here #### +POSTHOOK: query: -- 20 +select count(*) from partition_date_1 where dt <> date '2000-01-01' and region = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_date_1 +POSTHOOK: Input: default@partition_date_1@dt=2013-08-08/region=1 +#### A masked pattern was here #### +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +20 +PREHOOK: query: drop table partition_date_1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@partition_date_1 +PREHOOK: Output: default@partition_date_1 +POSTHOOK: query: drop table partition_date_1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@partition_date_1 +POSTHOOK: Output: default@partition_date_1 +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2000-01-01,region=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] diff --git ql/src/test/results/clientpositive/union_date.q.out ql/src/test/results/clientpositive/union_date.q.out new file mode 100644 index 0000000..cf8ca44 --- /dev/null +++ ql/src/test/results/clientpositive/union_date.q.out @@ -0,0 +1,128 @@ +PREHOOK: query: drop table union_date_1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table union_date_1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table union_date_2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table union_date_2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table union_date_1 ( + ORIGIN_CITY_NAME string, + DEST_CITY_NAME string, + FL_DATE date, + ARR_DELAY float, + FL_NUM int +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table union_date_1 ( + ORIGIN_CITY_NAME string, + DEST_CITY_NAME string, + FL_DATE date, + ARR_DELAY float, + FL_NUM int +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@union_date_1 +PREHOOK: query: create table union_date_2 ( + ORIGIN_CITY_NAME string, + DEST_CITY_NAME string, + FL_DATE date, + ARR_DELAY float, + FL_NUM int +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table union_date_2 ( + ORIGIN_CITY_NAME string, + DEST_CITY_NAME string, + FL_DATE date, + ARR_DELAY float, + FL_NUM int +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@union_date_2 +PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/flights_join.txt' OVERWRITE INTO TABLE union_date_1 +PREHOOK: type: LOAD +PREHOOK: Output: default@union_date_1 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/flights_join.txt' OVERWRITE INTO TABLE union_date_1 +POSTHOOK: type: LOAD +POSTHOOK: Output: default@union_date_1 +PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/flights_join.txt' OVERWRITE INTO TABLE union_date_2 +PREHOOK: type: LOAD +PREHOOK: Output: default@union_date_2 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/flights_join.txt' OVERWRITE INTO TABLE union_date_2 +POSTHOOK: type: LOAD +POSTHOOK: Output: default@union_date_2 +PREHOOK: query: select * from ( + select fl_num, fl_date from union_date_1 + union all + select fl_num, fl_date from union_date_2 +) union_result order by fl_date, fl_num +PREHOOK: type: QUERY +PREHOOK: Input: default@union_date_1 +PREHOOK: Input: default@union_date_2 +#### A masked pattern was here #### +POSTHOOK: query: select * from ( + select fl_num, fl_date from union_date_1 + union all + select fl_num, fl_date from union_date_2 +) union_result order by fl_date, fl_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@union_date_1 +POSTHOOK: Input: default@union_date_2 +#### A masked pattern was here #### +1064 2000-11-20 +1064 2000-11-20 +1142 2000-11-21 +1142 2000-11-21 +1599 2000-11-22 +1599 2000-11-22 +361 2000-11-23 +361 2000-11-23 +897 2000-11-24 +897 2000-11-24 +1531 2000-11-25 +1531 2000-11-25 +1610 2000-11-26 +1610 2000-11-26 +3198 2000-11-27 +3198 2000-11-27 +1064 2000-11-28 +1064 2000-11-28 +1142 2000-11-28 +1142 2000-11-28 +1064 2010-10-20 +1064 2010-10-20 +1142 2010-10-21 +1142 2010-10-21 +1599 2010-10-22 +1599 2010-10-22 +361 2010-10-23 +361 2010-10-23 +897 2010-10-24 +897 2010-10-24 +1531 2010-10-25 +1531 2010-10-25 +1610 2010-10-26 +1610 2010-10-26 +3198 2010-10-27 +3198 2010-10-27 +1064 2010-10-28 +1064 2010-10-28 +1142 2010-10-29 +1142 2010-10-29 +PREHOOK: query: drop table union_date_1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@union_date_1 +PREHOOK: Output: default@union_date_1 +POSTHOOK: query: drop table union_date_1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@union_date_1 +POSTHOOK: Output: default@union_date_1 +PREHOOK: query: drop table union_date_2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@union_date_2 +PREHOOK: Output: default@union_date_2 +POSTHOOK: query: drop table union_date_2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@union_date_2 +POSTHOOK: Output: default@union_date_2 diff --git serde/src/java/org/apache/hadoop/hive/serde2/RegexSerDe.java serde/src/java/org/apache/hadoop/hive/serde2/RegexSerDe.java index 2736d4b..5d0eb0c 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/RegexSerDe.java +++ serde/src/java/org/apache/hadoop/hive/serde2/RegexSerDe.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hive.serde2; +import java.sql.Date; +import java.sql.Timestamp; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -134,6 +136,10 @@ public void initialize(Configuration conf, Properties tbl) columnOIs.add(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector); } else if (typeName.equals(serdeConstants.BOOLEAN_TYPE_NAME)) { columnOIs.add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector); + } else if (typeName.equals(serdeConstants.TIMESTAMP_TYPE_NAME)) { + columnOIs.add(PrimitiveObjectInspectorFactory.javaTimestampObjectInspector); + } else if (typeName.equals(serdeConstants.DATE_TYPE_NAME)) { + columnOIs.add(PrimitiveObjectInspectorFactory.javaDateObjectInspector); } else if (typeName.equals(serdeConstants.DECIMAL_TYPE_NAME)) { columnOIs.add(PrimitiveObjectInspectorFactory.javaHiveDecimalObjectInspector); } else { @@ -229,6 +235,14 @@ public Object deserialize(Writable blob) throws SerDeException { Boolean b; b = Boolean.valueOf(t); row.set(c, b); + } else if (typeName.equals(serdeConstants.TIMESTAMP_TYPE_NAME)) { + Timestamp ts; + ts = Timestamp.valueOf(t); + row.set(c, ts); + } else if (typeName.equals(serdeConstants.DATE_TYPE_NAME)) { + Date d; + d = Date.valueOf(t); + row.set(c, d); } else if (typeName.equals(serdeConstants.DECIMAL_TYPE_NAME)) { HiveDecimal bd; bd = new HiveDecimal(t); diff --git serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java index f68388f..f88fd94 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.Text; @@ -283,6 +284,13 @@ static void buildJSONString(StringBuilder sb, Object o, ObjectInspector oi, Stri sb.append('"'); break; } + case DATE: { + sb.append('"'); + sb.append(((DateObjectInspector) poi) + .getPrimitiveWritableObject(o)); + sb.append('"'); + break; + } case TIMESTAMP: { sb.append('"'); sb.append(((TimestampObjectInspector) poi) diff --git serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java index 2cb15cc..46c949a 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java +++ serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.SerDeStats; import org.apache.hadoop.hive.serde2.io.ByteWritable; +import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -52,6 +53,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector; @@ -225,11 +227,7 @@ static Object deserialize(InputByteBuffer buffer, TypeInfo type, } case INT: { IntWritable r = reuse == null ? new IntWritable() : (IntWritable) reuse; - int v = buffer.read(invert) ^ 0x80; - for (int i = 0; i < 3; i++) { - v = (v << 8) + (buffer.read(invert) & 0xff); - } - r.set(v); + r.set(deserializeInt(buffer, invert)); return r; } case LONG: { @@ -368,6 +366,13 @@ static Object deserialize(InputByteBuffer buffer, TypeInfo type, return bw; } + case DATE: { + DateWritable d = reuse == null ? new DateWritable() + : (DateWritable) reuse; + d.set(deserializeInt(buffer, invert)); + return d; + } + case TIMESTAMP: TimestampWritable t = (reuse == null ? new TimestampWritable() : (TimestampWritable) reuse); @@ -539,6 +544,14 @@ static Object deserialize(InputByteBuffer buffer, TypeInfo type, } } + private static int deserializeInt(InputByteBuffer buffer, boolean invert) throws IOException { + int v = buffer.read(invert) ^ 0x80; + for (int i = 0; i < 3; i++) { + v = (v << 8) + (buffer.read(invert) & 0xff); + } + return v; + } + BytesWritable serializeBytesWritable = new BytesWritable(); OutputByteBuffer outputByteBuffer = new OutputByteBuffer(); @@ -596,10 +609,7 @@ static void serialize(OutputByteBuffer buffer, Object o, ObjectInspector oi, case INT: { IntObjectInspector ioi = (IntObjectInspector) poi; int v = ioi.get(o); - buffer.write((byte) ((v >> 24) ^ 0x80), invert); - buffer.write((byte) (v >> 16), invert); - buffer.write((byte) (v >> 8), invert); - buffer.write((byte) v, invert); + serializeInt(buffer, v, invert); return; } case LONG: { @@ -666,6 +676,12 @@ static void serialize(OutputByteBuffer buffer, Object o, ObjectInspector oi, serializeBytes(buffer, toSer, ba.getLength(), invert); return; } + case DATE: { + DateObjectInspector doi = (DateObjectInspector) poi; + int v = doi.getPrimitiveWritableObject(o).getDays(); + serializeInt(buffer, v, invert); + return; + } case TIMESTAMP: { TimestampObjectInspector toi = (TimestampObjectInspector) poi; TimestampWritable t = toi.getPrimitiveWritableObject(o); @@ -788,6 +804,14 @@ private static void serializeBytes(OutputByteBuffer buffer, byte[] data, int len } buffer.write((byte) 0, invert); } + + private static void serializeInt(OutputByteBuffer buffer, int v, boolean invert) { + buffer.write((byte) ((v >> 24) ^ 0x80), invert); + buffer.write((byte) (v >> 16), invert); + buffer.write((byte) (v >> 8), invert); + buffer.write((byte) v, invert); + } + @Override public SerDeStats getSerDeStats() { // no support for statistics diff --git serde/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java serde/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java new file mode 100644 index 0000000..1f4ccdd --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java @@ -0,0 +1,183 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.io; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.sql.Date; +import java.util.Calendar; +import java.util.TimeZone; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.serde2.ByteStream.Output; +import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils; +import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils.VInt; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.io.WritableUtils; + + +/** + * DateWritable + * Writable equivalent of java.sql.Date. + * + * Dates are of the format + * YYYY-MM-DD + * + */ +public class DateWritable implements WritableComparable { + private static final Log LOG = LogFactory.getLog(DateWritable.class); + + private static final long MILLIS_PER_DAY = TimeUnit.DAYS.toMillis(1); + + // Local time zone. + // Java TimeZone has no mention of thread safety. Use thread local instance to be safe. + private static final ThreadLocal LOCAL_TIMEZONE = new ThreadLocal() { + @Override + protected TimeZone initialValue() { + return Calendar.getInstance().getTimeZone(); + } + }; + + // Internal representation is an integer representing day offset from our epoch value 1970-01-01 + private int daysSinceEpoch = 0; + + /* Constructors */ + public DateWritable() { + } + + public DateWritable(DateWritable d) { + set(d); + } + + public DateWritable(Date d) { + set(d); + } + + public DateWritable(int d) { + set(d); + } + + /** + * Set the DateWritable based on the days since epoch date. + * @param d integer value representing days since epoch date + */ + public void set(int d) { + daysSinceEpoch = d; + } + + /** + * Set the DateWritable based on the year/month/day of the date in the local timezone. + * @param d Date value + */ + public void set(Date d) { + if (d == null) { + daysSinceEpoch = 0; + return; + } + + set(dateToDays(d)); + } + + public void set(DateWritable d) { + set(d.daysSinceEpoch); + } + + /** + * + * @return Date value corresponding to the date in the local time zone + */ + public Date get() { + return new Date(daysToMillis(daysSinceEpoch)); + } + + public int getDays() { + return daysSinceEpoch; + } + + /** + * + * @return time in seconds corresponding to this DateWritable + */ + public long getTimeInSeconds() { + return get().getTime() / 1000; + } + + public static Date timeToDate(long l) { + return new Date(l * 1000); + } + + public static long daysToMillis(int d) { + // Convert from day offset to ms in UTC, then apply local timezone offset. + long millisUtc = d * MILLIS_PER_DAY; + return millisUtc - LOCAL_TIMEZONE.get().getOffset(millisUtc); + } + + public static int dateToDays(Date d) { + // convert to equivalent time in UTC, then get day offset + long millisLocal = d.getTime(); + long millisUtc = millisLocal + LOCAL_TIMEZONE.get().getOffset(millisLocal); + return (int)(millisUtc / MILLIS_PER_DAY); + } + + public void setFromBytes(byte[] bytes, int offset, int length, VInt vInt) { + LazyBinaryUtils.readVInt(bytes, offset, vInt); + assert (length == vInt.length); + set(vInt.value); + } + + public void writeToByteStream(Output byteStream) { + LazyBinaryUtils.writeVInt(byteStream, getDays()); + } + + + @Override + public void readFields(DataInput in) throws IOException { + daysSinceEpoch = WritableUtils.readVInt(in); + } + + @Override + public void write(DataOutput out) throws IOException { + WritableUtils.writeVInt(out, daysSinceEpoch); + } + + @Override + public int compareTo(DateWritable d) { + return daysSinceEpoch - d.daysSinceEpoch; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof DateWritable)) { + return false; + } + return compareTo((DateWritable) o) == 0; + } + + @Override + public String toString() { + return get().toString(); + } + + @Override + public int hashCode() { + return daysSinceEpoch; + } +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDate.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDate.java new file mode 100644 index 0000000..8492522 --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDate.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.lazy; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.sql.Date; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyDateObjectInspector; +import org.apache.hadoop.io.Text; + +/** + * + * LazyDate. + * Serializes and deserializes a Date in the SQL date format + * + * YYYY-MM-DD + * + */ +public class LazyDate extends LazyPrimitive { + private static final Log LOG = LogFactory.getLog(LazyDate.class); + + public LazyDate(LazyDateObjectInspector oi) { + super(oi); + data = new DateWritable(); + } + + public LazyDate(LazyDate copy) { + super(copy); + data = new DateWritable(copy.data); + } + + /** + * Initializes LazyDate object by interpreting the input bytes as a SQL date string. + * + * @param bytes + * @param start + * @param length + */ + @Override + public void init(ByteArrayRef bytes, int start, int length) { + String s = null; + try { + s = Text.decode(bytes.getData(), start, length); + data.set(Date.valueOf(s)); + } catch (Exception e) { + isNull = true; + logExceptionMessage(bytes, start, length, "DATE"); + } + } + + /** + * Writes a Date in SQL date format to the output stream. + * @param out + * The output stream + * @param i + * The Date to write + * @throws IOException + */ + public static void writeUTF8(OutputStream out, DateWritable d) + throws IOException { + ByteBuffer b = Text.encode(d.toString()); + out.write(b.array(), 0, b.limit()); + } + +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java index e945116..a08b4a8 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyPrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyShortObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyStringObjectInspector; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyDateObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyTimestampObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyVoidObjectInspector; import org.apache.hadoop.hive.serde2.lazydio.LazyDioBoolean; @@ -111,6 +112,8 @@ return new LazyDouble((LazyDoubleObjectInspector) oi); case STRING: return new LazyString((LazyStringObjectInspector) oi); + case DATE: + return new LazyDate((LazyDateObjectInspector) oi); case TIMESTAMP: return new LazyTimestamp((LazyTimestampObjectInspector) oi); case BINARY: diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java index 31cc270..8a538c0 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.io.BytesWritable; @@ -232,6 +233,11 @@ public static void writePrimitiveUTF8(OutputStream out, Object o, out.write(toWrite, 0, toWrite.length); break; } + case DATE: { + LazyDate.writeUTF8(out, + ((DateObjectInspector) oi).getPrimitiveWritableObject(o)); + break; + } case TIMESTAMP: { LazyTimestamp.writeUTF8(out, ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o)); diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyDateObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyDateObjectInspector.java new file mode 100644 index 0000000..679e5ea --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyDateObjectInspector.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive; + +import java.sql.Date; + +import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.lazy.LazyDate; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; + +/** + * A WritableDateObjectInspector inspects a DateWritable Object. + */ +public class LazyDateObjectInspector + extends AbstractPrimitiveLazyObjectInspector + implements DateObjectInspector { + + protected LazyDateObjectInspector() { + super(PrimitiveObjectInspectorUtils.dateTypeEntry); + } + + @Override + public Object copyObject(Object o) { + return o == null ? null : new LazyDate((LazyDate) o); + } + + @Override + public Date getPrimitiveJavaObject(Object o) { + return o == null ? null : ((LazyDate) o).getWritableObject().get(); + } +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java index 8652a46..afbf454 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java @@ -53,6 +53,8 @@ new LazyDoubleObjectInspector(); public static final LazyVoidObjectInspector LAZY_VOID_OBJECT_INSPECTOR = new LazyVoidObjectInspector(); + public static final LazyDateObjectInspector LAZY_DATE_OBJECT_INSPECTOR = + new LazyDateObjectInspector(); public static final LazyTimestampObjectInspector LAZY_TIMESTAMP_OBJECT_INSPECTOR = new LazyTimestampObjectInspector(); public static final LazyBinaryObjectInspector LAZY_BINARY_OBJECT_INSPECTOR = @@ -101,6 +103,8 @@ public static LazyStringObjectInspector getLazyStringObjectInspector( return LAZY_BINARY_OBJECT_INSPECTOR; case VOID: return LAZY_VOID_OBJECT_INSPECTOR; + case DATE: + return LAZY_DATE_OBJECT_INSPECTOR; case TIMESTAMP: return LAZY_TIMESTAMP_OBJECT_INSPECTOR; case DECIMAL: diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryDate.java serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryDate.java new file mode 100644 index 0000000..d0c2504 --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryDate.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.lazybinary; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; +import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils.VInt; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDateObjectInspector; + +/** + * A LazyBinaryObject that encodes a java.sql.Date in a VInt. + * + */ +public class LazyBinaryDate extends + LazyBinaryPrimitive { + static final Log LOG = LogFactory.getLog(LazyBinaryDate.class); + + LazyBinaryDate(WritableDateObjectInspector oi) { + super(oi); + data = new DateWritable(); + } + + LazyBinaryDate(LazyBinaryDate copy) { + super(copy); + data = new DateWritable(copy.data); + } + + /** + * Reusable member for decoding integer. + */ + VInt vInt = new LazyBinaryUtils.VInt(); + + /** + * Initializes LazyBinaryDate object. + * @param bytes + * @param start + * @param length + */ + @Override + public void init(ByteArrayRef bytes, int start, int length) { + data.setFromBytes(bytes.getData(), start, length, vInt); + } +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java index 3219aa1..48b3c05 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableLongObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableShortObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableStringObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDateObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableTimestampObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableVoidObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -72,6 +73,8 @@ return new LazyBinaryString((WritableStringObjectInspector) oi); case VOID: // for NULL return new LazyBinaryVoid((WritableVoidObjectInspector) oi); + case DATE: + return new LazyBinaryDate((WritableDateObjectInspector) oi); case TIMESTAMP: return new LazyBinaryTimestamp((WritableTimestampObjectInspector) oi); case BINARY: diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java index e39c709..76acc6a 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hive.serde2.ByteStream.Output; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.SerDeStats; +import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; @@ -43,12 +44,13 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector; @@ -378,6 +380,11 @@ public static boolean serialize(Output byteStream, Object obj, return warnedOnceNullMapKey; } + case DATE: { + DateWritable d = ((DateObjectInspector) poi).getPrimitiveWritableObject(obj); + d.writeToByteStream(byteStream); + return warnedOnceNullMapKey; + } case TIMESTAMP: { TimestampObjectInspector toi = (TimestampObjectInspector) poi; TimestampWritable t = toi.getPrimitiveWritableObject(obj); @@ -557,6 +564,7 @@ public static boolean serialize(Output byteStream, Object obj, * Returns the statistics after (de)serialization) */ + @Override public SerDeStats getSerDeStats() { // must be different assert (lastOperationSerialize != lastOperationDeserialize); diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java index cac20ff..f0c5535 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java @@ -196,6 +196,10 @@ public static void checkObjectByteInfo(ObjectInspector objectInspector, recordInfo.elementOffset = vInt.length; recordInfo.elementSize = vInt.value; break; + case DATE: + recordInfo.elementOffset = 0; + recordInfo.elementSize = WritableUtils.decodeVIntSize(bytes[offset]); + break; case TIMESTAMP: recordInfo.elementOffset = 0; recordInfo.elementSize = 4; diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java index 0b1ccd8..5336643 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableIntObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableLongObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableShortObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableDateObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableTimestampObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.VoidObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableStringObjectInspector; @@ -100,6 +101,10 @@ private static Converter getConverter(PrimitiveObjectInspector inputOI, return new PrimitiveObjectInspectorConverter.StringConverter( inputOI); } + case DATE: + return new PrimitiveObjectInspectorConverter.DateConverter( + inputOI, + (SettableDateObjectInspector) outputOI); case TIMESTAMP: return new PrimitiveObjectInspectorConverter.TimestampConverter( inputOI, diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java index c534adc..f7418bc 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java @@ -30,6 +30,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; @@ -38,6 +39,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector; @@ -488,6 +490,8 @@ public static int hashCode(Object o, ObjectInspector objIns) { case BINARY: return ((BinaryObjectInspector) poi).getPrimitiveWritableObject(o).hashCode(); + case DATE: + return ((DateObjectInspector) poi).getPrimitiveWritableObject(o).hashCode(); case TIMESTAMP: TimestampWritable t = ((TimestampObjectInspector) poi) .getPrimitiveWritableObject(o); @@ -684,6 +688,13 @@ public static int compare(Object o1, ObjectInspector oi1, Object o2, return bw1.compareTo(bw2); } + case DATE: { + DateWritable d1 = ((DateObjectInspector) poi1) + .getPrimitiveWritableObject(o1); + DateWritable d2 = ((DateObjectInspector) poi2) + .getPrimitiveWritableObject(o2); + return d1.compareTo(d2); + } case TIMESTAMP: { TimestampWritable t1 = ((TimestampObjectInspector) poi1) .getPrimitiveWritableObject(o1); diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java index aace3bb..f1b8bdf 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java @@ -27,7 +27,8 @@ * The primitive types supported by Hive. */ public static enum PrimitiveCategory { - VOID, BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, TIMESTAMP, BINARY, DECIMAL, UNKNOWN + VOID, BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, + DATE, TIMESTAMP, BINARY, DECIMAL, UNKNOWN }; /** diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/DateObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/DateObjectInspector.java new file mode 100644 index 0000000..e335463 --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/DateObjectInspector.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import java.sql.Date; + +import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; + +/** + * A DateObjectInspector inspects an Object representing a Date. + */ +public interface DateObjectInspector extends PrimitiveObjectInspector { + + DateWritable getPrimitiveWritableObject(Object o); + + Date getPrimitiveJavaObject(Object o); +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaDateObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaDateObjectInspector.java new file mode 100644 index 0000000..458dc3f --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaDateObjectInspector.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import java.sql.Date; + +import org.apache.hadoop.hive.serde2.io.DateWritable; + +/** + * A JavaDateObjectInspector inspects a Java Date Object. + */ +public class JavaDateObjectInspector + extends AbstractPrimitiveJavaObjectInspector + implements SettableDateObjectInspector { + + protected JavaDateObjectInspector() { + super(PrimitiveObjectInspectorUtils.dateTypeEntry); + } + + public DateWritable getPrimitiveWritableObject(Object o) { + return o == null ? null : new DateWritable((Date) o); + } + + @Override + public Date getPrimitiveJavaObject(Object o) { + return o == null ? null : (Date) o; + } + + public Date get(Object o) { + return (Date) o; + } + + public Object set(Object o, Date value) { + ((Date) o).setTime(value.getTime()); + return o; + } + + public Object set(Object o, DateWritable d) { + ((Date) o).setTime(d.get().getTime()); + return o; + } + + public Object create(Date value) { + return new Date(value.getTime()); + } + +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java index 742493c..2cd2b13 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.serde2.objectinspector.primitive; +import java.sql.Date; import java.sql.Timestamp; import org.apache.hadoop.hive.common.type.HiveDecimal; @@ -237,6 +238,27 @@ public Object convert(Object input) { } } + public static class DateConverter implements Converter { + PrimitiveObjectInspector inputOI; + SettableDateObjectInspector outputOI; + Object r; + + public DateConverter(PrimitiveObjectInspector inputOI, + SettableDateObjectInspector outputOI) { + this.inputOI = inputOI; + this.outputOI = outputOI; + r = outputOI.create(new Date(0)); + } + + public Object convert(Object input) { + if (input == null) { + return null; + } + return outputOI.set(r, PrimitiveObjectInspectorUtils.getDate(input, + inputOI)); + } + } + public static class TimestampConverter implements Converter { PrimitiveObjectInspector inputOI; SettableTimestampObjectInspector outputOI; @@ -276,7 +298,7 @@ public Object convert(Object input) { if (input == null) { return null; } - + try { return outputOI.set(r, PrimitiveObjectInspectorUtils.getHiveDecimal(input, inputOI)); @@ -368,6 +390,9 @@ public Text convert(Object input) { t.set(((StringObjectInspector) inputOI).getPrimitiveJavaObject(input)); } return t; + case DATE: + t.set(((DateObjectInspector) inputOI).getPrimitiveWritableObject(input).toString()); + return t; case TIMESTAMP: t.set(((TimestampObjectInspector) inputOI) .getPrimitiveWritableObject(input).toString()); diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java index 5234a88..281f730 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; +import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; @@ -66,6 +67,8 @@ new JavaStringObjectInspector(); public static final JavaVoidObjectInspector javaVoidObjectInspector = new JavaVoidObjectInspector(); + public static final JavaDateObjectInspector javaDateObjectInspector = + new JavaDateObjectInspector(); public static final JavaTimestampObjectInspector javaTimestampObjectInspector = new JavaTimestampObjectInspector(); public static final JavaBinaryObjectInspector javaByteArrayObjectInspector = @@ -91,6 +94,8 @@ new WritableStringObjectInspector(); public static final WritableVoidObjectInspector writableVoidObjectInspector = new WritableVoidObjectInspector(); + public static final WritableDateObjectInspector writableDateObjectInspector = + new WritableDateObjectInspector(); public static final WritableTimestampObjectInspector writableTimestampObjectInspector = new WritableTimestampObjectInspector(); public static final WritableBinaryObjectInspector writableBinaryObjectInspector = @@ -119,6 +124,8 @@ writableStringObjectInspector); cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.VOID, writableVoidObjectInspector); + cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.DATE, + writableDateObjectInspector); cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.TIMESTAMP, writableTimestampObjectInspector); cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.BINARY, @@ -148,6 +155,8 @@ javaStringObjectInspector); cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.VOID, javaVoidObjectInspector); + cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.DATE, + javaDateObjectInspector); cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.TIMESTAMP, javaTimestampObjectInspector); cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.BINARY, @@ -198,6 +207,8 @@ public static ConstantObjectInspector getPrimitiveWritableConstantObjectInspecto return new WritableConstantDoubleObjectInspector((DoubleWritable)value); case STRING: return new WritableConstantStringObjectInspector((Text)value); + case DATE: + return new WritableConstantDateObjectInspector((DateWritable)value); case TIMESTAMP: return new WritableConstantTimestampObjectInspector((TimestampWritable)value); case DECIMAL: diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java index 94062dd..772eb43 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java @@ -21,6 +21,7 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.sql.Date; import java.sql.Timestamp; import java.util.HashMap; import java.util.Map; @@ -28,6 +29,7 @@ import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.io.ByteWritable; +import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -177,7 +179,9 @@ static void registerType(PrimitiveTypeEntry t) { public static final PrimitiveTypeEntry shortTypeEntry = new PrimitiveTypeEntry( PrimitiveCategory.SHORT, serdeConstants.SMALLINT_TYPE_NAME, Short.TYPE, Short.class, ShortWritable.class); - + public static final PrimitiveTypeEntry dateTypeEntry = new PrimitiveTypeEntry( + PrimitiveCategory.DATE, serdeConstants.DATE_TYPE_NAME, null, + Date.class, DateWritable.class); public static final PrimitiveTypeEntry timestampTypeEntry = new PrimitiveTypeEntry( PrimitiveCategory.TIMESTAMP, serdeConstants.TIMESTAMP_TYPE_NAME, null, Timestamp.class, TimestampWritable.class); @@ -200,6 +204,7 @@ static void registerType(PrimitiveTypeEntry t) { registerType(doubleTypeEntry); registerType(byteTypeEntry); registerType(shortTypeEntry); + registerType(dateTypeEntry); registerType(timestampTypeEntry); registerType(decimalTypeEntry); registerType(unknownTypeEntry); @@ -361,6 +366,10 @@ public static boolean comparePrimitiveObjects(Object o1, .getPrimitiveWritableObject(o2); return t1.equals(t2); } + case DATE: { + return ((DateObjectInspector) oi1).getPrimitiveWritableObject(o1) + .equals(((DateObjectInspector) oi2).getPrimitiveWritableObject(o2)); + } case TIMESTAMP: { return ((TimestampObjectInspector) oi1).getPrimitiveWritableObject(o1) .equals(((TimestampObjectInspector) oi2).getPrimitiveWritableObject(o2)); @@ -404,6 +413,7 @@ public static double convertPrimitiveToDouble(Object o, PrimitiveObjectInspector .getDouble(); case DECIMAL: return ((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o).doubleValue(); + case DATE: // unsupported conversion default: throw new NumberFormatException(); } @@ -482,8 +492,9 @@ public static boolean getBoolean(Object o, PrimitiveObjectInspector oi) { result = HiveDecimal.ZERO.compareTo( ((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o)) != 0; break; + case DATE: // unsupported conversion default: - throw new RuntimeException("Hive 2 Internal error: unknown type: " + throw new RuntimeException("Hive 2 Internal error: unsupported conversion from type: " + oi.getTypeName()); } return result; @@ -566,8 +577,9 @@ public static int getInt(Object o, PrimitiveObjectInspector oi) { result = ((HiveDecimalObjectInspector) oi) .getPrimitiveJavaObject(o).intValue(); break; + case DATE: // unsupported conversion default: { - throw new RuntimeException("Hive 2 Internal error: unknown type: " + throw new RuntimeException("Hive 2 Internal error: unsupported conversion from type: " + oi.getTypeName()); } } @@ -624,8 +636,9 @@ public static long getLong(Object o, PrimitiveObjectInspector oi) { result = ((HiveDecimalObjectInspector) oi) .getPrimitiveJavaObject(o).longValue(); break; + case DATE: // unsupported conversion default: - throw new RuntimeException("Hive 2 Internal error: unknown type: " + throw new RuntimeException("Hive 2 Internal error: unsupported conversion from type: " + oi.getTypeName()); } return result; @@ -675,8 +688,9 @@ public static double getDouble(Object o, PrimitiveObjectInspector oi) { result = ((HiveDecimalObjectInspector) oi) .getPrimitiveJavaObject(o).doubleValue(); break; + case DATE: // unsupported conversion default: - throw new RuntimeException("Hive 2 Internal error: unknown type: " + throw new RuntimeException("Hive 2 Internal error: unsupported conversion from type: " + oi.getTypeName()); } return result; @@ -732,6 +746,9 @@ public static String getString(Object o, PrimitiveObjectInspector oi) { StringObjectInspector soi = (StringObjectInspector) oi; result = soi.getPrimitiveJavaObject(o); break; + case DATE: + result = ((DateObjectInspector) oi).getPrimitiveWritableObject(o).toString(); + break; case TIMESTAMP: result = ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o).toString(); break; @@ -817,13 +834,48 @@ public static HiveDecimal getHiveDecimal(Object o, PrimitiveObjectInspector oi) case DECIMAL: result = ((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o); break; + case DATE: // unsupported conversion default: - throw new RuntimeException("Hive 2 Internal error: unknown type: " + throw new RuntimeException("Hive 2 Internal error: unsupported conversion from type: " + oi.getTypeName()); } return result; } + public static Date getDate(Object o, PrimitiveObjectInspector oi) { + if (o == null) { + return null; + } + + Date result = null; + switch (oi.getPrimitiveCategory()) { + case VOID: + result = null; + break; + case STRING: + StringObjectInspector soi = (StringObjectInspector) oi; + String s = soi.getPrimitiveJavaObject(o).trim(); + try { + result = Date.valueOf(s); + } catch (IllegalArgumentException e) { + result = null; + } + break; + case DATE: + result = ((DateObjectInspector) oi).getPrimitiveWritableObject(o).get(); + break; + case TIMESTAMP: + result = DateWritable.timeToDate( + ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o).getSeconds()); + break; + default: + throw new RuntimeException("Cannot convert to Date from: " + + oi.getTypeName()); + } + + return result; + } + public static Timestamp getTimestamp(Object o, PrimitiveObjectInspector oi) { if (o == null) { return null; @@ -876,6 +928,10 @@ public static Timestamp getTimestamp(Object o, PrimitiveObjectInspector oi) { result = null; } break; + case DATE: + result = new Timestamp( + ((DateObjectInspector) oi).getPrimitiveWritableObject(o).get().getTime()); + break; case TIMESTAMP: result = ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o).getTimestamp(); break; diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableDateObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableDateObjectInspector.java new file mode 100644 index 0000000..6aa8688 --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableDateObjectInspector.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import java.sql.Date; + +import org.apache.hadoop.hive.serde2.io.DateWritable; + +/** + * A SettableDecimalObjectInspector can set a Date value to an object. + */ +public interface SettableDateObjectInspector extends DateObjectInspector { + Object set(Object o, Date d); + + Object set(Object o, DateWritable d); + + Object create(Date d); +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantDateObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantDateObjectInspector.java new file mode 100644 index 0000000..3bd87e9 --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantDateObjectInspector.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; + +import org.apache.hadoop.hive.serde2.io.DateWritable; + +/** + * A WritableConstantDateObjectInspector is a WritableDateObjectInspector + * that implements ConstantObjectInspector. + */ +public class WritableConstantDateObjectInspector extends + WritableDateObjectInspector implements + ConstantObjectInspector { + + private DateWritable value; + + WritableConstantDateObjectInspector(DateWritable value) { + super(); + this.value = value; + } + + @Override + public DateWritable getWritableConstantValue() { + return value; + } +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableDateObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableDateObjectInspector.java new file mode 100644 index 0000000..bd0aec3 --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableDateObjectInspector.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import java.sql.Date; + +import org.apache.hadoop.hive.serde2.io.DateWritable; + +/** + * A WritableDateObjectInspector inspects a DateWritable Object. + */ +public class WritableDateObjectInspector extends + AbstractPrimitiveWritableObjectInspector implements + SettableDateObjectInspector { + + public WritableDateObjectInspector() { + super(PrimitiveObjectInspectorUtils.dateTypeEntry); + } + + @Override + public DateWritable getPrimitiveWritableObject(Object o) { + return o == null ? null : (DateWritable) o; + } + + public Date getPrimitiveJavaObject(Object o) { + return o == null ? null : ((DateWritable) o).get(); + } + + public Object copyObject(Object o) { + return o == null ? null : new DateWritable((DateWritable) o); + } + + public Object set(Object o, Date d) { + ((DateWritable) o).set(d); + return o; + } + + public Object set(Object o, DateWritable d) { + ((DateWritable) o).set(d); + return o; + } + + public Object create(Date d) { + return new DateWritable(d); + } +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java index f1b5eb5..542af09 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java +++ serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java @@ -62,6 +62,7 @@ public static TypeInfo getPrimitiveTypeInfo(String typeName) { public static final TypeInfo doubleTypeInfo = getPrimitiveTypeInfo(serdeConstants.DOUBLE_TYPE_NAME); public static final TypeInfo byteTypeInfo = getPrimitiveTypeInfo(serdeConstants.TINYINT_TYPE_NAME); public static final TypeInfo shortTypeInfo = getPrimitiveTypeInfo(serdeConstants.SMALLINT_TYPE_NAME); + public static final TypeInfo dateTypeInfo = getPrimitiveTypeInfo(serdeConstants.DATE_TYPE_NAME); public static final TypeInfo timestampTypeInfo = getPrimitiveTypeInfo(serdeConstants.TIMESTAMP_TYPE_NAME); public static final TypeInfo binaryTypeInfo = getPrimitiveTypeInfo(serdeConstants.BINARY_TYPE_NAME); public static final TypeInfo decimalTypeInfo = getPrimitiveTypeInfo(serdeConstants.DECIMAL_TYPE_NAME); diff --git serde/src/test/org/apache/hadoop/hive/serde2/TestStatsSerde.java serde/src/test/org/apache/hadoop/hive/serde2/TestStatsSerde.java index 3ba2699..9aa3c45 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/TestStatsSerde.java +++ serde/src/test/org/apache/hadoop/hive/serde2/TestStatsSerde.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hive.serde2; -import java.math.BigInteger; +import java.sql.Date; import java.util.List; import java.util.Properties; import java.util.Random; @@ -114,13 +114,14 @@ public void testLazyBinarySerDe() throws Throwable { Double d = randField > 5 ? null : Double.valueOf(r.nextDouble()); String st = randField > 6 ? null : TestBinarySortableSerDe .getRandString(r); - HiveDecimal bd = randField > 8 ? null : TestBinarySortableSerDe.getRandHiveDecimal(r); + HiveDecimal bd = randField > 7 ? null : TestBinarySortableSerDe.getRandHiveDecimal(r); + Date date = randField > 8 ? null : TestBinarySortableSerDe.getRandDate(r); MyTestInnerStruct is = randField > 9 ? null : new MyTestInnerStruct(r .nextInt(5) - 2, r.nextInt(5) - 2); List li = randField > 10 ? null : TestBinarySortableSerDe .getRandIntegerArray(r); byte[] ba = TestBinarySortableSerDe.getRandBA(r, i); - MyTestClass t = new MyTestClass(b, s, n, l, f, d, st, bd, is, li,ba); + MyTestClass t = new MyTestClass(b, s, n, l, f, d, st, bd, date, is, li,ba); rows[i] = t; } diff --git serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestClass.java serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestClass.java index 0f364fd..d1d5760 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestClass.java +++ serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestClass.java @@ -17,10 +17,11 @@ */ package org.apache.hadoop.hive.serde2.binarysortable; -import org.apache.hadoop.hive.common.type.HiveDecimal; - +import java.sql.Date; import java.util.List; +import org.apache.hadoop.hive.common.type.HiveDecimal; + public class MyTestClass { Byte myByte; Short myShort; @@ -30,6 +31,7 @@ Double myDouble; String myString; HiveDecimal myDecimal; + Date myDate; MyTestInnerStruct myStruct; List myList; byte[] myBA; @@ -38,7 +40,7 @@ public MyTestClass() { } public MyTestClass(Byte b, Short s, Integer i, Long l, Float f, Double d, - String st, HiveDecimal bd, MyTestInnerStruct is, List li, byte[] ba) { + String st, HiveDecimal bd, Date date, MyTestInnerStruct is, List li, byte[] ba) { myByte = b; myShort = s; myInt = i; @@ -47,6 +49,7 @@ public MyTestClass(Byte b, Short s, Integer i, Long l, Float f, Double d, myDouble = d; myString = st; myDecimal = bd; + myDate = date; myStruct = is; myList = li; myBA = ba; diff --git serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableSerDe.java serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableSerDe.java index 229cdff..82feeec 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableSerDe.java +++ serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableSerDe.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.serde2.binarysortable; +import java.sql.Date; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -153,6 +154,15 @@ public static HiveDecimal getRandHiveDecimal(Random r) { return bd; } + public static Date getRandDate(Random r) { + String dateStr = String.format("%d-%02d-%02d", + Integer.valueOf(1800 + r.nextInt(500)), // year + Integer.valueOf(1 + r.nextInt(12)), // month + Integer.valueOf(1 + r.nextInt(28))); // day + Date dateVal = Date.valueOf(dateStr); + return dateVal; + } + public static String getRandString(Random r) { return getRandString(r, null, r.nextInt(10)); } @@ -196,7 +206,7 @@ public void testBinarySortableSerDe() throws Throwable { MyTestClass rows[] = new MyTestClass[num]; for (int i = 0; i < num; i++) { - int randField = r.nextInt(10); + int randField = r.nextInt(11); MyTestClass t = new MyTestClass(); t.myByte = randField > 0 ? null : Byte.valueOf((byte) r.nextInt()); t.myShort = randField > 1 ? null : Short.valueOf((short) r.nextInt()); @@ -208,9 +218,10 @@ public void testBinarySortableSerDe() throws Throwable { .valueOf(r.nextDouble() * 10 - 5); t.myString = randField > 6 ? null : getRandString(r); t.myDecimal = randField > 7 ? null : getRandHiveDecimal(r); - t.myStruct = randField > 8 ? null : new MyTestInnerStruct( + t.myDate = randField > 8 ? null : getRandDate(r); + t.myStruct = randField > 9 ? null : new MyTestInnerStruct( r.nextInt(5) - 2, r.nextInt(5) - 2); - t.myList = randField > 9 ? null : getRandIntegerArray(r); + t.myList = randField > 10 ? null : getRandIntegerArray(r); t.myBA = getRandBA(r, i); rows[i] = t; } @@ -224,9 +235,9 @@ public void testBinarySortableSerDe() throws Throwable { String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowOI); testBinarySortableSerDe(rows, rowOI, getSerDe(fieldNames, fieldTypes, - "+++++++++++"), true); + "++++++++++++"), true); testBinarySortableSerDe(rows, rowOI, getSerDe(fieldNames, fieldTypes, - "-----------"), false); + "------------"), false); System.out.println("Test testTBinarySortableProtocol passed!"); } catch (Throwable e) { diff --git serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassBigger.java serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassBigger.java index 8fa3c03..b6467ef 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassBigger.java +++ serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassBigger.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.serde2.lazybinary; +import java.sql.Date; import java.util.List; import java.util.Map; @@ -36,6 +37,7 @@ Double myDouble; String myString; HiveDecimal myDecimal; + Date myDate; MyTestInnerStruct myStruct; List myList; byte[] myBA; @@ -45,7 +47,7 @@ public MyTestClassBigger() { } public MyTestClassBigger(Byte b, Short s, Integer i, Long l, Float f, - Double d, String st, HiveDecimal bd, MyTestInnerStruct is, List li, + Double d, String st, HiveDecimal bd, Date date, MyTestInnerStruct is, List li, byte[] ba, Map> mp) { myByte = b; myShort = s; @@ -55,6 +57,7 @@ public MyTestClassBigger(Byte b, Short s, Integer i, Long l, Float f, myDouble = d; myString = st; myDecimal = bd; + myDate = date; myStruct = is; myList = li; myBA = ba; diff --git serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassSmaller.java serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassSmaller.java index 52e6145..8c7ffba 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassSmaller.java +++ serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassSmaller.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hive.serde2.lazybinary; +import java.sql.Date; + import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.serde2.binarysortable.MyTestInnerStruct; @@ -29,13 +31,14 @@ Double myDouble; String myString; HiveDecimal myDecimal; + Date myDate; MyTestInnerStruct myStruct; public MyTestClassSmaller() { } - + public MyTestClassSmaller(Byte b, Short s, Integer i, Long l, Float f, - Double d, String st, HiveDecimal bd, MyTestInnerStruct is) { + Double d, String st, HiveDecimal bd, Date date, MyTestInnerStruct is) { myByte = b; myShort = s; myInt = i; @@ -44,6 +47,7 @@ public MyTestClassSmaller(Byte b, Short s, Integer i, Long l, Float f, myDouble = d; myString = st; myDecimal = bd; + myDate = date; myStruct = is; } } diff --git serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java index b4f414e..954a4b8 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java +++ serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.serde2.lazybinary; +import java.sql.Date; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -192,7 +193,7 @@ private void testShorterSchemaDeserialization(Random r) throws Throwable { int num = 100; for (int itest = 0; itest < num; itest++) { - int randField = r.nextInt(10); + int randField = r.nextInt(11); Byte b = randField > 0 ? null : Byte.valueOf((byte) r.nextInt()); Short s = randField > 1 ? null : Short.valueOf((short) r.nextInt()); Integer n = randField > 2 ? null : Integer.valueOf(r.nextInt()); @@ -202,9 +203,10 @@ private void testShorterSchemaDeserialization(Random r) throws Throwable { String st = randField > 6 ? null : TestBinarySortableSerDe .getRandString(r); HiveDecimal bd = randField > 7 ? null : TestBinarySortableSerDe.getRandHiveDecimal(r); - MyTestInnerStruct is = randField > 7 ? null : new MyTestInnerStruct(r + Date date = randField > 8 ? null : TestBinarySortableSerDe.getRandDate(r); + MyTestInnerStruct is = randField > 9 ? null : new MyTestInnerStruct(r .nextInt(5) - 2, r.nextInt(5) - 2); - List li = randField > 8 ? null : TestBinarySortableSerDe + List li = randField > 10 ? null : TestBinarySortableSerDe .getRandIntegerArray(r); byte[] ba = TestBinarySortableSerDe.getRandBA(r, itest); Map> mp = new HashMap>(); @@ -218,7 +220,7 @@ private void testShorterSchemaDeserialization(Random r) throws Throwable { List value2 = getRandStructArray(r); mp.put(key2, value2); - MyTestClassBigger input = new MyTestClassBigger(b, s, n, l, f, d, st, bd, is, + MyTestClassBigger input = new MyTestClassBigger(b, s, n, l, f, d, st, bd, date, is, li, ba, mp); BytesWritable bw = (BytesWritable) serde1.serialize(input, rowOI1); Object output = serde2.deserialize(bw); @@ -260,7 +262,7 @@ private void testShorterSchemaDeserialization1(Random r) throws Throwable { int num = 100; for (int itest = 0; itest < num; itest++) { - int randField = r.nextInt(11); + int randField = r.nextInt(12); Byte b = randField > 0 ? null : Byte.valueOf((byte) r.nextInt()); Short s = randField > 1 ? null : Short.valueOf((short) r.nextInt()); Integer n = randField > 2 ? null : Integer.valueOf(r.nextInt()); @@ -270,12 +272,13 @@ private void testShorterSchemaDeserialization1(Random r) throws Throwable { String st = randField > 6 ? null : TestBinarySortableSerDe .getRandString(r); HiveDecimal bd = randField > 7 ? null : TestBinarySortableSerDe.getRandHiveDecimal(r); - MyTestInnerStruct is = randField > 8 ? null : new MyTestInnerStruct(r + Date date = randField > 8 ? null : TestBinarySortableSerDe.getRandDate(r); + MyTestInnerStruct is = randField > 9 ? null : new MyTestInnerStruct(r .nextInt(5) - 2, r.nextInt(5) - 2); - List li = randField > 9 ? null : TestBinarySortableSerDe + List li = randField > 10 ? null : TestBinarySortableSerDe .getRandIntegerArray(r); byte[] ba = TestBinarySortableSerDe.getRandBA(r, itest); - MyTestClass input = new MyTestClass(b, s, n, l, f, d, st, bd, is, li, ba); + MyTestClass input = new MyTestClass(b, s, n, l, f, d, st, bd, date, is, li, ba); BytesWritable bw = (BytesWritable) serde1.serialize(input, rowOI1); Object output = serde2.deserialize(bw); @@ -316,7 +319,7 @@ void testLongerSchemaDeserialization(Random r) throws Throwable { int num = 100; for (int itest = 0; itest < num; itest++) { - int randField = r.nextInt(11); + int randField = r.nextInt(12); Byte b = randField > 0 ? null : Byte.valueOf((byte) r.nextInt()); Short s = randField > 1 ? null : Short.valueOf((short) r.nextInt()); Integer n = randField > 2 ? null : Integer.valueOf(r.nextInt()); @@ -326,12 +329,13 @@ void testLongerSchemaDeserialization(Random r) throws Throwable { String st = randField > 6 ? null : TestBinarySortableSerDe .getRandString(r); HiveDecimal bd = randField > 7 ? null : TestBinarySortableSerDe.getRandHiveDecimal(r); - MyTestInnerStruct is = randField > 8 ? null : new MyTestInnerStruct(r + Date date = randField > 8 ? null : TestBinarySortableSerDe.getRandDate(r); + MyTestInnerStruct is = randField > 9 ? null : new MyTestInnerStruct(r .nextInt(5) - 2, r.nextInt(5) - 2); - List li = randField > 9 ? null : TestBinarySortableSerDe + List li = randField > 10 ? null : TestBinarySortableSerDe .getRandIntegerArray(r); byte[] ba = TestBinarySortableSerDe.getRandBA(r, itest); - MyTestClass input = new MyTestClass(b, s, n, l, f, d, st, bd, is, li,ba); + MyTestClass input = new MyTestClass(b, s, n, l, f, d, st, bd, date, is, li,ba); BytesWritable bw = (BytesWritable) serde1.serialize(input, rowOI1); Object output = serde2.deserialize(bw); @@ -382,10 +386,11 @@ void testLongerSchemaDeserialization1(Random r) throws Throwable { String st = randField > 6 ? null : TestBinarySortableSerDe .getRandString(r); HiveDecimal bd = randField > 7 ? null : TestBinarySortableSerDe.getRandHiveDecimal(r); + Date date = randField > 7 ? null : TestBinarySortableSerDe.getRandDate(r); MyTestInnerStruct is = randField > 7 ? null : new MyTestInnerStruct(r .nextInt(5) - 2, r.nextInt(5) - 2); - MyTestClassSmaller input = new MyTestClassSmaller(b, s, n, l, f, d, st, bd, + MyTestClassSmaller input = new MyTestClassSmaller(b, s, n, l, f, d, st, bd, date, is); BytesWritable bw = (BytesWritable) serde1.serialize(input, rowOI1); Object output = serde2.deserialize(bw); @@ -415,13 +420,13 @@ void testLazyBinaryMap(Random r) throws Throwable { StructObjectInspector soi1 = (StructObjectInspector) serdeOI; List fields1 = soi1.getAllStructFieldRefs(); LazyBinaryMapObjectInspector lazympoi = (LazyBinaryMapObjectInspector) fields1 - .get(11).getFieldObjectInspector(); + .get(12).getFieldObjectInspector(); ObjectInspector lazympkeyoi = lazympoi.getMapKeyObjectInspector(); ObjectInspector lazympvalueoi = lazympoi.getMapValueObjectInspector(); StructObjectInspector soi2 = rowOI; List fields2 = soi2.getAllStructFieldRefs(); - MapObjectInspector inputmpoi = (MapObjectInspector) fields2.get(11) + MapObjectInspector inputmpoi = (MapObjectInspector) fields2.get(12) .getFieldObjectInspector(); ObjectInspector inputmpkeyoi = inputmpoi.getMapKeyObjectInspector(); ObjectInspector inputmpvalueoi = inputmpoi.getMapValueObjectInspector(); @@ -441,10 +446,10 @@ void testLazyBinaryMap(Random r) throws Throwable { } MyTestClassBigger input = new MyTestClassBigger(null, null, null, null, - null, null, null, null, null, null, null, mp); + null, null, null, null, null, null, null, null, mp); BytesWritable bw = (BytesWritable) serde.serialize(input, rowOI); Object output = serde.deserialize(bw); - Object lazyobj = soi1.getStructFieldData(output, fields1.get(11)); + Object lazyobj = soi1.getStructFieldData(output, fields1.get(12)); Map outputmp = lazympoi.getMap(lazyobj); if (outputmp.size() != mp.size()) { @@ -491,7 +496,7 @@ public void testLazyBinarySerDe() throws Throwable { Random r = new Random(1234); MyTestClass rows[] = new MyTestClass[num]; for (int i = 0; i < num; i++) { - int randField = r.nextInt(11); + int randField = r.nextInt(12); Byte b = randField > 0 ? null : Byte.valueOf((byte) r.nextInt()); Short s = randField > 1 ? null : Short.valueOf((short) r.nextInt()); Integer n = randField > 2 ? null : Integer.valueOf(r.nextInt()); @@ -500,13 +505,14 @@ public void testLazyBinarySerDe() throws Throwable { Double d = randField > 5 ? null : Double.valueOf(r.nextDouble()); String st = randField > 6 ? null : TestBinarySortableSerDe .getRandString(r); - HiveDecimal bd = randField > 7 ? null : TestBinarySortableSerDe.getRandHiveDecimal(r); - MyTestInnerStruct is = randField > 8 ? null : new MyTestInnerStruct(r + HiveDecimal bd = randField > 7 ? null : TestBinarySortableSerDe.getRandHiveDecimal(r); + Date date = randField > 8 ? null : TestBinarySortableSerDe.getRandDate(r); + MyTestInnerStruct is = randField > 9 ? null : new MyTestInnerStruct(r .nextInt(5) - 2, r.nextInt(5) - 2); - List li = randField > 9 ? null : TestBinarySortableSerDe + List li = randField > 10 ? null : TestBinarySortableSerDe .getRandIntegerArray(r); byte[] ba = TestBinarySortableSerDe.getRandBA(r, i); - MyTestClass t = new MyTestClass(b, s, n, l, f, d, st, bd, is, li, ba); + MyTestClass t = new MyTestClass(b, s, n, l, f, d, st, bd, date, is, li, ba); rows[i] = t; } diff --git service/README.txt service/README.txt new file mode 100644 index 0000000..0cc5195 --- /dev/null +++ service/README.txt @@ -0,0 +1,7 @@ + +Thrift commands to generate files from TCLIService.thrift: +-------------------- +thrift --gen java:beans,hashcode -o src/gen/thrift if/TCLIService.thrift +thrift --gen cpp -o src/gen/thrift if/TCLIService.thrift +thrift --gen py -o src/gen/thrift if/TCLIService.thrift +thrift --gen rb -o src/gen/thrift if/TCLIService.thrift diff --git service/if/TCLIService.thrift service/if/TCLIService.thrift index 3ddb453..8dc2a90 100644 --- service/if/TCLIService.thrift +++ service/if/TCLIService.thrift @@ -59,6 +59,7 @@ enum TTypeId { USER_DEFINED_TYPE, DECIMAL_TYPE, NULL_TYPE, + DATE_TYPE } const set PRIMITIVE_TYPES = [ @@ -74,6 +75,7 @@ const set PRIMITIVE_TYPES = [ TTypeId.BINARY_TYPE, TTypeId.DECIMAL_TYPE, TTypeId.NULL_TYPE + TTypeId.DATE_TYPE ] const set COMPLEX_TYPES = [ @@ -106,6 +108,7 @@ const map TYPE_NAMES = { TTypeId.UNION_TYPE: "UNIONTYPE", TTypeId.DECIMAL_TYPE: "DECIMAL", TTypeId.NULL_TYPE: "NULL" + TTypeId.DATE_TYPE: "DATE" } // Thrift does not support recursively defined types or forward declarations, diff --git service/src/gen/thrift/gen-cpp/TCLIService_constants.cpp service/src/gen/thrift/gen-cpp/TCLIService_constants.cpp index c16e186..7284cc5 100644 --- service/src/gen/thrift/gen-cpp/TCLIService_constants.cpp +++ service/src/gen/thrift/gen-cpp/TCLIService_constants.cpp @@ -23,6 +23,7 @@ TCLIServiceConstants::TCLIServiceConstants() { PRIMITIVE_TYPES.insert((TTypeId::type)9); PRIMITIVE_TYPES.insert((TTypeId::type)15); PRIMITIVE_TYPES.insert((TTypeId::type)16); + PRIMITIVE_TYPES.insert((TTypeId::type)17); COMPLEX_TYPES.insert((TTypeId::type)10); COMPLEX_TYPES.insert((TTypeId::type)11); @@ -49,6 +50,7 @@ TCLIServiceConstants::TCLIServiceConstants() { TYPE_NAMES.insert(std::make_pair((TTypeId::type)13, "UNIONTYPE")); TYPE_NAMES.insert(std::make_pair((TTypeId::type)15, "DECIMAL")); TYPE_NAMES.insert(std::make_pair((TTypeId::type)16, "NULL")); + TYPE_NAMES.insert(std::make_pair((TTypeId::type)17, "DATE")); } diff --git service/src/gen/thrift/gen-cpp/TCLIService_types.cpp service/src/gen/thrift/gen-cpp/TCLIService_types.cpp index a1314b7..f9c9fb9 100644 --- service/src/gen/thrift/gen-cpp/TCLIService_types.cpp +++ service/src/gen/thrift/gen-cpp/TCLIService_types.cpp @@ -35,7 +35,8 @@ int _kTTypeIdValues[] = { TTypeId::UNION_TYPE, TTypeId::USER_DEFINED_TYPE, TTypeId::DECIMAL_TYPE, - TTypeId::NULL_TYPE + TTypeId::NULL_TYPE, + TTypeId::DATE_TYPE }; const char* _kTTypeIdNames[] = { "BOOLEAN_TYPE", @@ -54,9 +55,10 @@ const char* _kTTypeIdNames[] = { "UNION_TYPE", "USER_DEFINED_TYPE", "DECIMAL_TYPE", - "NULL_TYPE" + "NULL_TYPE", + "DATE_TYPE" }; -const std::map _TTypeId_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(17, _kTTypeIdValues, _kTTypeIdNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL)); +const std::map _TTypeId_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(18, _kTTypeIdValues, _kTTypeIdNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL)); int _kTStatusCodeValues[] = { TStatusCode::SUCCESS_STATUS, diff --git service/src/gen/thrift/gen-cpp/TCLIService_types.h service/src/gen/thrift/gen-cpp/TCLIService_types.h index b791ede..f670179 100644 --- service/src/gen/thrift/gen-cpp/TCLIService_types.h +++ service/src/gen/thrift/gen-cpp/TCLIService_types.h @@ -42,7 +42,8 @@ struct TTypeId { UNION_TYPE = 13, USER_DEFINED_TYPE = 14, DECIMAL_TYPE = 15, - NULL_TYPE = 16 + NULL_TYPE = 16, + DATE_TYPE = 17 }; }; diff --git service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TCLIServiceConstants.java service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TCLIServiceConstants.java index d64dd2b..83b73fa 100644 --- service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TCLIServiceConstants.java +++ service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TCLIServiceConstants.java @@ -47,6 +47,7 @@ PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.BINARY_TYPE); PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.DECIMAL_TYPE); PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.NULL_TYPE); + PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.DATE_TYPE); } public static final Set COMPLEX_TYPES = new HashSet(); @@ -82,6 +83,7 @@ TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.UNION_TYPE, "UNIONTYPE"); TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.DECIMAL_TYPE, "DECIMAL"); TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.NULL_TYPE, "NULL"); + TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.DATE_TYPE, "DATE"); } } diff --git service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeId.java service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeId.java index d2b539d..aecc59e 100644 --- service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeId.java +++ service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeId.java @@ -28,7 +28,8 @@ UNION_TYPE(13), USER_DEFINED_TYPE(14), DECIMAL_TYPE(15), - NULL_TYPE(16); + NULL_TYPE(16), + DATE_TYPE(17); private final int value; @@ -83,6 +84,8 @@ public static TTypeId findByValue(int value) { return DECIMAL_TYPE; case 16: return NULL_TYPE; + case 17: + return DATE_TYPE; default: return null; } diff --git service/src/gen/thrift/gen-py/TCLIService/TCLIService-remote service/src/gen/thrift/gen-py/TCLIService/TCLIService-remote old mode 100644 new mode 100755 diff --git service/src/gen/thrift/gen-py/TCLIService/constants.py service/src/gen/thrift/gen-py/TCLIService/constants.py index 09850d7..a81196b 100644 --- service/src/gen/thrift/gen-py/TCLIService/constants.py +++ service/src/gen/thrift/gen-py/TCLIService/constants.py @@ -22,6 +22,7 @@ 9, 15, 16, + 17, ]) COMPLEX_TYPES = set([ 10, @@ -51,4 +52,5 @@ 13 : "UNIONTYPE", 15 : "DECIMAL", 16 : "NULL", + 17 : "DATE", } diff --git service/src/gen/thrift/gen-py/TCLIService/ttypes.py service/src/gen/thrift/gen-py/TCLIService/ttypes.py index 97eb8ec..bf72ad3 100644 --- service/src/gen/thrift/gen-py/TCLIService/ttypes.py +++ service/src/gen/thrift/gen-py/TCLIService/ttypes.py @@ -45,6 +45,7 @@ class TTypeId: USER_DEFINED_TYPE = 14 DECIMAL_TYPE = 15 NULL_TYPE = 16 + DATE_TYPE = 17 _VALUES_TO_NAMES = { 0: "BOOLEAN_TYPE", @@ -64,6 +65,7 @@ class TTypeId: 14: "USER_DEFINED_TYPE", 15: "DECIMAL_TYPE", 16: "NULL_TYPE", + 17: "DATE_TYPE", } _NAMES_TO_VALUES = { @@ -84,6 +86,7 @@ class TTypeId: "USER_DEFINED_TYPE": 14, "DECIMAL_TYPE": 15, "NULL_TYPE": 16, + "DATE_TYPE": 17, } class TStatusCode: diff --git service/src/gen/thrift/gen-rb/t_c_l_i_service_constants.rb service/src/gen/thrift/gen-rb/t_c_l_i_service_constants.rb index 2f0fede..f576689 100644 --- service/src/gen/thrift/gen-rb/t_c_l_i_service_constants.rb +++ service/src/gen/thrift/gen-rb/t_c_l_i_service_constants.rb @@ -20,6 +20,7 @@ PRIMITIVE_TYPES = Set.new([ 9, 15, 16, + 17, ]) COMPLEX_TYPES = Set.new([ @@ -52,5 +53,6 @@ TYPE_NAMES = { 13 => %q"UNIONTYPE", 15 => %q"DECIMAL", 16 => %q"NULL", + 17 => %q"DATE", } diff --git service/src/gen/thrift/gen-rb/t_c_l_i_service_types.rb service/src/gen/thrift/gen-rb/t_c_l_i_service_types.rb index 885ca2d..1da48ca 100644 --- service/src/gen/thrift/gen-rb/t_c_l_i_service_types.rb +++ service/src/gen/thrift/gen-rb/t_c_l_i_service_types.rb @@ -30,8 +30,9 @@ module TTypeId USER_DEFINED_TYPE = 14 DECIMAL_TYPE = 15 NULL_TYPE = 16 - VALUE_MAP = {0 => "BOOLEAN_TYPE", 1 => "TINYINT_TYPE", 2 => "SMALLINT_TYPE", 3 => "INT_TYPE", 4 => "BIGINT_TYPE", 5 => "FLOAT_TYPE", 6 => "DOUBLE_TYPE", 7 => "STRING_TYPE", 8 => "TIMESTAMP_TYPE", 9 => "BINARY_TYPE", 10 => "ARRAY_TYPE", 11 => "MAP_TYPE", 12 => "STRUCT_TYPE", 13 => "UNION_TYPE", 14 => "USER_DEFINED_TYPE", 15 => "DECIMAL_TYPE", 16 => "NULL_TYPE"} - VALID_VALUES = Set.new([BOOLEAN_TYPE, TINYINT_TYPE, SMALLINT_TYPE, INT_TYPE, BIGINT_TYPE, FLOAT_TYPE, DOUBLE_TYPE, STRING_TYPE, TIMESTAMP_TYPE, BINARY_TYPE, ARRAY_TYPE, MAP_TYPE, STRUCT_TYPE, UNION_TYPE, USER_DEFINED_TYPE, DECIMAL_TYPE, NULL_TYPE]).freeze + DATE_TYPE = 17 + VALUE_MAP = {0 => "BOOLEAN_TYPE", 1 => "TINYINT_TYPE", 2 => "SMALLINT_TYPE", 3 => "INT_TYPE", 4 => "BIGINT_TYPE", 5 => "FLOAT_TYPE", 6 => "DOUBLE_TYPE", 7 => "STRING_TYPE", 8 => "TIMESTAMP_TYPE", 9 => "BINARY_TYPE", 10 => "ARRAY_TYPE", 11 => "MAP_TYPE", 12 => "STRUCT_TYPE", 13 => "UNION_TYPE", 14 => "USER_DEFINED_TYPE", 15 => "DECIMAL_TYPE", 16 => "NULL_TYPE", 17 => "DATE_TYPE"} + VALID_VALUES = Set.new([BOOLEAN_TYPE, TINYINT_TYPE, SMALLINT_TYPE, INT_TYPE, BIGINT_TYPE, FLOAT_TYPE, DOUBLE_TYPE, STRING_TYPE, TIMESTAMP_TYPE, BINARY_TYPE, ARRAY_TYPE, MAP_TYPE, STRUCT_TYPE, UNION_TYPE, USER_DEFINED_TYPE, DECIMAL_TYPE, NULL_TYPE, DATE_TYPE]).freeze end module TStatusCode diff --git service/src/java/org/apache/hive/service/cli/ColumnValue.java service/src/java/org/apache/hive/service/cli/ColumnValue.java index 9351e7b..75b5407 100644 --- service/src/java/org/apache/hive/service/cli/ColumnValue.java +++ service/src/java/org/apache/hive/service/cli/ColumnValue.java @@ -18,6 +18,7 @@ package org.apache.hive.service.cli; +import java.sql.Date; import java.sql.Timestamp; import org.apache.hadoop.hive.common.type.HiveDecimal; @@ -117,6 +118,14 @@ public static ColumnValue stringValue(String value) { return new ColumnValue(TColumnValue.stringVal(tStringValue)); } + public static ColumnValue dateValue(Date value) { + TStringValue tStringValue = new TStringValue(); + if (value != null) { + tStringValue.setValue(value.toString()); + } + return new ColumnValue(TColumnValue.stringVal(tStringValue)); + } + public static ColumnValue timestampValue(Timestamp value) { TStringValue tStringValue = new TStringValue(); if (value != null) { @@ -151,6 +160,8 @@ public static ColumnValue newColumnValue(Type type, Object value) { return doubleValue((Double)value); case STRING_TYPE: return stringValue((String)value); + case DATE_TYPE: + return dateValue((Date)value); case TIMESTAMP_TYPE: return timestampValue((Timestamp)value); case DECIMAL_TYPE: diff --git service/src/java/org/apache/hive/service/cli/Type.java service/src/java/org/apache/hive/service/cli/Type.java index 53f6807..3423262 100644 --- service/src/java/org/apache/hive/service/cli/Type.java +++ service/src/java/org/apache/hive/service/cli/Type.java @@ -54,6 +54,9 @@ STRING_TYPE("STRING", java.sql.Types.VARCHAR, TTypeId.STRING_TYPE), + DATE_TYPE("DATE", + java.sql.Types.DATE, + TTypeId.DATE_TYPE), TIMESTAMP_TYPE("TIMESTAMP", java.sql.Types.TIMESTAMP, TTypeId.TIMESTAMP_TYPE), @@ -213,6 +216,7 @@ public Integer getScale() { switch (this) { case BOOLEAN_TYPE: case STRING_TYPE: + case DATE_TYPE: case TIMESTAMP_TYPE: case TINYINT_TYPE: case SMALLINT_TYPE: @@ -247,6 +251,8 @@ public Integer getColumnSize() { case STRING_TYPE: case BINARY_TYPE: return Integer.MAX_VALUE; + case DATE_TYPE: + return 10; case TIMESTAMP_TYPE: return 30; default: