diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java index 8d3a84f..56851b6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java @@ -57,10 +57,6 @@ import org.apache.hadoop.hive.ql.udf.UDFCeil; import org.apache.hadoop.hive.ql.udf.UDFConv; import org.apache.hadoop.hive.ql.udf.UDFCos; -import org.apache.hadoop.hive.ql.udf.UDFDate; -import org.apache.hadoop.hive.ql.udf.UDFDateAdd; -import org.apache.hadoop.hive.ql.udf.UDFDateDiff; -import org.apache.hadoop.hive.ql.udf.UDFDateSub; import org.apache.hadoop.hive.ql.udf.UDFDayOfMonth; import org.apache.hadoop.hive.ql.udf.UDFDegrees; import org.apache.hadoop.hive.ql.udf.UDFE; @@ -274,12 +270,12 @@ registerUDF("minute", UDFMinute.class, false); registerUDF("second", UDFSecond.class, false); registerUDF("from_unixtime", UDFFromUnixTime.class, false); - registerUDF("to_date", UDFDate.class, false); + registerGenericUDF("to_date", GenericUDFDate.class); registerUDF("weekofyear", UDFWeekOfYear.class, false); - registerUDF("date_add", UDFDateAdd.class, false); - registerUDF("date_sub", UDFDateSub.class, false); - registerUDF("datediff", UDFDateDiff.class, false); + registerGenericUDF("date_add", GenericUDFDateAdd.class); + registerGenericUDF("date_sub", GenericUDFDateSub.class); + registerGenericUDF("datediff", GenericUDFDateDiff.class); registerUDF("get_json_object", UDFJson.class, false); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDate.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDate.java deleted file mode 100644 index 3df453c..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDate.java +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.udf; - -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; - -import org.apache.hadoop.hive.ql.exec.Description; -import org.apache.hadoop.hive.ql.exec.UDF; -import org.apache.hadoop.hive.serde2.io.DateWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; -import org.apache.hadoop.io.Text; - -/** - * UDFDate. - * - */ -@Description(name = "to_date", - value = "_FUNC_(expr) - Extracts the date part of the date or datetime expression expr", - extended = "Example:\n " - + " > SELECT _FUNC_('2009-30-07 04:17:52') FROM src LIMIT 1;\n" - + " '2009-30-07'") -public class UDFDate extends UDF { - private final SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); - - private Text t = new Text(); - - public UDFDate() { - } - - /** - * Get the date part of a date time string. - * - * @param dateString - * the date string in the format of "yyyy-MM-dd HH:mm:ss" or - * "yyyy-MM-dd". - * @return the date in the format of "yyyy-MM-dd". - */ - public Text evaluate(Text dateString) { - - if (dateString == null) { - return null; - } - - try { - Date date = formatter.parse(dateString.toString()); - t.set(formatter.format(date)); - return t; - } catch (ParseException e) { - return null; - } - } - - public Text evaluate(DateWritable d) { - if (d == null) { - return null; - } - - t.set(formatter.format(d.get())); - return t; - } - - public Text evaluate(TimestampWritable i) { - if (i == null) { - return null; - } - - t.set(formatter.format(i.getTimestamp())); - return t; - } - -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateAdd.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateAdd.java deleted file mode 100644 index b1b0bf2..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateAdd.java +++ /dev/null @@ -1,108 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.udf; - -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; - -import org.apache.hadoop.hive.ql.exec.Description; -import org.apache.hadoop.hive.ql.exec.UDF; -import org.apache.hadoop.hive.serde2.io.DateWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.Text; - -/** - * UDFDateAdd. - * - */ -@Description(name = "date_add", - value = "_FUNC_(start_date, num_days) - Returns the date that is num_days after start_date.", - extended = "start_date is a string in the format 'yyyy-MM-dd HH:mm:ss' or" - + " 'yyyy-MM-dd'. num_days is a number. The time part of start_date is " - + "ignored.\n" - + "Example:\n " - + " > SELECT _FUNC_('2009-30-07', 1) FROM src LIMIT 1;\n" - + " '2009-31-07'") -public class UDFDateAdd extends UDF { - private final SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); - private final Calendar calendar = Calendar.getInstance(); - - private Text result = new Text(); - - public UDFDateAdd() { - } - - /** - * Add a number of days to the date. The time part of the string will be - * ignored. - * - * NOTE: This is a subset of what MySQL offers as: - * http://dev.mysql.com/doc/refman - * /5.1/en/date-and-time-functions.html#function_date-add - * - * @param dateString1 - * the date string in the format of "yyyy-MM-dd HH:mm:ss" or - * "yyyy-MM-dd". - * @param days - * The number of days to add. - * @return the date in the format of "yyyy-MM-dd". - */ - public Text evaluate(Text dateString1, IntWritable days) { - - if (dateString1 == null || days == null) { - return null; - } - - try { - calendar.setTime(formatter.parse(dateString1.toString())); - calendar.add(Calendar.DAY_OF_MONTH, days.get()); - Date newDate = calendar.getTime(); - result.set(formatter.format(newDate)); - return result; - } catch (ParseException e) { - return null; - } - } - - public Text evaluate(DateWritable d, IntWritable days) { - if (d == null || days == null) { - return null; - } - calendar.setTime(d.get()); - calendar.add(Calendar.DAY_OF_MONTH, days.get()); - Date newDate = calendar.getTime(); - result.set(formatter.format(newDate)); - return result; - } - - public Text evaluate(TimestampWritable t, IntWritable days) { - if (t == null || days == null) { - return null; - } - calendar.setTime(t.getTimestamp()); - calendar.add(Calendar.DAY_OF_MONTH, days.get()); - Date newDate = calendar.getTime(); - result.set(formatter.format(newDate)); - return result; - } - -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateDiff.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateDiff.java deleted file mode 100644 index da14c4f..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateDiff.java +++ /dev/null @@ -1,138 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.udf; - -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.TimeZone; - -import org.apache.hadoop.hive.ql.exec.Description; -import org.apache.hadoop.hive.ql.exec.UDF; -import org.apache.hadoop.hive.serde2.io.DateWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.Text; - -/** - * UDFDateDiff. - * - */ -@Description(name = "datediff", - value = "_FUNC_(date1, date2) - Returns the number of days between date1 and date2", - extended = "date1 and date2 are strings in the format " - + "'yyyy-MM-dd HH:mm:ss' or 'yyyy-MM-dd'. The time parts are ignored." - + "If date1 is earlier than date2, the result is negative.\n" - + "Example:\n " - + " > SELECT _FUNC_('2009-30-07', '2009-31-07') FROM src LIMIT 1;\n" - + " 1") -public class UDFDateDiff extends UDF { - private final SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); - - private final IntWritable result = new IntWritable(); - - public UDFDateDiff() { - formatter.setTimeZone(TimeZone.getTimeZone("UTC")); - } - - /** - * Calculate the difference in the number of days. The time part of the string - * will be ignored. If dateString1 is earlier than dateString2, then the - * result can be negative. - * - * @param dateString1 - * the date string in the format of "yyyy-MM-dd HH:mm:ss" or - * "yyyy-MM-dd". - * @param dateString2 - * the date string in the format of "yyyy-MM-dd HH:mm:ss" or - * "yyyy-MM-dd". - * @return the difference in days. - */ - public IntWritable evaluate(Text dateString1, Text dateString2) { - return evaluate(toDate(dateString1), toDate(dateString2)); - } - - public IntWritable evaluate(TimestampWritable t1, TimestampWritable t2) { - return evaluate(toDate(t1), toDate(t2)); - } - - public IntWritable evaluate(TimestampWritable t, Text dateString) { - return evaluate(toDate(t), toDate(dateString)); - } - - public IntWritable evaluate(Text dateString, TimestampWritable t) { - return evaluate(toDate(dateString), toDate(t)); - } - - public IntWritable evaluate(Text dateString, DateWritable d) { - return evaluate(toDate(dateString), d.get()); - } - - public IntWritable evaluate(TimestampWritable t, DateWritable d) { - return evaluate(toDate(t), d.get()); - } - - public IntWritable evaluate(DateWritable d1, DateWritable d2) { - return evaluate(d1.get(), d2.get()); - } - - public IntWritable evaluate(DateWritable d, Text dateString) { - return evaluate(d.get(), toDate(dateString)); - } - - public IntWritable evaluate(DateWritable d, TimestampWritable t) { - return evaluate(d.get(), toDate(t)); - } - - private IntWritable evaluate(Date date, Date date2) { - if (date == null || date2 == null) { - return null; - } - - // NOTE: This implementation avoids the extra-second problem - // by comparing with UTC epoch and integer division. - // 86400 is the number of seconds in a day - long diffInMilliSeconds = date.getTime() - date2.getTime(); - result.set((int) (diffInMilliSeconds / (86400 * 1000))); - return result; - } - - private Date format(String dateString) { - try { - return formatter.parse(dateString); - } catch (ParseException e) { - return null; - } - } - - private Date toDate(Text dateString) { - if (dateString == null) { - return null; - } - return format(dateString.toString()); - } - - private Date toDate(TimestampWritable t) { - if (t == null) { - return null; - } - return t.getTimestamp(); - } - -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateSub.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateSub.java deleted file mode 100644 index c8a1d1f..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateSub.java +++ /dev/null @@ -1,108 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.udf; - -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; - -import org.apache.hadoop.hive.ql.exec.Description; -import org.apache.hadoop.hive.ql.exec.UDF; -import org.apache.hadoop.hive.serde2.io.DateWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.Text; - -/** - * UDFDateSub. - * - */ -@Description(name = "date_sub", - value = "_FUNC_(start_date, num_days) - Returns the date that is num_days before start_date.", - extended = "start_date is a string in the format 'yyyy-MM-dd HH:mm:ss' or" - + " 'yyyy-MM-dd'. num_days is a number. The time part of start_date is " - + "ignored.\n" - + "Example:\n " - + " > SELECT _FUNC_('2009-30-07', 1) FROM src LIMIT 1;\n" - + " '2009-29-07'") -public class UDFDateSub extends UDF { - private final SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); - private final Calendar calendar = Calendar.getInstance(); - - private Text result = new Text(); - - public UDFDateSub() { - } - - /** - * Subtract a number of days to the date. The time part of the string will be - * ignored. - * - * NOTE: This is a subset of what MySQL offers as: - * http://dev.mysql.com/doc/refman - * /5.1/en/date-and-time-functions.html#function_date-sub - * - * @param dateString1 - * the date string in the format of "yyyy-MM-dd HH:mm:ss" or - * "yyyy-MM-dd". - * @param days - * the number of days to subtract. - * @return the date in the format of "yyyy-MM-dd". - */ - public Text evaluate(Text dateString1, IntWritable days) { - - if (dateString1 == null || days == null) { - return null; - } - - try { - calendar.setTime(formatter.parse(dateString1.toString())); - calendar.add(Calendar.DAY_OF_MONTH, -days.get()); - Date newDate = calendar.getTime(); - result.set(formatter.format(newDate)); - return result; - } catch (ParseException e) { - return null; - } - } - - public Text evaluate(DateWritable d, IntWritable days) { - if (d == null || days == null) { - return null; - } - calendar.setTime(d.get()); - calendar.add(Calendar.DAY_OF_MONTH, -days.get()); - Date newDate = calendar.getTime(); - result.set(formatter.format(newDate)); - return result; - } - - public Text evaluate(TimestampWritable t, IntWritable days) { - if (t == null || days == null) { - return null; - } - calendar.setTime(t.getTimestamp()); - calendar.add(Calendar.DAY_OF_MONTH, -days.get()); - Date newDate = calendar.getTime(); - result.set(formatter.format(newDate)); - return result; - } - -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDate.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDate.java new file mode 100644 index 0000000..53b0bdb --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDate.java @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.udf.generic; + +import java.sql.Timestamp; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter.TimestampConverter; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.Text; + +/** + * UDFDate. + * + */ +@Description(name = "to_date", + value = "_FUNC_(expr) - Extracts the date part of the date or datetime expression expr", + extended = "Example:\n " + + " > SELECT _FUNC_('2009-07-30 04:17:52') FROM src LIMIT 1;\n" + + " '2009-07-30'") +public class GenericUDFDate extends GenericUDF { + private transient SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); + private transient TimestampConverter timestampConverter; + private transient Converter textConverter; + private transient Converter dateWritableConverter; + private transient PrimitiveCategory inputType; + private transient PrimitiveObjectInspector argumentOI; + private final Text output = new Text(); + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + if (arguments.length != 1) { + throw new UDFArgumentLengthException( + "to_date() requires 1 argument, got " + arguments.length); + } + argumentOI = (PrimitiveObjectInspector) arguments[0]; + inputType = argumentOI.getPrimitiveCategory(); + ObjectInspector outputOI = PrimitiveObjectInspectorFactory.writableStringObjectInspector; + switch (inputType) { + case STRING: + // textConverter = new TextConverter(argumentOI); + textConverter = ObjectInspectorConverters.getConverter( + argumentOI, PrimitiveObjectInspectorFactory.writableStringObjectInspector); + break; + case TIMESTAMP: + timestampConverter = new TimestampConverter(argumentOI, + PrimitiveObjectInspectorFactory.writableTimestampObjectInspector); + break; + case DATE: + dateWritableConverter = ObjectInspectorConverters.getConverter(argumentOI, + PrimitiveObjectInspectorFactory.writableDateObjectInspector); + break; + default: + throw new UDFArgumentException( + "TO_DATE() only takes STRING/TIMESTAMP/DATEWRITABLE types, got " + inputType); + } + return outputOI; + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + switch (inputType) { + case STRING: + Date date; + String dateString = textConverter.convert(arguments[0].get()).toString(); + try { + date = formatter.parse(dateString); + } catch (ParseException e) { + return null; + } + output.set(formatter.format(date)); + break; + case TIMESTAMP: + Timestamp ts = ((TimestampWritable) timestampConverter.convert(arguments[0].get())) + .getTimestamp(); + output.set(formatter.format(ts)); + break; + case DATE: + DateWritable dw = (DateWritable) dateWritableConverter.convert(arguments[0].get()); + output.set(formatter.format(dw.get())); + break; + default: + throw new UDFArgumentException( + "TO_DATE() only takes STRING/TIMESTAMP/DATEWRITABLE types, got " + inputType); + } + return output; + } + + @Override + public String getDisplayString(String[] children) { + StringBuilder sb = new StringBuilder(); + sb.append("to_date("); + if (children.length > 0) { + sb.append(children[0]); + for (int i = 1; i < children.length; i++) { + sb.append(", "); + sb.append(children[i]); + } + } + sb.append(")"); + return sb.toString(); + } + +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDateAdd.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDateAdd.java new file mode 100644 index 0000000..a3fe770 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDateAdd.java @@ -0,0 +1,172 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.udf.generic; + +import java.sql.Timestamp; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter.TimestampConverter; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.IntWritable; + +/** + * UDFDateAdd. + * + * Add a number of days to the date. The time part of the string will be + * ignored. + * + * NOTE: This is a subset of what MySQL offers as: + * http://dev.mysql.com/doc/refman + * /5.1/en/date-and-time-functions.html#function_date-add + * + */ +@Description(name = "date_add", + value = "_FUNC_(start_date, num_days) - Returns the date that is num_days after start_date.", + extended = "start_date is a string in the format 'yyyy-MM-dd HH:mm:ss' or" + + " 'yyyy-MM-dd'. num_days is a number. The time part of start_date is " + + "ignored.\n" + + "Example:\n " + + " > SELECT _FUNC_('2009-30-07', 1) FROM src LIMIT 1;\n" + + " '2009-31-07'") +public class GenericUDFDateAdd extends GenericUDF { + private transient SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); + private transient TimestampConverter timestampConverter; + private transient Converter textConverter; + private transient Converter dateWritableConverter; + private transient Converter intWritableConverter; + private transient PrimitiveCategory inputType1; + private transient PrimitiveCategory inputType2; + private final Calendar calendar = Calendar.getInstance(); + private final Text output = new Text(); + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + if (arguments.length != 2) { + throw new UDFArgumentLengthException( + "date_add() requires 2 argument, got " + arguments.length); + } + if (arguments[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { + throw new UDFArgumentTypeException(0, + "Only primitive type arguments are accepted but " + + arguments[0].getTypeName() + " is passed. as first arguments"); + } + if (arguments[1].getCategory() != ObjectInspector.Category.PRIMITIVE) { + throw new UDFArgumentTypeException(1, + "Only primitive type arguments are accepted but " + + arguments[2].getTypeName() + " is passed. as second arguments"); + } + + inputType1 = ((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory(); + ObjectInspector outputOI = PrimitiveObjectInspectorFactory.writableStringObjectInspector; + switch (inputType1) { + case STRING: + textConverter = ObjectInspectorConverters.getConverter( + (PrimitiveObjectInspector) arguments[0], + PrimitiveObjectInspectorFactory.writableStringObjectInspector); + break; + case TIMESTAMP: + timestampConverter = new TimestampConverter((PrimitiveObjectInspector) arguments[0], + PrimitiveObjectInspectorFactory.writableTimestampObjectInspector); + break; + case DATE: + dateWritableConverter = ObjectInspectorConverters.getConverter( + (PrimitiveObjectInspector) arguments[0], + PrimitiveObjectInspectorFactory.writableDateObjectInspector); + break; + default: + throw new UDFArgumentException( + " DATE_ADD() only takes STRING/TIMESTAMP/DATEWRITABLE types as first argument, got " + + inputType1); + } + + inputType2 = ((PrimitiveObjectInspector) arguments[1]).getPrimitiveCategory(); + if (inputType2 != PrimitiveCategory.INT) { + throw new UDFArgumentException( + " DATE_ADD() only takes INT types as second argument, got " + inputType2); + } + intWritableConverter = ObjectInspectorConverters.getConverter( + (PrimitiveObjectInspector) arguments[1], + PrimitiveObjectInspectorFactory.writableIntObjectInspector); + return outputOI; + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + + IntWritable toBeAdded = (IntWritable) intWritableConverter.convert(arguments[1].get()); + switch (inputType1) { + case STRING: + String dateString = textConverter.convert(arguments[0].get()).toString(); + try { + calendar.setTime(formatter.parse(dateString.toString())); + calendar.add(Calendar.DAY_OF_MONTH, toBeAdded.get()); + } catch (ParseException e) { + return null; + } + break; + case TIMESTAMP: + Timestamp ts = ((TimestampWritable) timestampConverter.convert(arguments[0].get())) + .getTimestamp(); + calendar.setTime(ts); + calendar.add(Calendar.DAY_OF_MONTH, toBeAdded.get()); + break; + case DATE: + DateWritable dw = (DateWritable) dateWritableConverter.convert(arguments[0].get()); + calendar.setTime(dw.get()); + calendar.add(Calendar.DAY_OF_MONTH, toBeAdded.get()); + break; + default: + throw new UDFArgumentException( + "DATE_ADD() only takes STRING/TIMESTAMP/DATEWRITABLE types, got " + inputType1); + } + Date newDate = calendar.getTime(); + output.set(formatter.format(newDate)); + return output; + } + + @Override + public String getDisplayString(String[] children) { + StringBuilder sb = new StringBuilder(); + sb.append("date_add("); + if (children.length > 0) { + sb.append(children[0]); + for (int i = 1; i < children.length; i++) { + sb.append(", "); + sb.append(children[i]); + } + } + sb.append(")"); + return sb.toString(); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDateDiff.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDateDiff.java new file mode 100644 index 0000000..c88b392 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDateDiff.java @@ -0,0 +1,181 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.udf.generic; + +import java.sql.Timestamp; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.TimeZone; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter.TimestampConverter; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.IntWritable; + +/** + * UDFDateDiff. + * + * Calculate the difference in the number of days. The time part of the string + * will be ignored. If dateString1 is earlier than dateString2, then the + * result can be negative. + * + */ +@Description(name = "datediff", + value = "_FUNC_(date1, date2) - Returns the number of days between date1 and date2", + extended = "date1 and date2 are strings in the format " + + "'yyyy-MM-dd HH:mm:ss' or 'yyyy-MM-dd'. The time parts are ignored." + + "If date1 is earlier than date2, the result is negative.\n" + + "Example:\n " + + " > SELECT _FUNC_('2009-30-07', '2009-31-07') FROM src LIMIT 1;\n" + + " 1") +public class GenericUDFDateDiff extends GenericUDF { + private transient SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); + private transient Converter inputConverter1; + private transient Converter inputConverter2; + private IntWritable output = new IntWritable(); + private transient PrimitiveCategory inputType1; + private transient PrimitiveCategory inputType2; + + public GenericUDFDateDiff() { + formatter.setTimeZone(TimeZone.getTimeZone("UTC")); + } + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + if (arguments.length != 2) { + throw new UDFArgumentLengthException( + "datediff() requires 2 argument, got " + arguments.length); + } + inputConverter1 = checkArguments(arguments, 0); + inputConverter2 = checkArguments(arguments, 1); + inputType1 = ((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory(); + inputType2 = ((PrimitiveObjectInspector) arguments[1]).getPrimitiveCategory(); + ObjectInspector outputOI = PrimitiveObjectInspectorFactory.writableIntObjectInspector; + return outputOI; + } + + @Override + public IntWritable evaluate(DeferredObject[] arguments) throws HiveException { + output = evaluate(convertToDate(inputType1, inputConverter1, arguments[0]), + convertToDate(inputType2, inputConverter2, arguments[1])); + return output; + } + + @Override + public String getDisplayString(String[] children) { + StringBuilder sb = new StringBuilder(); + sb.append("datediff("); + if (children.length > 0) { + sb.append(children[0]); + for (int i = 1; i < children.length; i++) { + sb.append(", "); + sb.append(children[i]); + } + } + sb.append(")"); + return sb.toString(); + } + + private Date convertToDate(PrimitiveCategory inputType, Converter converter, DeferredObject argument) + throws HiveException { + assert(converter != null); + assert(argument != null); + Date date; + switch (inputType) { + case STRING: + String dateString = converter.convert(argument.get()).toString(); + try { + date = formatter.parse(dateString); + } catch (ParseException e) { + return null; + } + break; + case TIMESTAMP: + Timestamp ts = ((TimestampWritable) converter.convert(argument.get())) + .getTimestamp(); + date = new Date(ts.getTime()); + break; + case DATE: + DateWritable dw = (DateWritable) converter.convert(argument.get()); + date = dw.get(); + break; + default: + throw new UDFArgumentException( + "TO_DATE() only takes STRING/TIMESTAMP/DATEWRITABLE types, got " + inputType); + } + return date; + } + + private Converter checkArguments(ObjectInspector[] arguments, int i) throws UDFArgumentException { + if (arguments[i].getCategory() != ObjectInspector.Category.PRIMITIVE) { + throw new UDFArgumentTypeException(0, + "Only primitive type arguments are accepted but " + + arguments[i].getTypeName() + " is passed. as first arguments"); + } + PrimitiveCategory inputType = ((PrimitiveObjectInspector) arguments[i]).getPrimitiveCategory(); + Converter converter; + switch (inputType) { + case STRING: + converter = ObjectInspectorConverters.getConverter( + (PrimitiveObjectInspector) arguments[i], + PrimitiveObjectInspectorFactory.writableStringObjectInspector); + break; + case TIMESTAMP: + converter = new TimestampConverter((PrimitiveObjectInspector) arguments[i], + PrimitiveObjectInspectorFactory.writableTimestampObjectInspector); + break; + case DATE: + converter = ObjectInspectorConverters.getConverter((PrimitiveObjectInspector)arguments[i], + PrimitiveObjectInspectorFactory.writableDateObjectInspector); + break; + default: + throw new UDFArgumentException( + " DATEDIFF() only takes STRING/TIMESTAMP/DATEWRITABLE types as " + (i + 1) + + "-th argument, got " + inputType); + } + return converter; + } + + private IntWritable evaluate(Date date, Date date2) { + + if (date == null || date2 == null) { + return null; + } + IntWritable result = new IntWritable(); + // NOTE: This implementation avoids the extra-second problem + // by comparing with UTC epoch and integer division. + // 86400 is the number of seconds in a day + long diffInMilliSeconds = date.getTime() - date2.getTime(); + result.set((int) (diffInMilliSeconds / (86400 * 1000))); + return result; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDateSub.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDateSub.java new file mode 100644 index 0000000..ec72e87 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDateSub.java @@ -0,0 +1,172 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.udf.generic; + +import java.sql.Timestamp; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter.TimestampConverter; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.IntWritable; + +/** + * UDFDateSub. + * + * Subtract a number of days to the date. The time part of the string will be + * ignored. + * + * NOTE: This is a subset of what MySQL offers as: + * http://dev.mysql.com/doc/refman + * /5.1/en/date-and-time-functions.html#function_date-sub + * + */ +@Description(name = "date_sub", + value = "_FUNC_(start_date, num_days) - Returns the date that is num_days before start_date.", + extended = "start_date is a string in the format 'yyyy-MM-dd HH:mm:ss' or" + + " 'yyyy-MM-dd'. num_days is a number. The time part of start_date is " + + "ignored.\n" + + "Example:\n " + + " > SELECT _FUNC_('2009-30-07', 1) FROM src LIMIT 1;\n" + + " '2009-29-07'") +public class GenericUDFDateSub extends GenericUDF { + private transient SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); + private transient TimestampConverter timestampConverter; + private transient Converter textConverter; + private transient Converter dateWritableConverter; + private transient Converter intWritableConverter; + private transient PrimitiveCategory inputType1; + private transient PrimitiveCategory inputType2; + private final Calendar calendar = Calendar.getInstance(); + private final Text output = new Text(); + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + if (arguments.length != 2) { + throw new UDFArgumentLengthException( + "date_sub() requires 2 argument, got " + arguments.length); + } + if (arguments[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { + throw new UDFArgumentTypeException(0, + "Only primitive type arguments are accepted but " + + arguments[0].getTypeName() + " is passed. as first arguments"); + } + if (arguments[1].getCategory() != ObjectInspector.Category.PRIMITIVE) { + throw new UDFArgumentTypeException(1, + "Only primitive type arguments are accepted but " + + arguments[2].getTypeName() + " is passed. as second arguments"); + } + + inputType1 = ((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory(); + ObjectInspector outputOI = PrimitiveObjectInspectorFactory.writableStringObjectInspector; + switch (inputType1) { + case STRING: + textConverter = ObjectInspectorConverters.getConverter( + (PrimitiveObjectInspector) arguments[0], + PrimitiveObjectInspectorFactory.writableStringObjectInspector); + break; + case TIMESTAMP: + timestampConverter = new TimestampConverter((PrimitiveObjectInspector) arguments[0], + PrimitiveObjectInspectorFactory.writableTimestampObjectInspector); + break; + case DATE: + dateWritableConverter = ObjectInspectorConverters.getConverter( + (PrimitiveObjectInspector) arguments[0], + PrimitiveObjectInspectorFactory.writableDateObjectInspector); + break; + default: + throw new UDFArgumentException( + "DATE_SUB() only takes STRING/TIMESTAMP/DATEWRITABLE types as first argument, got " + + inputType1); + } + + inputType2 = ((PrimitiveObjectInspector) arguments[1]).getPrimitiveCategory(); + if (inputType2 != PrimitiveCategory.INT) { + throw new UDFArgumentException( + " DATE_SUB() only takes INT types as second argument, got " + inputType2); + } + intWritableConverter = ObjectInspectorConverters.getConverter( + (PrimitiveObjectInspector) arguments[1], + PrimitiveObjectInspectorFactory.writableIntObjectInspector); + return outputOI; + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + + IntWritable toBeSubed = (IntWritable) intWritableConverter.convert(arguments[1].get()); + switch (inputType1) { + case STRING: + String dateString = textConverter.convert(arguments[0].get()).toString(); + try { + calendar.setTime(formatter.parse(dateString.toString())); + calendar.add(Calendar.DAY_OF_MONTH, -toBeSubed.get()); + } catch (ParseException e) { + return null; + } + break; + case TIMESTAMP: + Timestamp ts = ((TimestampWritable) timestampConverter.convert(arguments[0].get())) + .getTimestamp(); + calendar.setTime(ts); + calendar.add(Calendar.DAY_OF_MONTH, -toBeSubed.get()); + break; + case DATE: + DateWritable dw = (DateWritable) dateWritableConverter.convert(arguments[0].get()); + calendar.setTime(dw.get()); + calendar.add(Calendar.DAY_OF_MONTH, -toBeSubed.get()); + break; + default: + throw new UDFArgumentException( + "DATE_SUB() only takes STRING/TIMESTAMP/DATEWRITABLE types, got " + inputType1); + } + Date newDate = calendar.getTime(); + output.set(formatter.format(newDate)); + return output; + } + + @Override + public String getDisplayString(String[] children) { + StringBuilder sb = new StringBuilder(); + sb.append("date_sub("); + if (children.length > 0) { + sb.append(children[0]); + for (int i = 1; i < children.length; i++) { + sb.append(", "); + sb.append(children[i]); + } + } + sb.append(")"); + return sb.toString(); + } +} diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/TestGenericUDFDate.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/TestGenericUDFDate.java new file mode 100644 index 0000000..384ce4e --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/TestGenericUDFDate.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.udf; + +import java.sql.Date; +import java.sql.Timestamp; + +import junit.framework.TestCase; + +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFDate; +import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.Text; + +public class TestGenericUDFDate extends TestCase { + public void testStringToDate() throws HiveException { + GenericUDFDate udf = new GenericUDFDate(); + ObjectInspector valueOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector; + ObjectInspector[] arguments = {valueOI}; + + udf.initialize(arguments); + DeferredObject valueObj = new DeferredJavaObject(new Text("2009-07-30")); + DeferredObject[] args = {valueObj}; + Text output = (Text) udf.evaluate(args); + + assertEquals("to_date() test for STRING failed ", "2009-07-30", output.toString()); + } + + public void testTimestampToDate() throws HiveException { + GenericUDFDate udf = new GenericUDFDate(); + ObjectInspector valueOI = PrimitiveObjectInspectorFactory.writableTimestampObjectInspector; + ObjectInspector[] arguments = {valueOI}; + + udf.initialize(arguments); + DeferredObject valueObj = new DeferredJavaObject(new TimestampWritable(new Timestamp(109, 06, + 30, 4, 17, 52, 0))); + DeferredObject[] args = {valueObj}; + Text output = (Text) udf.evaluate(args); + + assertEquals("to_date() test for TIMESTAMP failed ", "2009-07-30", output.toString()); + } + + public void testDateWritablepToDate() throws HiveException { + GenericUDFDate udf = new GenericUDFDate(); + ObjectInspector valueOI = PrimitiveObjectInspectorFactory.writableDateObjectInspector; + ObjectInspector[] arguments = {valueOI}; + + udf.initialize(arguments); + DeferredObject valueObj = new DeferredJavaObject(new DateWritable(new Date(109, 06, 30))); + DeferredObject[] args = {valueObj}; + Text output = (Text) udf.evaluate(args); + + assertEquals("to_date() test for DATEWRITABLE failed ", "2009-07-30", output.toString()); + } + +} diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/TestGenericUDFDateAdd.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/TestGenericUDFDateAdd.java new file mode 100644 index 0000000..ee841d3 --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/TestGenericUDFDateAdd.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.udf; + +import java.sql.Date; +import java.sql.Timestamp; + +import junit.framework.TestCase; + +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFDateAdd; +import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.Text; + +public class TestGenericUDFDateAdd extends TestCase { + public void testStringToDate() throws HiveException { + GenericUDFDateAdd udf = new GenericUDFDateAdd(); + ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory.javaStringObjectInspector; + ObjectInspector valueOI2 = PrimitiveObjectInspectorFactory.javaIntObjectInspector; + ObjectInspector[] arguments = {valueOI1, valueOI2}; + + udf.initialize(arguments); + DeferredObject valueObj1 = new DeferredJavaObject(new Text("2009-07-20 04:17:52")); + DeferredObject valueObj2 = new DeferredJavaObject(new Integer("2")); + DeferredObject[] args = {valueObj1, valueObj2}; + Text output = (Text) udf.evaluate(args); + + assertEquals("date_add() test for STRING failed ", "2009-07-22", output.toString()); + } + + public void testTimestampToDate() throws HiveException { + GenericUDFDateAdd udf = new GenericUDFDateAdd(); + ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory.writableTimestampObjectInspector; + ObjectInspector valueOI2 = PrimitiveObjectInspectorFactory.javaIntObjectInspector; + ObjectInspector[] arguments = {valueOI1, valueOI2}; + + udf.initialize(arguments); + DeferredObject valueObj1 = new DeferredJavaObject(new TimestampWritable(new Timestamp(109, 06, + 20, 4, 17, 52, 0))); + DeferredObject valueObj2 = new DeferredJavaObject(new Integer("3")); + DeferredObject[] args = {valueObj1, valueObj2}; + Text output = (Text) udf.evaluate(args); + + assertEquals("date_add() test for TIMESTAMP failed ", "2009-07-23", output.toString()); + } + + public void testDateWritablepToDate() throws HiveException { + GenericUDFDateAdd udf = new GenericUDFDateAdd(); + ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory.writableDateObjectInspector; + ObjectInspector valueOI2 = PrimitiveObjectInspectorFactory.javaIntObjectInspector; + ObjectInspector[] arguments = {valueOI1, valueOI2}; + + + udf.initialize(arguments); + DeferredObject valueObj1 = new DeferredJavaObject(new DateWritable(new Date(109, 06, 20))); + DeferredObject valueObj2 = new DeferredJavaObject(new Integer("4")); + DeferredObject[] args = {valueObj1, valueObj2}; + Text output = (Text) udf.evaluate(args); + + assertEquals("date_add() test for DATEWRITABLE failed ", "2009-07-24", output.toString()); + } + +} diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/TestGenericUDFDateDiff.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/TestGenericUDFDateDiff.java new file mode 100644 index 0000000..936c554 --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/TestGenericUDFDateDiff.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.udf; + +import java.sql.Date; +import java.sql.Timestamp; + +import junit.framework.TestCase; + +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFDateDiff; +import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; + +public class TestGenericUDFDateDiff extends TestCase { + public void testStringToDate() throws HiveException { + GenericUDFDateDiff udf = new GenericUDFDateDiff(); + ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory.javaStringObjectInspector; + ObjectInspector valueOI2 = PrimitiveObjectInspectorFactory.javaStringObjectInspector; + ObjectInspector[] arguments = {valueOI1, valueOI2}; + + udf.initialize(arguments); + DeferredObject valueObj1 = new DeferredJavaObject(new Text("2009-07-20")); + DeferredObject valueObj2 = new DeferredJavaObject(new Text("2009-07-22")); + DeferredObject[] args = {valueObj1, valueObj2}; + IntWritable output = (IntWritable) udf.evaluate(args); + + assertEquals("date_iff() test for STRING failed ", "-2", output.toString()); + } + + public void testTimestampToDate() throws HiveException { + GenericUDFDateDiff udf = new GenericUDFDateDiff(); + ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory.writableTimestampObjectInspector; + ObjectInspector valueOI2 = PrimitiveObjectInspectorFactory.writableTimestampObjectInspector; + ObjectInspector[] arguments = {valueOI1, valueOI2}; + + udf.initialize(arguments); + DeferredObject valueObj1 = new DeferredJavaObject(new TimestampWritable(new Timestamp(109, 06, + 20, 0, 0, 0, 0))); + DeferredObject valueObj2 = new DeferredJavaObject(new TimestampWritable(new Timestamp(109, 06, + 17, 0, 0, 0, 0))); + DeferredObject[] args = {valueObj1, valueObj2}; + IntWritable output = (IntWritable) udf.evaluate(args); + + assertEquals("datediff() test for TIMESTAMP failed ", "3", output.toString()); + } + + public void testDateWritablepToDate() throws HiveException { + GenericUDFDateDiff udf = new GenericUDFDateDiff(); + ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory.writableDateObjectInspector; + ObjectInspector valueOI2 = PrimitiveObjectInspectorFactory.writableDateObjectInspector; + ObjectInspector[] arguments = {valueOI1, valueOI2}; + + + udf.initialize(arguments); + DeferredObject valueObj1 = new DeferredJavaObject(new DateWritable(new Date(109, 06, 20))); + DeferredObject valueObj2 = new DeferredJavaObject(new DateWritable(new Date(109, 06, 10))); + DeferredObject[] args = {valueObj1, valueObj2}; + IntWritable output = (IntWritable) udf.evaluate(args); + + assertEquals("datediff() test for DATEWRITABLE failed ", "10", output.toString()); + } + +} diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/TestGenericUDFDateSub.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/TestGenericUDFDateSub.java new file mode 100644 index 0000000..46153e0 --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/TestGenericUDFDateSub.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.udf; + +import java.sql.Date; +import java.sql.Timestamp; + +import junit.framework.TestCase; + +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFDateSub; +import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.Text; + +public class TestGenericUDFDateSub extends TestCase { + public void testStringToDate() throws HiveException { + GenericUDFDateSub udf = new GenericUDFDateSub(); + ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory.javaStringObjectInspector; + ObjectInspector valueOI2 = PrimitiveObjectInspectorFactory.javaIntObjectInspector; + ObjectInspector[] arguments = {valueOI1, valueOI2}; + + udf.initialize(arguments); + DeferredObject valueObj1 = new DeferredJavaObject(new Text("2009-07-20 04:17:52")); + DeferredObject valueObj2 = new DeferredJavaObject(new Integer("2")); + DeferredObject[] args = {valueObj1, valueObj2}; + Text output = (Text) udf.evaluate(args); + + assertEquals("date_sub() test for STRING failed ", "2009-07-18", output.toString()); + } + + public void testTimestampToDate() throws HiveException { + GenericUDFDateSub udf = new GenericUDFDateSub(); + ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory.writableTimestampObjectInspector; + ObjectInspector valueOI2 = PrimitiveObjectInspectorFactory.javaIntObjectInspector; + ObjectInspector[] arguments = {valueOI1, valueOI2}; + + udf.initialize(arguments); + DeferredObject valueObj1 = new DeferredJavaObject(new TimestampWritable(new Timestamp(109, 06, + 20, 4, 17, 52, 0))); + DeferredObject valueObj2 = new DeferredJavaObject(new Integer("3")); + DeferredObject[] args = {valueObj1, valueObj2}; + Text output = (Text) udf.evaluate(args); + + assertEquals("date_sub() test for TIMESTAMP failed ", "2009-07-17", output.toString()); + } + + public void testDateWritablepToDate() throws HiveException { + GenericUDFDateSub udf = new GenericUDFDateSub(); + ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory.writableDateObjectInspector; + ObjectInspector valueOI2 = PrimitiveObjectInspectorFactory.javaIntObjectInspector; + ObjectInspector[] arguments = {valueOI1, valueOI2}; + + + udf.initialize(arguments); + DeferredObject valueObj1 = new DeferredJavaObject(new DateWritable(new Date(109, 06, 20))); + DeferredObject valueObj2 = new DeferredJavaObject(new Integer("4")); + DeferredObject[] args = {valueObj1, valueObj2}; + Text output = (Text) udf.evaluate(args); + + assertEquals("date_sub() test for DATEWRITABLE failed ", "2009-07-16", output.toString()); + } + +} diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFDateAdd.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFDateAdd.java deleted file mode 100644 index f0af069..0000000 --- a/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFDateAdd.java +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.udf; - -import java.util.TimeZone; - -import junit.framework.TestCase; - -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.Text; - -/** - * JUnit test for UDFDateAdd. - */ -public class TestUDFDateAdd extends TestCase { - - /** - * Verify if adding dates across a daylight savings time change - * from daylight to standard time. The timezone tested is west - * coast US (PDT/PST) with a 1 hour shift back in time at 02:00 AM - * on 2009-10-31. - */ - public void testFallBack() throws Exception { - // set the default time zone so that the dates cover - // the zone's daylight saving time adjustment (2009-10-31) - // from daylight to standard time - TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles")); - final IntWritable nmDays = new IntWritable(7); - final Text dtStr = new Text("2009-10-26"); - - final Text result = new UDFDateAdd().evaluate(dtStr, nmDays); - assertEquals("2009-11-02", result.toString()); - } - - /** - * Verify if adding dates across a daylight savings time change - * from standard to daylight time. The timezone tested is west - * coast US (PDT/PST) with a 1 hour shift forward in time at 02:00 AM - * on 2010-03-14. - */ - public void testSpringAhead() throws Exception { - // set the default time zone so that the dates cover - // the zone's daylight saving time adjustment (2010-03-14) - // from standard to daylight time - TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles")); - final IntWritable nmDays = new IntWritable(7); - final Text dtStr = new Text("2010-03-08"); - - final Text result = new UDFDateAdd().evaluate(dtStr, nmDays); - assertEquals("2010-03-15", result.toString()); - } -} diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFDateDiff.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFDateDiff.java deleted file mode 100644 index 8a6dbc3..0000000 --- a/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFDateDiff.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.udf; - -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.Text; - -import junit.framework.TestCase; - -import java.util.TimeZone; - -/** - * JUnit test for UDFDateDiff. - */ -public class TestUDFDateDiff extends TestCase { - - /** - * Verify differences of dates crossing a daylight savings time change - * are correct. The timezone tested is west coast US (PDT/PST) with a - * 1 hour shift back in time at 02:00 AM on 2009-10-31 and a - * 1 hour shift forward in time at 02:00 AM on 2010-03-14. - */ - public void testDaylightChange() throws Exception { - TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles")); - - // time moves ahead an hour at 02:00 on 2009-10-31 - // which results in a 23 hour long day - Text date1 = new Text("2009-11-01"); - Text date2 = new Text("2009-10-25"); - - IntWritable result = new UDFDateDiff().evaluate(date1, date2); - assertEquals(7, result.get()); - - result = new UDFDateDiff().evaluate(date2, date1); - assertEquals(-7, result.get()); - - - // time moves back an hour at 02:00 on 2010-03-14 - // which results in a 25 hour long day - date1 = new Text("2010-03-15"); - date2 = new Text("2010-03-08"); - - result = new UDFDateDiff().evaluate(date1, date2); - assertEquals(7, result.get()); - - result = new UDFDateDiff().evaluate(date2, date1); - assertEquals(-7, result.get()); - - } -} diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFDateSub.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFDateSub.java deleted file mode 100644 index fa722a9..0000000 --- a/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFDateSub.java +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.udf; - -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.Text; - -import junit.framework.TestCase; - -import java.util.TimeZone; - -/** - * JUnit test for UDFDateSub. - */ -public class TestUDFDateSub extends TestCase { - - /** - * Verify if subtracting dates across a daylight savings time change - * from daylight to standard time. The timezone tested is west - * coast US (PDT/PST) with a 1 hour shift back in time at 02:00 AM - * on 2009-10-31. - */ - public void testFallBack() throws Exception { - // set the default time zone so that the dates cover - // the zone's daylight saving time adjustment (2009-10-31) - // from daylight to standard time - TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles")); - final IntWritable nmDays = new IntWritable(7); - final Text dtStr = new Text("2009-11-02"); - - final Text result = new UDFDateSub().evaluate(dtStr, nmDays); - assertEquals("2009-10-26", result.toString()); - } - - /** - * Verify if subtracting dates across a daylight savings time change - * from standard to daylight time. The timezone tested is west - * coast US (PDT/PST) with a 1 hour shift forward in time at 02:00 AM - * on 2010-03-14. - */ - public void testSpringAhead() throws Exception { - // set the default time zone so that the dates cover - // the zone's daylight saving time adjustment (2010-03-14) - // from standard to daylight time - TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles")); - final IntWritable nmDays = new IntWritable(7); - final Text dtStr = new Text("2010-03-15"); - - final Text result = new UDFDateSub().evaluate(dtStr, nmDays); - assertEquals("2010-03-08", result.toString()); - } -} diff --git a/ql/src/test/results/clientpositive/udf_to_date.q.out b/ql/src/test/results/clientpositive/udf_to_date.q.out index 6ff5ee8..85676f3 100644 --- a/ql/src/test/results/clientpositive/udf_to_date.q.out +++ b/ql/src/test/results/clientpositive/udf_to_date.q.out @@ -9,5 +9,5 @@ POSTHOOK: query: DESCRIBE FUNCTION EXTENDED to_date POSTHOOK: type: DESCFUNCTION to_date(expr) - Extracts the date part of the date or datetime expression expr Example: - > SELECT to_date('2009-30-07 04:17:52') FROM src LIMIT 1; - '2009-30-07' + > SELECT to_date('2009-07-30 04:17:52') FROM src LIMIT 1; + '2009-07-30'