From 30e1032b7f144c6215fcfeb910d94625499dbaf1 Mon Sep 17 00:00:00 2001 From: Ashutosh Chauhan Date: Tue, 1 May 2018 22:38:08 -0700 Subject: [PATCH] HIVE-19383 : Add ArrayList$SubList kryo serializer --- .../test/resources/testconfiguration.properties | 1 + .../hive/ql/exec/SerializationUtilities.java | 73 +++++++++++++++ ql/src/test/queries/clientpositive/kryo.q | 16 ++++ ql/src/test/results/clientpositive/llap/kryo.q.out | 104 +++++++++++++++++++++ 4 files changed, 194 insertions(+) create mode 100644 ql/src/test/queries/clientpositive/kryo.q create mode 100644 ql/src/test/results/clientpositive/llap/kryo.q.out diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 0e4e7064eb..4343624fe3 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -535,6 +535,7 @@ minillaplocal.query.files=\ join_nullsafe.q,\ join_is_not_distinct_from.q,\ join_reordering_no_stats.q,\ + kryo.q,\ leftsemijoin_mr.q,\ limit_join_transpose.q,\ lineage2.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java index e1f40cf5b1..f773293db4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java @@ -222,6 +222,8 @@ public Registration readClass(Input input) { } } + private static final Object FAKE_REFERENCE = new Object(); + private static KryoFactory factory = new KryoFactory() { public Kryo create() { KryoWithHooks kryo = new KryoWithHooks(); @@ -230,6 +232,11 @@ public Kryo create() { kryo.register(TimestampTZ.class, new TimestampTZSerializer()); kryo.register(Path.class, new PathSerializer()); kryo.register(Arrays.asList("").getClass(), new ArraysAsListSerializer()); + try{ + kryo.register(Class.forName("java.util.ArrayList$SubList"), new ArrayListSubListSerializer()); + } catch (Exception e) { + throw new RuntimeException(e); + } kryo.register(CopyOnFirstWriteProperties.class, new CopyOnFirstWritePropertiesSerializer()); ((Kryo.DefaultInstantiatorStrategy) kryo.getInstantiatorStrategy()) @@ -362,6 +369,72 @@ public Path read(Kryo kryo, Input input, Class type) { } } + /** + * Supports sublists created via {@link ArrayList#subList(int, int)} since java7 (oracle jdk, + * represented by java.util.ArrayList$SubList). + */ + public static class ArrayListSubListSerializer extends com.esotericsoftware.kryo.Serializer> { + + private Field _parentField; + private Field _parentOffsetField; + private Field _sizeField; + + public ArrayListSubListSerializer() { + try { + final Class clazz = Class.forName("java.util.ArrayList$SubList"); + _parentField = clazz.getDeclaredField("parent"); + _parentOffsetField = clazz.getDeclaredField( "parentOffset" ); + _sizeField = clazz.getDeclaredField( "size" ); + _parentField.setAccessible( true ); + _parentOffsetField.setAccessible( true ); + _sizeField.setAccessible( true ); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public List read(final Kryo kryo, final Input input, final Class> clazz) { + kryo.reference(FAKE_REFERENCE); + final List list = (List) kryo.readClassAndObject(input); + final int fromIndex = input.readInt(true); + final int toIndex = input.readInt(true); + return list.subList(fromIndex, toIndex); + } + + @Override + public void write(final Kryo kryo, final Output output, final List obj) { + try { + kryo.writeClassAndObject(output, _parentField.get(obj)); + final int parentOffset = _parentOffsetField.getInt( obj ); + final int fromIndex = parentOffset; + output.writeInt(fromIndex, true); + final int toIndex = fromIndex + _sizeField.getInt( obj ); + output.writeInt(toIndex, true); + } catch (final RuntimeException e) { + // Don't eat and wrap RuntimeExceptions because the ObjectBuffer.write... + // handles SerializationException specifically (resizing the buffer)... + throw e; + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public List copy(final Kryo kryo, final List original) { + kryo.reference(FAKE_REFERENCE); + try { + final List list = (List) _parentField.get(original); + final int parentOffset = _parentOffsetField.getInt( original ); + final int fromIndex = parentOffset; + final int toIndex = fromIndex + _sizeField.getInt( original ); + return kryo.copy(list).subList(fromIndex, toIndex); + } catch(final Exception e) { + throw new RuntimeException(e); + } + } + } + /** * A kryo {@link Serializer} for lists created via {@link Arrays#asList(Object...)}. *

diff --git a/ql/src/test/queries/clientpositive/kryo.q b/ql/src/test/queries/clientpositive/kryo.q new file mode 100644 index 0000000000..eba10c1cf6 --- /dev/null +++ b/ql/src/test/queries/clientpositive/kryo.q @@ -0,0 +1,16 @@ +CREATE TABLE `DateTime` ( + ID INT, + `Time` STRING, + Date_ STRING, + `DateTime` STRING, + Random_ DOUBLE, + Diagonal DOUBLE, + Curved DOUBLE + ) +STORED AS orc; + +explain SELECT IF(((`DateTime_`.`id` = 1) OR (`DateTime_`.`id` = 2)),IF((`DateTime_`.`id` = 1),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE NULL END),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE TO_DATE(`DateTime_`.`time`) END)),IF((`DateTime_`.`id` = 3),(CASE WHEN (`DateTime_`.`date_` IS NULL) OR (TO_DATE(`DateTime_`.`datetime`) IS NULL) THEN NULL WHEN `DateTime_`.`date_` > TO_DATE(`DateTime_`.`datetime`) THEN `DateTime_`.`date_` ELSE TO_DATE(`DateTime_`.`datetime`) END),NULL)) AS `none_calculation2_ok`, `DateTime_`.`id` AS `none_id_ok`, (CASE WHEN (`DateTime_`.`id` = 6) THEN (CASE WHEN (CONCAT(`DateTime_`.`date_`, ' 00:00:00') IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') > `DateTime_`.`datetime` THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 5) THEN (CASE WHEN (`DateTime_`.`date_` IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN `DateTime_`.`date_` > `DateTime_`.`datetime` THEN `DateTime_`.`date_` ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 3) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 4) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) WHEN (`DateTime_`.`id` = 1) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 2) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) ELSE NULL END) AS `none_mixed_type_calcs_ok` FROM `DateTime` `DateTime_` WHERE (`DateTime_`.`id` = 1 OR `DateTime_`.`id` = 2 OR `DateTime_`.`id` = 3 OR `DateTime_`.`id` = 4 OR `DateTime_`.`id` = 5 OR `DateTime_`.`id` = 6) GROUP BY IF(((`DateTime_`.`id` = 1) OR (`DateTime_`.`id` = 2)),IF((`DateTime_`.`id` = 1),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE NULL END),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE TO_DATE(`DateTime_`.`time`) END)),IF((`DateTime_`.`id` = 3),(CASE WHEN (`DateTime_`.`date_` IS NULL) OR (TO_DATE(`DateTime_`.`datetime`) IS NULL) THEN NULL WHEN `DateTime_`.`date_` > TO_DATE(`DateTime_`.`datetime`) THEN `DateTime_`.`date_` ELSE TO_DATE(`DateTime_`.`datetime`) END),NULL)), `DateTime_`.`id`, (CASE WHEN (`DateTime_`.`id` = 6) THEN (CASE WHEN (CONCAT(`DateTime_`.`date_`, ' 00:00:00') IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') > `DateTime_`.`datetime` THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 5) THEN (CASE WHEN (`DateTime_`.`date_` IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN `DateTime_`.`date_` > `DateTime_`.`datetime` THEN `DateTime_`.`date_` ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 3) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 4) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) WHEN (`DateTime_`.`id` = 1) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 2) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) ELSE NULL END) ; + +SELECT IF(((`DateTime_`.`id` = 1) OR (`DateTime_`.`id` = 2)),IF((`DateTime_`.`id` = 1),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE NULL END),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE TO_DATE(`DateTime_`.`time`) END)),IF((`DateTime_`.`id` = 3),(CASE WHEN (`DateTime_`.`date_` IS NULL) OR (TO_DATE(`DateTime_`.`datetime`) IS NULL) THEN NULL WHEN `DateTime_`.`date_` > TO_DATE(`DateTime_`.`datetime`) THEN `DateTime_`.`date_` ELSE TO_DATE(`DateTime_`.`datetime`) END),NULL)) AS `none_calculation2_ok`, `DateTime_`.`id` AS `none_id_ok`, (CASE WHEN (`DateTime_`.`id` = 6) THEN (CASE WHEN (CONCAT(`DateTime_`.`date_`, ' 00:00:00') IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') > `DateTime_`.`datetime` THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 5) THEN (CASE WHEN (`DateTime_`.`date_` IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN `DateTime_`.`date_` > `DateTime_`.`datetime` THEN `DateTime_`.`date_` ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 3) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 4) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) WHEN (`DateTime_`.`id` = 1) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 2) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) ELSE NULL END) AS `none_mixed_type_calcs_ok` FROM `DateTime` `DateTime_` WHERE (`DateTime_`.`id` = 1 OR `DateTime_`.`id` = 2 OR `DateTime_`.`id` = 3 OR `DateTime_`.`id` = 4 OR `DateTime_`.`id` = 5 OR `DateTime_`.`id` = 6) GROUP BY IF(((`DateTime_`.`id` = 1) OR (`DateTime_`.`id` = 2)),IF((`DateTime_`.`id` = 1),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE NULL END),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE TO_DATE(`DateTime_`.`time`) END)),IF((`DateTime_`.`id` = 3),(CASE WHEN (`DateTime_`.`date_` IS NULL) OR (TO_DATE(`DateTime_`.`datetime`) IS NULL) THEN NULL WHEN `DateTime_`.`date_` > TO_DATE(`DateTime_`.`datetime`) THEN `DateTime_`.`date_` ELSE TO_DATE(`DateTime_`.`datetime`) END),NULL)), `DateTime_`.`id`, (CASE WHEN (`DateTime_`.`id` = 6) THEN (CASE WHEN (CONCAT(`DateTime_`.`date_`, ' 00:00:00') IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') > `DateTime_`.`datetime` THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 5) THEN (CASE WHEN (`DateTime_`.`date_` IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN `DateTime_`.`date_` > `DateTime_`.`datetime` THEN `DateTime_`.`date_` ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 3) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 4) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) WHEN (`DateTime_`.`id` = 1) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 2) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) ELSE NULL END) ; + +drop table DateTime; diff --git a/ql/src/test/results/clientpositive/llap/kryo.q.out b/ql/src/test/results/clientpositive/llap/kryo.q.out new file mode 100644 index 0000000000..1463dfb967 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/kryo.q.out @@ -0,0 +1,104 @@ +PREHOOK: query: CREATE TABLE `DateTime` ( + ID INT, + `Time` STRING, + Date_ STRING, + `DateTime` STRING, + Random_ DOUBLE, + Diagonal DOUBLE, + Curved DOUBLE + ) +STORED AS orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@DateTime +POSTHOOK: query: CREATE TABLE `DateTime` ( + ID INT, + `Time` STRING, + Date_ STRING, + `DateTime` STRING, + Random_ DOUBLE, + Diagonal DOUBLE, + Curved DOUBLE + ) +STORED AS orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@DateTime +PREHOOK: query: explain SELECT IF(((`DateTime_`.`id` = 1) OR (`DateTime_`.`id` = 2)),IF((`DateTime_`.`id` = 1),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE NULL END),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE TO_DATE(`DateTime_`.`time`) END)),IF((`DateTime_`.`id` = 3),(CASE WHEN (`DateTime_`.`date_` IS NULL) OR (TO_DATE(`DateTime_`.`datetime`) IS NULL) THEN NULL WHEN `DateTime_`.`date_` > TO_DATE(`DateTime_`.`datetime`) THEN `DateTime_`.`date_` ELSE TO_DATE(`DateTime_`.`datetime`) END),NULL)) AS `none_calculation2_ok`, `DateTime_`.`id` AS `none_id_ok`, (CASE WHEN (`DateTime_`.`id` = 6) THEN (CASE WHEN (CONCAT(`DateTime_`.`date_`, ' 00:00:00') IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') > `DateTime_`.`datetime` THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 5) THEN (CASE WHEN (`DateTime_`.`date_` IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN `DateTime_`.`date_` > `DateTime_`.`datetime` THEN `DateTime_`.`date_` ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 3) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 4) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) WHEN (`DateTime_`.`id` = 1) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 2) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) ELSE NULL END) AS `none_mixed_type_calcs_ok` FROM `DateTime` `DateTime_` WHERE (`DateTime_`.`id` = 1 OR `DateTime_`.`id` = 2 OR `DateTime_`.`id` = 3 OR `DateTime_`.`id` = 4 OR `DateTime_`.`id` = 5 OR `DateTime_`.`id` = 6) GROUP BY IF(((`DateTime_`.`id` = 1) OR (`DateTime_`.`id` = 2)),IF((`DateTime_`.`id` = 1),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE NULL END),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE TO_DATE(`DateTime_`.`time`) END)),IF((`DateTime_`.`id` = 3),(CASE WHEN (`DateTime_`.`date_` IS NULL) OR (TO_DATE(`DateTime_`.`datetime`) IS NULL) THEN NULL WHEN `DateTime_`.`date_` > TO_DATE(`DateTime_`.`datetime`) THEN `DateTime_`.`date_` ELSE TO_DATE(`DateTime_`.`datetime`) END),NULL)), `DateTime_`.`id`, (CASE WHEN (`DateTime_`.`id` = 6) THEN (CASE WHEN (CONCAT(`DateTime_`.`date_`, ' 00:00:00') IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') > `DateTime_`.`datetime` THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 5) THEN (CASE WHEN (`DateTime_`.`date_` IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN `DateTime_`.`date_` > `DateTime_`.`datetime` THEN `DateTime_`.`date_` ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 3) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 4) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) WHEN (`DateTime_`.`id` = 1) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 2) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) ELSE NULL END) +PREHOOK: type: QUERY +POSTHOOK: query: explain SELECT IF(((`DateTime_`.`id` = 1) OR (`DateTime_`.`id` = 2)),IF((`DateTime_`.`id` = 1),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE NULL END),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE TO_DATE(`DateTime_`.`time`) END)),IF((`DateTime_`.`id` = 3),(CASE WHEN (`DateTime_`.`date_` IS NULL) OR (TO_DATE(`DateTime_`.`datetime`) IS NULL) THEN NULL WHEN `DateTime_`.`date_` > TO_DATE(`DateTime_`.`datetime`) THEN `DateTime_`.`date_` ELSE TO_DATE(`DateTime_`.`datetime`) END),NULL)) AS `none_calculation2_ok`, `DateTime_`.`id` AS `none_id_ok`, (CASE WHEN (`DateTime_`.`id` = 6) THEN (CASE WHEN (CONCAT(`DateTime_`.`date_`, ' 00:00:00') IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') > `DateTime_`.`datetime` THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 5) THEN (CASE WHEN (`DateTime_`.`date_` IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN `DateTime_`.`date_` > `DateTime_`.`datetime` THEN `DateTime_`.`date_` ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 3) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 4) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) WHEN (`DateTime_`.`id` = 1) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 2) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) ELSE NULL END) AS `none_mixed_type_calcs_ok` FROM `DateTime` `DateTime_` WHERE (`DateTime_`.`id` = 1 OR `DateTime_`.`id` = 2 OR `DateTime_`.`id` = 3 OR `DateTime_`.`id` = 4 OR `DateTime_`.`id` = 5 OR `DateTime_`.`id` = 6) GROUP BY IF(((`DateTime_`.`id` = 1) OR (`DateTime_`.`id` = 2)),IF((`DateTime_`.`id` = 1),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE NULL END),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE TO_DATE(`DateTime_`.`time`) END)),IF((`DateTime_`.`id` = 3),(CASE WHEN (`DateTime_`.`date_` IS NULL) OR (TO_DATE(`DateTime_`.`datetime`) IS NULL) THEN NULL WHEN `DateTime_`.`date_` > TO_DATE(`DateTime_`.`datetime`) THEN `DateTime_`.`date_` ELSE TO_DATE(`DateTime_`.`datetime`) END),NULL)), `DateTime_`.`id`, (CASE WHEN (`DateTime_`.`id` = 6) THEN (CASE WHEN (CONCAT(`DateTime_`.`date_`, ' 00:00:00') IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') > `DateTime_`.`datetime` THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 5) THEN (CASE WHEN (`DateTime_`.`date_` IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN `DateTime_`.`date_` > `DateTime_`.`datetime` THEN `DateTime_`.`date_` ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 3) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 4) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) WHEN (`DateTime_`.`id` = 1) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 2) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) ELSE NULL END) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: datetime_ + Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((id = 1) or (id = 2) or (id = 3) or (id = 4) or (id = 5) or (id = 6)) (type: boolean) + Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: if(((id = 1) or (id = 2)), if((id = 1), date_, date_), if((id = 3), CASE WHEN ((date_ is null or to_date(datetime) is null)) THEN (null) WHEN ((CAST( date_ AS DATE) > to_date(datetime))) THEN (date_) ELSE (to_date(datetime)) END, null)) (type: string), id (type: int), CASE WHEN ((id = 6)) THEN (CASE WHEN ((concat(date_, ' 00:00:00') is null or datetime is null)) THEN (null) WHEN ((concat(date_, ' 00:00:00') > datetime)) THEN (concat(date_, ' 00:00:00')) ELSE (datetime) END) WHEN ((id = 5)) THEN (CASE WHEN ((date_ is null or datetime is null)) THEN (null) WHEN ((date_ > datetime)) THEN (date_) ELSE (datetime) END) WHEN ((id = 3)) THEN (concat(date_, ' 00:00:00')) WHEN ((id = 4)) THEN (concat(date_, ' 00:00:00')) WHEN ((id = 1)) THEN (date_) WHEN ((id = 2)) THEN (date_) ELSE (null) END (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: int), _col2 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: int), _col2 (type: string) + Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: int), KEY._col2 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT IF(((`DateTime_`.`id` = 1) OR (`DateTime_`.`id` = 2)),IF((`DateTime_`.`id` = 1),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE NULL END),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE TO_DATE(`DateTime_`.`time`) END)),IF((`DateTime_`.`id` = 3),(CASE WHEN (`DateTime_`.`date_` IS NULL) OR (TO_DATE(`DateTime_`.`datetime`) IS NULL) THEN NULL WHEN `DateTime_`.`date_` > TO_DATE(`DateTime_`.`datetime`) THEN `DateTime_`.`date_` ELSE TO_DATE(`DateTime_`.`datetime`) END),NULL)) AS `none_calculation2_ok`, `DateTime_`.`id` AS `none_id_ok`, (CASE WHEN (`DateTime_`.`id` = 6) THEN (CASE WHEN (CONCAT(`DateTime_`.`date_`, ' 00:00:00') IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') > `DateTime_`.`datetime` THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 5) THEN (CASE WHEN (`DateTime_`.`date_` IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN `DateTime_`.`date_` > `DateTime_`.`datetime` THEN `DateTime_`.`date_` ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 3) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 4) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) WHEN (`DateTime_`.`id` = 1) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 2) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) ELSE NULL END) AS `none_mixed_type_calcs_ok` FROM `DateTime` `DateTime_` WHERE (`DateTime_`.`id` = 1 OR `DateTime_`.`id` = 2 OR `DateTime_`.`id` = 3 OR `DateTime_`.`id` = 4 OR `DateTime_`.`id` = 5 OR `DateTime_`.`id` = 6) GROUP BY IF(((`DateTime_`.`id` = 1) OR (`DateTime_`.`id` = 2)),IF((`DateTime_`.`id` = 1),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE NULL END),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE TO_DATE(`DateTime_`.`time`) END)),IF((`DateTime_`.`id` = 3),(CASE WHEN (`DateTime_`.`date_` IS NULL) OR (TO_DATE(`DateTime_`.`datetime`) IS NULL) THEN NULL WHEN `DateTime_`.`date_` > TO_DATE(`DateTime_`.`datetime`) THEN `DateTime_`.`date_` ELSE TO_DATE(`DateTime_`.`datetime`) END),NULL)), `DateTime_`.`id`, (CASE WHEN (`DateTime_`.`id` = 6) THEN (CASE WHEN (CONCAT(`DateTime_`.`date_`, ' 00:00:00') IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') > `DateTime_`.`datetime` THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 5) THEN (CASE WHEN (`DateTime_`.`date_` IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN `DateTime_`.`date_` > `DateTime_`.`datetime` THEN `DateTime_`.`date_` ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 3) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 4) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) WHEN (`DateTime_`.`id` = 1) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 2) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) ELSE NULL END) +PREHOOK: type: QUERY +PREHOOK: Input: default@datetime +#### A masked pattern was here #### +POSTHOOK: query: SELECT IF(((`DateTime_`.`id` = 1) OR (`DateTime_`.`id` = 2)),IF((`DateTime_`.`id` = 1),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE NULL END),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE TO_DATE(`DateTime_`.`time`) END)),IF((`DateTime_`.`id` = 3),(CASE WHEN (`DateTime_`.`date_` IS NULL) OR (TO_DATE(`DateTime_`.`datetime`) IS NULL) THEN NULL WHEN `DateTime_`.`date_` > TO_DATE(`DateTime_`.`datetime`) THEN `DateTime_`.`date_` ELSE TO_DATE(`DateTime_`.`datetime`) END),NULL)) AS `none_calculation2_ok`, `DateTime_`.`id` AS `none_id_ok`, (CASE WHEN (`DateTime_`.`id` = 6) THEN (CASE WHEN (CONCAT(`DateTime_`.`date_`, ' 00:00:00') IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') > `DateTime_`.`datetime` THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 5) THEN (CASE WHEN (`DateTime_`.`date_` IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN `DateTime_`.`date_` > `DateTime_`.`datetime` THEN `DateTime_`.`date_` ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 3) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 4) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) WHEN (`DateTime_`.`id` = 1) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 2) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) ELSE NULL END) AS `none_mixed_type_calcs_ok` FROM `DateTime` `DateTime_` WHERE (`DateTime_`.`id` = 1 OR `DateTime_`.`id` = 2 OR `DateTime_`.`id` = 3 OR `DateTime_`.`id` = 4 OR `DateTime_`.`id` = 5 OR `DateTime_`.`id` = 6) GROUP BY IF(((`DateTime_`.`id` = 1) OR (`DateTime_`.`id` = 2)),IF((`DateTime_`.`id` = 1),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE NULL END),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE TO_DATE(`DateTime_`.`time`) END)),IF((`DateTime_`.`id` = 3),(CASE WHEN (`DateTime_`.`date_` IS NULL) OR (TO_DATE(`DateTime_`.`datetime`) IS NULL) THEN NULL WHEN `DateTime_`.`date_` > TO_DATE(`DateTime_`.`datetime`) THEN `DateTime_`.`date_` ELSE TO_DATE(`DateTime_`.`datetime`) END),NULL)), `DateTime_`.`id`, (CASE WHEN (`DateTime_`.`id` = 6) THEN (CASE WHEN (CONCAT(`DateTime_`.`date_`, ' 00:00:00') IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') > `DateTime_`.`datetime` THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 5) THEN (CASE WHEN (`DateTime_`.`date_` IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN `DateTime_`.`date_` > `DateTime_`.`datetime` THEN `DateTime_`.`date_` ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 3) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 4) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) WHEN (`DateTime_`.`id` = 1) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 2) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) ELSE NULL END) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@datetime +#### A masked pattern was here #### +PREHOOK: query: drop table DateTime +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@datetime +PREHOOK: Output: default@datetime +POSTHOOK: query: drop table DateTime +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@datetime +POSTHOOK: Output: default@datetime -- 2.15.1 (Apple Git-101)