diff --git druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java index da7642c..b1b8177 100644 --- druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java +++ druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java @@ -144,13 +144,21 @@ af = new DoubleSumAggregatorFactory(columnNames.get(i), columnNames.get(i)); break; case TIMESTAMP: + // Granularity column String tColumnName = columnNames.get(i); - if (!tColumnName.equals(DruidStorageHandlerUtils.DEFAULT_TIMESTAMP_COLUMN) && !tColumnName - .equals(Constants.DRUID_TIMESTAMP_GRANULARITY_COL_NAME)) { + if (!!tColumnName.equals(Constants.DRUID_TIMESTAMP_GRANULARITY_COL_NAME)) { throw new IOException("Dimension " + tColumnName + " does not have STRING type: " + primitiveCategory); } continue; + case TIMESTAMPLOCALTZ: + // Druid timestamp column + String tLocalTZColumnName = columnNames.get(i); + if (!tLocalTZColumnName.equals(DruidStorageHandlerUtils.DEFAULT_TIMESTAMP_COLUMN)) { + throw new IOException("Dimension " + tLocalTZColumnName + " does not have STRING type: " + + primitiveCategory); + } + continue; default: // Dimension String dColumnName = columnNames.get(i); diff --git druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java index a43f62e..4a7952e 100644 --- druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java +++ druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java @@ -19,7 +19,8 @@ import java.io.IOException; import java.io.InputStream; -import java.sql.Timestamp; +import java.time.Instant; +import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -31,6 +32,7 @@ import org.apache.hadoop.hive.common.type.HiveChar; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.hive.common.type.TimestampTZ; import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.druid.DruidStorageHandler; @@ -47,7 +49,7 @@ import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.io.TimestampLocalTZWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.StructField; @@ -64,9 +66,11 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampLocalTZObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TimestampLocalTZTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; import org.apache.hadoop.io.BooleanWritable; @@ -175,7 +179,7 @@ public void initialize(Configuration configuration, Properties properties) throw if (columnInfo.getKey().equals(DruidStorageHandlerUtils.DEFAULT_TIMESTAMP_COLUMN)) { // Special handling for timestamp column columnNames.add(columnInfo.getKey()); // field name - PrimitiveTypeInfo type = TypeInfoFactory.timestampTypeInfo; // field type + PrimitiveTypeInfo type = TypeInfoFactory.timestampLocalTZTypeInfo; // field type columnTypes.add(type); inspectors .add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(type)); @@ -300,7 +304,7 @@ private void inferSchema(TimeseriesQuery query, Map mapColumnNamesTypes) { // Timestamp column columnNames.add(DruidStorageHandlerUtils.DEFAULT_TIMESTAMP_COLUMN); - columnTypes.add(TypeInfoFactory.timestampTypeInfo); + columnTypes.add(TypeInfoFactory.timestampLocalTZTypeInfo); // Aggregator columns for (AggregatorFactory af : query.getAggregatorSpecs()) { columnNames.add(af.getName()); @@ -328,7 +332,7 @@ private void inferSchema(TopNQuery query, Map mapColumnNamesTypes) { // Timestamp column columnNames.add(DruidStorageHandlerUtils.DEFAULT_TIMESTAMP_COLUMN); - columnTypes.add(TypeInfoFactory.timestampTypeInfo); + columnTypes.add(TypeInfoFactory.timestampLocalTZTypeInfo); // Dimension column columnNames.add(query.getDimensionSpec().getOutputName()); columnTypes.add(TypeInfoFactory.stringTypeInfo); @@ -360,7 +364,7 @@ private void inferSchema(SelectQuery query, throws SerDeException { // Timestamp column columnNames.add(DruidStorageHandlerUtils.DEFAULT_TIMESTAMP_COLUMN); - columnTypes.add(TypeInfoFactory.timestampTypeInfo); + columnTypes.add(TypeInfoFactory.timestampLocalTZTypeInfo); // Dimension columns for (DimensionSpec ds : query.getDimensions()) { columnNames.add(ds.getOutputName()); @@ -402,7 +406,7 @@ private void inferSchema(GroupByQuery query, Map mapColumnNamesTypes) { // Timestamp column columnNames.add(DruidStorageHandlerUtils.DEFAULT_TIMESTAMP_COLUMN); - columnTypes.add(TypeInfoFactory.timestampTypeInfo); + columnTypes.add(TypeInfoFactory.timestampLocalTZTypeInfo); // Dimension columns for (DimensionSpec ds : query.getDimensions()) { columnNames.add(ds.getOutputName()); @@ -456,10 +460,9 @@ public Writable serialize(Object o, ObjectInspector objectInspector) throws SerD } final Object res; switch (types[i].getPrimitiveCategory()) { - case TIMESTAMP: - res = ((TimestampObjectInspector) fields.get(i).getFieldObjectInspector()) - .getPrimitiveJavaObject( - values.get(i)).getTime(); + case TIMESTAMPLOCALTZ: + res = ((TimestampLocalTZObjectInspector) fields.get(i).getFieldObjectInspector()) + .getPrimitiveJavaObject(values.get(i)).getZonedDateTime().toInstant().toEpochMilli(); break; case BYTE: res = ((ByteObjectInspector) fields.get(i).getFieldObjectInspector()).get(values.get(i)); @@ -529,8 +532,13 @@ public Object deserialize(Writable writable) throws SerDeException { continue; } switch (types[i].getPrimitiveCategory()) { - case TIMESTAMP: - output.add(new TimestampWritable(new Timestamp((Long) value))); + case TIMESTAMPLOCALTZ: + output.add( + new TimestampLocalTZWritable( + new TimestampTZ( + ZonedDateTime.ofInstant( + Instant.ofEpochMilli((Long) value), + ((TimestampLocalTZTypeInfo) types[i]).timeZone())))); break; case BYTE: output.add(new ByteWritable(((Number) value).byteValue())); diff --git druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDeUtils.java druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDeUtils.java index 86c325b..bf55308 100644 --- druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDeUtils.java +++ druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDeUtils.java @@ -73,7 +73,7 @@ public static PrimitiveTypeInfo extractTypeFromDimension(DimensionSpec ds) { TimeFormatExtractionFn tfe = (TimeFormatExtractionFn) eds.getExtractionFn(); if (tfe.getFormat() == null || tfe.getFormat().equals(ISO_TIME_FORMAT)) { // Timestamp (null or default used by FLOOR) - return TypeInfoFactory.timestampTypeInfo; + return TypeInfoFactory.timestampLocalTZTypeInfo; } else { // EXTRACT from timestamp return TypeInfoFactory.intTypeInfo; diff --git druid-handler/src/test/org/apache/hadoop/hive/druid/serde/TestDruidSerDe.java druid-handler/src/test/org/apache/hadoop/hive/druid/serde/TestDruidSerDe.java index 655a96a..2d351e4 100644 --- druid-handler/src/test/org/apache/hadoop/hive/druid/serde/TestDruidSerDe.java +++ druid-handler/src/test/org/apache/hadoop/hive/druid/serde/TestDruidSerDe.java @@ -28,6 +28,8 @@ import java.io.InputStream; import java.lang.reflect.InvocationTargetException; import java.sql.Timestamp; +import java.time.Instant; +import java.time.ZoneOffset; import java.util.ArrayList; import java.util.List; import java.util.Map.Entry; @@ -47,6 +49,7 @@ import org.apache.hadoop.hive.common.type.HiveChar; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.hive.common.type.TimestampTZ; import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.druid.DruidStorageHandlerUtils; import org.apache.hadoop.hive.druid.QTestDruidSerDe; @@ -62,6 +65,7 @@ import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; +import org.apache.hadoop.hive.serde2.io.TimestampLocalTZWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; @@ -148,19 +152,19 @@ // Timeseries query results as records private static final Object[][] TIMESERIES_QUERY_RESULTS_RECORDS = new Object[][] { - new Object[] { new TimestampWritable(new Timestamp(1325376000000L)), new LongWritable(0), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1325376000000L).atZone(ZoneOffset.UTC))), new LongWritable(0), new FloatWritable(1.0F), new FloatWritable(2.2222F) }, - new Object[] { new TimestampWritable(new Timestamp(1325462400000L)), new LongWritable(2), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1325462400000L).atZone(ZoneOffset.UTC))), new LongWritable(2), new FloatWritable(3.32F), new FloatWritable(4F) } }; // Timeseries query results as records (types defined by metastore) private static final String TIMESERIES_COLUMN_NAMES = "__time,sample_name1,sample_name2,sample_divide"; - private static final String TIMESERIES_COLUMN_TYPES = "timestamp,smallint,double,float"; + private static final String TIMESERIES_COLUMN_TYPES = "timestamp with local time zone,smallint,double,float"; private static final Object[][] TIMESERIES_QUERY_RESULTS_RECORDS_2 = new Object[][] { - new Object[] { new TimestampWritable(new Timestamp(1325376000000L)), new ShortWritable((short) 0), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1325376000000L).atZone(ZoneOffset.UTC))), new ShortWritable((short) 0), new DoubleWritable(1.0d), new FloatWritable(2.2222F) }, - new Object[] { new TimestampWritable(new Timestamp(1325462400000L)), new ShortWritable((short) 2), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1325462400000L).atZone(ZoneOffset.UTC))), new ShortWritable((short) 2), new DoubleWritable(3.32d), new FloatWritable(4F) } }; @@ -263,40 +267,40 @@ // TopN query results as records private static final Object[][] TOPN_QUERY_RESULTS_RECORDS = new Object[][] { - new Object[] { new TimestampWritable(new Timestamp(1377907200000L)), new Text("dim1_val"), - new LongWritable(111), new FloatWritable(10669F), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))), + new Text("dim1_val"), new LongWritable(111), new FloatWritable(10669F), new FloatWritable(96.11711711711712F) }, - new Object[] { new TimestampWritable(new Timestamp(1377907200000L)), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))), new Text("another_dim1_val"), new LongWritable(88), new FloatWritable(28344F), new FloatWritable(322.09090909090907F) }, - new Object[] { new TimestampWritable(new Timestamp(1377907200000L)), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))), new Text("dim1_val3"), new LongWritable(70), new FloatWritable(871F), new FloatWritable(12.442857142857143F) }, - new Object[] { new TimestampWritable(new Timestamp(1377907200000L)), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))), new Text("dim1_val4"), new LongWritable(62), new FloatWritable(815F), new FloatWritable(13.14516129032258F) }, - new Object[] { new TimestampWritable(new Timestamp(1377907200000L)), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))), new Text("dim1_val5"), new LongWritable(60), new FloatWritable(2787F), new FloatWritable(46.45F) } }; // TopN query results as records (types defined by metastore) private static final String TOPN_COLUMN_NAMES = "__time,sample_dim,count,some_metric,sample_divide"; - private static final String TOPN_COLUMN_TYPES = "timestamp,string,bigint,double,float"; + private static final String TOPN_COLUMN_TYPES = "timestamp with local time zone,string,bigint,double,float"; private static final Object[][] TOPN_QUERY_RESULTS_RECORDS_2 = new Object[][] { - new Object[] { new TimestampWritable(new Timestamp(1377907200000L)), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))), new Text("dim1_val"), new LongWritable(111), new DoubleWritable(10669d), new FloatWritable(96.11711711711712F) }, - new Object[] { new TimestampWritable(new Timestamp(1377907200000L)), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))), new Text("another_dim1_val"), new LongWritable(88), new DoubleWritable(28344d), new FloatWritable(322.09090909090907F) }, - new Object[] { new TimestampWritable(new Timestamp(1377907200000L)), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))), new Text("dim1_val3"), new LongWritable(70), new DoubleWritable(871d), new FloatWritable(12.442857142857143F) }, - new Object[] { new TimestampWritable(new Timestamp(1377907200000L)), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))), new Text("dim1_val4"), new LongWritable(62), new DoubleWritable(815d), new FloatWritable(13.14516129032258F) }, - new Object[] { new TimestampWritable(new Timestamp(1377907200000L)), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))), new Text("dim1_val5"), new LongWritable(60), new DoubleWritable(2787d), new FloatWritable(46.45F) } }; @@ -425,38 +429,39 @@ // GroupBy query results as records private static final Object[][] GROUP_BY_QUERY_EXTRACTION_RESULTS_RECORDS = new Object[][] { - new Object[] { new TimestampWritable(new Timestamp(1325376000000L)), - new TimestampWritable(new Timestamp(1325376000000L)), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1325376000000L).atZone(ZoneOffset.UTC))), + new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1325376000000L).atZone(ZoneOffset.UTC))), new LongWritable(200) }, - new Object[] { new TimestampWritable(new Timestamp(1325376012000L)), new TimestampWritable(new Timestamp(1325376012000L)), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1325376012000L).atZone(ZoneOffset.UTC))), + new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1325376012000L).atZone(ZoneOffset.UTC))), new LongWritable(400) } }; private static final Object[][] GROUP_BY_QUERY_RESULTS_RECORDS = new Object[][] { - new Object[] { new TimestampWritable(new Timestamp(1325376000000L)), new Text("India"), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1325376000000L).atZone(ZoneOffset.UTC))), new Text("India"), new Text("phone"), new LongWritable(88), new FloatWritable(29.91233453F), new FloatWritable(60.32F) }, - new Object[] { new TimestampWritable(new Timestamp(1325376012000L)), new Text("Spain"), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1325376012000L).atZone(ZoneOffset.UTC))), new Text("Spain"), new Text("pc"), new LongWritable(16), new FloatWritable(172.93494959F), new FloatWritable(6.333333F) } }; private static final Object[][] GB_MONTH_EXTRACTION_RESULTS_RECORDS = new Object[][] { - new Object[] { new TimestampWritable(new Timestamp(1325376000000L)), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1325376000000L).atZone(ZoneOffset.UTC))), new IntWritable(1), new LongWritable(200) }, - new Object[] { new TimestampWritable(new Timestamp(1325376012000L)), new IntWritable(1), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1325376012000L).atZone(ZoneOffset.UTC))), new IntWritable(1), new LongWritable(400) } }; // GroupBy query results as records (types defined by metastore) private static final String GROUP_BY_COLUMN_NAMES = "__time,country,device,total_usage,data_transfer,avg_usage"; - private static final String GROUP_BY_COLUMN_TYPES = "timestamp,string,string,int,double,float"; + private static final String GROUP_BY_COLUMN_TYPES = "timestamp with local time zone,string,string,int,double,float"; private static final Object[][] GROUP_BY_QUERY_RESULTS_RECORDS_2 = new Object[][] { - new Object[] { new TimestampWritable(new Timestamp(1325376000000L)), new Text("India"), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1325376000000L).atZone(ZoneOffset.UTC))), new Text("India"), new Text("phone"), new IntWritable(88), new DoubleWritable(29.91233453), new FloatWritable(60.32F) }, - new Object[] { new TimestampWritable(new Timestamp(1325376012000L)), new Text("Spain"), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1325376012000L).atZone(ZoneOffset.UTC))), new Text("Spain"), new Text("pc"), new IntWritable(16), new DoubleWritable(172.93494959), new FloatWritable(6.333333F) } }; @@ -577,28 +582,28 @@ // Select query results as records private static final Object[][] SELECT_QUERY_RESULTS_RECORDS = new Object[][] { - new Object[] { new TimestampWritable(new Timestamp(1356998400000L)), new Text("1"), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1356998400000L).atZone(ZoneOffset.UTC))), new Text("1"), new Text("article"), new Text("0"), new Text("0"), new Text("11._korpus_(NOVJ)"), new Text("sl"), new Text("0"), new Text("EmausBot"), new FloatWritable(1.0F), new FloatWritable(39.0F), new FloatWritable(39.0F), new FloatWritable(39.0F), new FloatWritable(0.0F) }, - new Object[] { new TimestampWritable(new Timestamp(1356998400000L)), new Text("0"), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1356998400000L).atZone(ZoneOffset.UTC))), new Text("0"), new Text("article"), new Text("0"), new Text("0"), new Text("112_U.S._580"), new Text("en"), new Text("1"), new Text("MZMcBride"), new FloatWritable(1.0F), new FloatWritable(70.0F), new FloatWritable(70.0F), new FloatWritable(70.0F), new FloatWritable(0.0F) }, - new Object[] { new TimestampWritable(new Timestamp(1356998412000L)), new Text("0"), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1356998412000L).atZone(ZoneOffset.UTC))), new Text("0"), new Text("article"), new Text("0"), new Text("0"), new Text("113_U.S._243"), new Text("en"), new Text("1"), new Text("MZMcBride"), new FloatWritable(1.0F), new FloatWritable(77.0F), new FloatWritable(77.0F), new FloatWritable(77.0F), new FloatWritable(0.0F) }, - new Object[] { new TimestampWritable(new Timestamp(1356998412000L)), new Text("0"), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1356998412000L).atZone(ZoneOffset.UTC))), new Text("0"), new Text("article"), new Text("0"), new Text("0"), new Text("113_U.S._73"), new Text("en"), new Text("1"), new Text("MZMcBride"), new FloatWritable(1.0F), new FloatWritable(70.0F), new FloatWritable(70.0F), new FloatWritable(70.0F), new FloatWritable(0.0F) }, - new Object[] { new TimestampWritable(new Timestamp(1356998412000L)), new Text("0"), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1356998412000L).atZone(ZoneOffset.UTC))), new Text("0"), new Text("article"), new Text("0"), new Text("0"), new Text("113_U.S._756"), new Text("en"), new Text("1"), new Text("MZMcBride"), new FloatWritable(1.0F), new FloatWritable(68.0F), new FloatWritable(68.0F), @@ -607,30 +612,30 @@ // Select query results as records (types defined by metastore) private static final String SELECT_COLUMN_NAMES = "__time,robot,namespace,anonymous,unpatrolled,page,language,newpage,user,count,added,delta,variation,deleted"; - private static final String SELECT_COLUMN_TYPES = "timestamp,string,string,string,string,string,string,string,string,double,double,float,float,float"; + private static final String SELECT_COLUMN_TYPES = "timestamp with local time zone,string,string,string,string,string,string,string,string,double,double,float,float,float"; private static final Object[][] SELECT_QUERY_RESULTS_RECORDS_2 = new Object[][] { - new Object[] { new TimestampWritable(new Timestamp(1356998400000L)), new Text("1"), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1356998400000L).atZone(ZoneOffset.UTC))), new Text("1"), new Text("article"), new Text("0"), new Text("0"), new Text("11._korpus_(NOVJ)"), new Text("sl"), new Text("0"), new Text("EmausBot"), new DoubleWritable(1.0d), new DoubleWritable(39.0d), new FloatWritable(39.0F), new FloatWritable(39.0F), new FloatWritable(0.0F) }, - new Object[] { new TimestampWritable(new Timestamp(1356998400000L)), new Text("0"), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1356998400000L).atZone(ZoneOffset.UTC))), new Text("0"), new Text("article"), new Text("0"), new Text("0"), new Text("112_U.S._580"), new Text("en"), new Text("1"), new Text("MZMcBride"), new DoubleWritable(1.0d), new DoubleWritable(70.0d), new FloatWritable(70.0F), new FloatWritable(70.0F), new FloatWritable(0.0F) }, - new Object[] { new TimestampWritable(new Timestamp(1356998412000L)), new Text("0"), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1356998412000L).atZone(ZoneOffset.UTC))), new Text("0"), new Text("article"), new Text("0"), new Text("0"), new Text("113_U.S._243"), new Text("en"), new Text("1"), new Text("MZMcBride"), new DoubleWritable(1.0d), new DoubleWritable(77.0d), new FloatWritable(77.0F), new FloatWritable(77.0F), new FloatWritable(0.0F) }, - new Object[] { new TimestampWritable(new Timestamp(1356998412000L)), new Text("0"), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1356998412000L).atZone(ZoneOffset.UTC))), new Text("0"), new Text("article"), new Text("0"), new Text("0"), new Text("113_U.S._73"), new Text("en"), new Text("1"), new Text("MZMcBride"), new DoubleWritable(1.0d), new DoubleWritable(70.0d), new FloatWritable(70.0F), new FloatWritable(70.0F), new FloatWritable(0.0F) }, - new Object[] { new TimestampWritable(new Timestamp(1356998412000L)), new Text("0"), + new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1356998412000L).atZone(ZoneOffset.UTC))), new Text("0"), new Text("article"), new Text("0"), new Text("0"), new Text("113_U.S._756"), new Text("en"), new Text("1"), new Text("MZMcBride"), new DoubleWritable(1.0d), new DoubleWritable(68.0d), new FloatWritable(68.0F), @@ -827,9 +832,9 @@ private void deserializeQueryResults(DruidSerDe serDe, String queryType, String private static final String COLUMN_NAMES = "__time,c0,c1,c2,c3,c4,c5,c6,c7,c8,c9"; - private static final String COLUMN_TYPES = "timestamp,string,char(6),varchar(8),double,float,decimal(38,18),bigint,int,smallint,tinyint"; + private static final String COLUMN_TYPES = "timestamp with local time zone,string,char(6),varchar(8),double,float,decimal(38,18),bigint,int,smallint,tinyint"; private static final Object[] ROW_OBJECT = new Object[] { - new TimestampWritable(new Timestamp(1377907200000L)), + new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))), new Text("dim1_val"), new HiveCharWritable(new HiveChar("dim2_v", 6)), new HiveVarcharWritable(new HiveVarchar("dim3_val", 8)), @@ -933,7 +938,7 @@ public ObjectInspector apply(PrimitiveTypeInfo type) { } private static final Object[] ROW_OBJECT_2 = new Object[] { - new TimestampWritable(new Timestamp(1377907200000L)), + new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))), new Text("dim1_val"), new HiveCharWritable(new HiveChar("dim2_v", 6)), new HiveVarcharWritable(new HiveVarchar("dim3_val", 8)), diff --git pom.xml pom.xml index ca088da..009987b 100644 --- pom.xml +++ pom.xml @@ -119,7 +119,7 @@ 1.10.0 1.7.7 0.8.0.RELEASE - 1.13.0 + 1.14.0 4.2.4 4.1.17 4.1.19 diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java index 1d1d2a3..2308bf4 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java @@ -497,6 +497,8 @@ system.registerGenericUDF("to_unix_timestamp", GenericUDFToUnixTimeStamp.class); system.registerGenericUDF("internal_interval", GenericUDFInternalInterval.class); + + system.registerGenericUDF("to_epoch_milli", GenericUDFEpochMilli.class); // Generic UDTF's system.registerGenericUDTF("explode", GenericUDTFExplode.class); system.registerGenericUDTF("replicate_rows", GenericUDTFReplicateRows.class); diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 68c9482..793ce9e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -1509,7 +1509,7 @@ public Table apply(org.apache.hadoop.hive.metastore.api.Table table) { Map qualifiedNameToView = new HashMap(); for (RelOptMaterialization materialization : cachedViews) { - qualifiedNameToView.put(materialization.table.getQualifiedName().get(0), materialization); + qualifiedNameToView.put(materialization.qualifiedTableName.get(0), materialization); } for (String table : tables) { // Compose qualified name diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java index 154ea68..423fcf0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java @@ -188,7 +188,8 @@ public RelOptMaterialization addMaterializedView(Table materializedViewTable) { " ignored; error parsing original query"); return null; } - RelOptMaterialization materialization = new RelOptMaterialization(tableRel, queryRel, null); + RelOptMaterialization materialization = new RelOptMaterialization(tableRel, queryRel, + null, tableRel.getTable().getQualifiedName()); cq.put(vk, materialization); if (LOG.isDebugEnabled()) { LOG.debug("Cached materialized view for rewriting: " + tableRel.getTable().getQualifiedName()); @@ -317,7 +318,8 @@ private static RelNode createTableScan(Table viewTable) { List intervals = Arrays.asList(DruidTable.DEFAULT_INTERVAL); DruidTable druidTable = new DruidTable(new DruidSchema(address, address, false), - dataSource, RelDataTypeImpl.proto(rowType), metrics, DruidTable.DEFAULT_TIMESTAMP_COLUMN, intervals); + dataSource, RelDataTypeImpl.proto(rowType), metrics, DruidTable.DEFAULT_TIMESTAMP_COLUMN, + intervals, null, null); final TableScan scan = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), optTable, viewTable.getTableName(), null, false, false); tableRel = DruidQuery.create(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionTimeGranularityOptimizer.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionTimeGranularityOptimizer.java index 4a248d6..b9c7965 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionTimeGranularityOptimizer.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionTimeGranularityOptimizer.java @@ -62,6 +62,8 @@ import org.apache.hadoop.hive.ql.udf.UDFDateFloorWeek; import org.apache.hadoop.hive.ql.udf.UDFDateFloorYear; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFEpochMilli; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFTimestamp; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; @@ -158,9 +160,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, fsParent.getChildOperators().clear(); // Create SelectOp with granularity column - Operator granularitySelOp = getGranularitySelOp(fsParent, - segmentGranularity - ); + Operator granularitySelOp = getGranularitySelOp(fsParent, segmentGranularity); // Create ReduceSinkOp operator ArrayList parentCols = Lists.newArrayList(granularitySelOp.getSchema().getSignature()); @@ -230,17 +230,17 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, descs.add(columnDesc); colNames.add(columnDesc.getExprString()); if (columnDesc.getTypeInfo().getCategory() == ObjectInspector.Category.PRIMITIVE - && ((PrimitiveTypeInfo) columnDesc.getTypeInfo()).getPrimitiveCategory() == PrimitiveCategory.TIMESTAMP) { + && ((PrimitiveTypeInfo) columnDesc.getTypeInfo()).getPrimitiveCategory() == PrimitiveCategory.TIMESTAMPLOCALTZ) { if (timestampPos != -1) { - throw new SemanticException("Multiple columns with timestamp type on query result; " - + "could not resolve which one is the timestamp column"); + throw new SemanticException("Multiple columns with timestamp with local time-zone type on query result; " + + "could not resolve which one is the timestamp with local time-zone column"); } timestampPos = i; } } if (timestampPos == -1) { - throw new SemanticException("No column with timestamp type on query result; " - + "one column should be of timestamp type"); + throw new SemanticException("No column with timestamp with local time-zone type on query result; " + + "one column should be of timestamp with local time-zone type"); } RowSchema selRS = new RowSchema(fsParent.getSchema()); // Granularity (partition) column @@ -279,11 +279,20 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, default: throw new SemanticException("Granularity for Druid segment not recognized"); } + // Read the column value ExprNodeDesc expr = new ExprNodeColumnDesc(parentCols.get(timestampPos)); - descs.add(new ExprNodeGenericFuncDesc( - TypeInfoFactory.timestampTypeInfo, - new GenericUDFBridge(udfName, false, udfClass.getName()), - Lists.newArrayList(expr))); + // UTC epoch for instant + ExprNodeGenericFuncDesc f1 = new ExprNodeGenericFuncDesc( + TypeInfoFactory.longTypeInfo, new GenericUDFEpochMilli(), Lists.newArrayList(expr)); + // Cast to timestamp + ExprNodeGenericFuncDesc f2 = new ExprNodeGenericFuncDesc( + TypeInfoFactory.timestampTypeInfo, new GenericUDFTimestamp(), Lists.newArrayList(f1)); + // We apply the granularity function + ExprNodeGenericFuncDesc f3 = new ExprNodeGenericFuncDesc( + TypeInfoFactory.timestampTypeInfo, + new GenericUDFBridge(udfName, false, udfClass.getName()), + Lists.newArrayList(f2)); + descs.add(f3); colNames.add(Constants.DRUID_TIMESTAMP_GRANULARITY_COL_NAME); // Add granularity to the row schema ColumnInfo ci = new ColumnInfo(Constants.DRUID_TIMESTAMP_GRANULARITY_COL_NAME, TypeInfoFactory.timestampTypeInfo, diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java index f70aa75..692d7a6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java @@ -964,7 +964,7 @@ public static AggregateCall createSingleArgAggCall(String funcName, RelOptCluste aggArgRelDTBldr.build(), aggFnRetType); List argList = new ArrayList(); argList.add(pos); - return new AggregateCall(aggFunction, false, argList, aggFnRetType, null); + return AggregateCall.create(aggFunction, false, argList, -1, aggFnRetType, null); } public static HiveTableFunctionScan createUDTFForSetOp(RelOptCluster cluster, RelNode input) diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveConfPlannerContext.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveConfPlannerContext.java index 756b671..60c3bdd 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveConfPlannerContext.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveConfPlannerContext.java @@ -17,10 +17,6 @@ */ package org.apache.hadoop.hive.ql.optimizer.calcite; -import org.apache.calcite.rel.RelNode; - -import java.util.Set; - public class HiveConfPlannerContext{ private boolean isCorrelatedColumns; diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java index 823b099..afebad3 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java @@ -44,10 +44,10 @@ import org.apache.calcite.tools.RelBuilderFactory; import org.apache.calcite.util.ImmutableBitSet; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveExcept; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveIntersect; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin; -import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveExcept; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSemiJoin; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit; @@ -193,7 +193,11 @@ public RelNode createSort(RelNode input, RelCollation collation, RexNode offset, public RelNode createAggregate(RelNode child, boolean indicator, ImmutableBitSet groupSet, ImmutableList groupSets, List aggCalls) { - return new HiveAggregate(child.getCluster(), child.getTraitSet(), child, indicator, + if (indicator) { + throw new IllegalStateException("Hive does not support indicator columns but Calcite " + + "created an Aggregate operator containing them"); + } + return new HiveAggregate(child.getCluster(), child.getTraitSet(), child, groupSet, groupSets, aggCalls); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveType.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveType.java deleted file mode 100644 index 31a628b..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveType.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.optimizer.calcite; - -import org.apache.calcite.sql.type.AbstractSqlType; -import org.apache.calcite.sql.type.SqlTypeName; - -/** - * Hive-specific type. - * - * TODO: Created to represent timestamp with time-zone type. - * It can be removed once the type exists in Calcite. - */ -public class HiveType extends AbstractSqlType { - private final Class clazz; - - public HiveType(Class clazz) { - super(SqlTypeName.NULL, true, null); - this.clazz = clazz; - computeDigest(); - } - - protected void generateTypeString(StringBuilder sb, boolean withDetail) { - sb.append("HiveType("); - sb.append(clazz); - sb.append(")"); - } - - public Class getTypeClass() { - return clazz; - } - -} diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeSystemImpl.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeSystemImpl.java index dad783d..226498b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeSystemImpl.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeSystemImpl.java @@ -70,6 +70,7 @@ public int getDefaultPrecision(SqlTypeName typeName) { case VARBINARY: case TIME: case TIMESTAMP: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: return getMaxPrecision(typeName); case CHAR: return DEFAULT_CHAR_PRECISION; @@ -109,6 +110,7 @@ public int getMaxPrecision(SqlTypeName typeName) { return MAX_BINARY_PRECISION; case TIME: case TIMESTAMP: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: return MAX_TIMESTAMP_PRECISION; case INTERVAL_YEAR: case INTERVAL_MONTH: diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/functions/HiveSqlCountAggFunction.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/functions/HiveSqlCountAggFunction.java index 75b7ad2..08f99b0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/functions/HiveSqlCountAggFunction.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/functions/HiveSqlCountAggFunction.java @@ -17,19 +17,29 @@ */ package org.apache.hadoop.hive.ql.optimizer.calcite.functions; +import java.util.ArrayList; +import java.util.List; + import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlFunctionCategory; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlSplittableAggFunction; import org.apache.calcite.sql.SqlSplittableAggFunction.CountSplitter; +import org.apache.calcite.sql.SqlSplittableAggFunction.Registry; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.type.SqlOperandTypeChecker; import org.apache.calcite.sql.type.SqlOperandTypeInference; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.ImmutableIntList; +import com.google.common.collect.ImmutableList; + public class HiveSqlCountAggFunction extends SqlAggFunction implements CanAggregateDistinct { final boolean isDistinct; @@ -69,11 +79,40 @@ public boolean isDistinct() { @Override public AggregateCall other(RelDataTypeFactory typeFactory, AggregateCall e) { - return AggregateCall.create( new HiveSqlCountAggFunction(isDistinct, returnTypeInference, operandTypeInference, operandTypeChecker), false, ImmutableIntList.of(), -1, typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.BIGINT), true), "count"); } + + @Override + public AggregateCall topSplit(RexBuilder rexBuilder, + Registry extra, int offset, RelDataType inputRowType, + AggregateCall aggregateCall, int leftSubTotal, int rightSubTotal) { + final List merges = new ArrayList<>(); + if (leftSubTotal >= 0) { + merges.add( + rexBuilder.makeInputRef(aggregateCall.type, leftSubTotal)); + } + if (rightSubTotal >= 0) { + merges.add( + rexBuilder.makeInputRef(aggregateCall.type, rightSubTotal)); + } + RexNode node; + switch (merges.size()) { + case 1: + node = merges.get(0); + break; + case 2: + node = rexBuilder.makeCall(SqlStdOperatorTable.MULTIPLY, merges); + break; + default: + throw new AssertionError("unexpected count " + merges); + } + int ordinal = extra.register(node); + return AggregateCall.create( + new HiveSqlSumEmptyIsZeroAggFunction(isDistinct, returnTypeInference, operandTypeInference, operandTypeChecker), + false, ImmutableList.of(ordinal), -1, aggregateCall.type, aggregateCall.name); + } } } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/functions/HiveSqlSumEmptyIsZeroAggFunction.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/functions/HiveSqlSumEmptyIsZeroAggFunction.java new file mode 100644 index 0000000..31f6022 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/functions/HiveSqlSumEmptyIsZeroAggFunction.java @@ -0,0 +1,59 @@ +package org.apache.hadoop.hive.ql.optimizer.calcite.functions; + +import java.util.List; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.sql.SqlAggFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlSplittableAggFunction; +import org.apache.calcite.sql.type.SqlOperandTypeChecker; +import org.apache.calcite.sql.type.SqlOperandTypeInference; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeName; + +import com.google.common.collect.ImmutableList; + +/** + * Sum0 is an aggregator which returns the sum of the values which + * go into it like Sum. It differs in that when no non null values + * are applied zero is returned instead of null. Can be used along with + * Count to implement Sum. + */ +public class HiveSqlSumEmptyIsZeroAggFunction extends SqlAggFunction { + //~ Constructors ----------------------------------------------------------- + + public HiveSqlSumEmptyIsZeroAggFunction(boolean isDistinct, SqlReturnTypeInference returnTypeInference, + SqlOperandTypeInference operandTypeInference, SqlOperandTypeChecker operandTypeChecker) { + super("$SUM0", + null, + SqlKind.SUM0, + returnTypeInference, + operandTypeInference, + operandTypeChecker, + SqlFunctionCategory.NUMERIC, + false, + false); + } + + //~ Methods ---------------------------------------------------------------- + + public List getParameterTypes(RelDataTypeFactory typeFactory) { + return ImmutableList.of( + typeFactory.createTypeWithNullability( + typeFactory.createSqlType(SqlTypeName.ANY), true)); + } + + public RelDataType getReturnType(RelDataTypeFactory typeFactory) { + return typeFactory.createTypeWithNullability( + typeFactory.createSqlType(SqlTypeName.ANY), true); + } + + @Override public T unwrap(Class clazz) { + if (clazz == SqlSplittableAggFunction.class) { + return clazz.cast(SqlSplittableAggFunction.SumSplitter.INSTANCE); + } + return super.unwrap(clazz); + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java index fa51b4e..1be15bc 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java @@ -34,8 +34,8 @@ import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.ImmutableBitSet; -import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelShuttle; +import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil; import com.google.common.collect.Sets; @@ -45,18 +45,22 @@ public HiveAggregate(RelOptCluster cluster, RelTraitSet traitSet, RelNode child, - boolean indicator, ImmutableBitSet groupSet, List groupSets, + ImmutableBitSet groupSet, List groupSets, List aggCalls) { - super(cluster, TraitsUtil.getDefaultTraitSet(cluster), child, indicator, groupSet, - groupSets, aggCalls); + super(cluster, TraitsUtil.getDefaultTraitSet(cluster), child, false, + groupSet, groupSets, aggCalls); } @Override public Aggregate copy(RelTraitSet traitSet, RelNode input, boolean indicator, ImmutableBitSet groupSet, List groupSets, List aggCalls) { - return new HiveAggregate(getCluster(), traitSet, input, indicator, groupSet, - groupSets, aggCalls); + if (indicator) { + throw new IllegalStateException("Hive does not support indicator columns but tried " + + "to create an Aggregate operator containing them"); + } + return new HiveAggregate(getCluster(), traitSet, input, + groupSet, groupSets, aggCalls); } @Override diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveGroupingID.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveGroupingID.java index 2fb9508..adcda26 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveGroupingID.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveGroupingID.java @@ -17,17 +17,17 @@ */ package org.apache.hadoop.hive.ql.optimizer.calcite.reloperators; -import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlFunctionCategory; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlAbstractGroupFunction; import org.apache.calcite.sql.type.InferTypes; import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; -public class HiveGroupingID extends SqlAggFunction { +public class HiveGroupingID extends SqlAbstractGroupFunction { - public static final SqlAggFunction INSTANCE = + public static final HiveGroupingID INSTANCE = new HiveGroupingID(); private HiveGroupingID() { @@ -36,7 +36,7 @@ private HiveGroupingID() { ReturnTypes.INTEGER, InferTypes.BOOLEAN, OperandTypes.NILADIC, - SqlFunctionCategory.USER_DEFINED_FUNCTION); + SqlFunctionCategory.SYSTEM); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateJoinTransposeRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateJoinTransposeRule.java index 21b46cf..a09e7f6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateJoinTransposeRule.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateJoinTransposeRule.java @@ -33,9 +33,9 @@ import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.JoinRelType; -import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.rules.AggregateJoinTransposeRule; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexInputRef; @@ -43,6 +43,8 @@ import org.apache.calcite.rex.RexUtil; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlSplittableAggFunction; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.tools.RelBuilderFactory; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.mapping.Mapping; import org.apache.calcite.util.mapping.Mappings; @@ -64,29 +66,17 @@ /** Extended instance of the rule that can push down aggregate functions. */ public static final HiveAggregateJoinTransposeRule INSTANCE = - new HiveAggregateJoinTransposeRule(HiveAggregate.class, HiveRelFactories.HIVE_AGGREGATE_FACTORY, - HiveJoin.class, HiveRelFactories.HIVE_JOIN_FACTORY, HiveRelFactories.HIVE_PROJECT_FACTORY, - true); - - private final RelFactories.AggregateFactory aggregateFactory; - - private final RelFactories.JoinFactory joinFactory; - - private final RelFactories.ProjectFactory projectFactory; + new HiveAggregateJoinTransposeRule(HiveAggregate.class, HiveJoin.class, + HiveRelFactories.HIVE_BUILDER, true); private final boolean allowFunctions; /** Creates an AggregateJoinTransposeRule that may push down functions. */ private HiveAggregateJoinTransposeRule(Class aggregateClass, - RelFactories.AggregateFactory aggregateFactory, Class joinClass, - RelFactories.JoinFactory joinFactory, - RelFactories.ProjectFactory projectFactory, + RelBuilderFactory relBuilderFactory, boolean allowFunctions) { - super(aggregateClass, aggregateFactory, joinClass, joinFactory, projectFactory, true); - this.aggregateFactory = aggregateFactory; - this.joinFactory = joinFactory; - this.projectFactory = projectFactory; + super(aggregateClass, joinClass, relBuilderFactory, true); this.allowFunctions = allowFunctions; } @@ -95,6 +85,7 @@ public void onMatch(RelOptRuleCall call) { final Aggregate aggregate = call.rel(0); final Join join = call.rel(1); final RexBuilder rexBuilder = aggregate.getCluster().getRexBuilder(); + final RelBuilder relBuilder = call.builder(); // If any aggregate functions do not support splitting, bail out // If any aggregate call has a filter, bail out @@ -119,8 +110,8 @@ public void onMatch(RelOptRuleCall call) { } // Do the columns used by the join appear in the output of the aggregate? - final RelMetadataQuery mq = call.getMetadataQuery(); final ImmutableBitSet aggregateColumns = aggregate.getGroupSet(); + final RelMetadataQuery mq = call.getMetadataQuery(); final ImmutableBitSet keyColumns = keyColumns(aggregateColumns, mq.getPulledUpPredicates(join).pulledUpPredicates); final ImmutableBitSet joinColumns = @@ -171,8 +162,8 @@ public void onMatch(RelOptRuleCall call) { // any functions experiencing a cartesian product effect. // // But finding out whether the input is already unique requires a call - // to areColumnsUnique that currently (until [CALCITE-794] "Detect - // cycles when computing statistics" is fixed) places a heavy load on + // to areColumnsUnique that currently (until [CALCITE-1048] "Make + // metadata more robust" is fixed) places a heavy load on // the metadata system. // // So we choose to imagine the the input is already unique, which is @@ -213,8 +204,10 @@ public void onMatch(RelOptRuleCall call) { + belowAggCallRegistry.register(call1)); } } - side.newInput = aggregateFactory.createAggregate(joinInput, false, - belowAggregateKey, null, belowAggCalls); + side.newInput = relBuilder.push(joinInput) + .aggregate(relBuilder.groupKey(belowAggregateKey, null), + belowAggCalls) + .build(); } offset += fieldCount; belowOffset += side.newInput.getRowType().getFieldCount(); @@ -231,7 +224,6 @@ public void onMatch(RelOptRuleCall call) { // Update condition final Mapping mapping = (Mapping) Mappings.target( new Function() { - @Override public Integer apply(Integer a0) { return map.get(a0); } @@ -242,9 +234,9 @@ public Integer apply(Integer a0) { RexUtil.apply(mapping, join.getCondition()); // Create new join - RelNode newJoin = joinFactory.createJoin(sides.get(0).newInput, - sides.get(1).newInput, newCondition, join.getJoinType(), - join.getVariablesStopped(), join.isSemiJoinDone()); + relBuilder.push(sides.get(0).newInput) + .push(sides.get(1).newInput) + .join(join.getJoinType(), newCondition); // Aggregate above to sum up the sub-totals final List newAggCalls = new ArrayList<>(); @@ -252,7 +244,8 @@ public Integer apply(Integer a0) { aggregate.getGroupCount() + aggregate.getIndicatorCount(); final int newLeftWidth = sides.get(0).newInput.getRowType().getFieldCount(); final List projects = - new ArrayList<>(rexBuilder.identityProjects(newJoin.getRowType())); + new ArrayList<>( + rexBuilder.identityProjects(relBuilder.peek().getRowType())); for (Ord aggCall : Ord.zip(aggregate.getAggCallList())) { final SqlAggFunction aggregation = aggCall.e.getAggregation(); final SqlSplittableAggFunction splitter = @@ -262,47 +255,45 @@ public Integer apply(Integer a0) { final Integer rightSubTotal = sides.get(1).split.get(aggCall.i); newAggCalls.add( splitter.topSplit(rexBuilder, registry(projects), - groupIndicatorCount, newJoin.getRowType(), aggCall.e, + groupIndicatorCount, relBuilder.peek().getRowType(), aggCall.e, leftSubTotal == null ? -1 : leftSubTotal, rightSubTotal == null ? -1 : rightSubTotal + newLeftWidth)); } - RelNode r = newJoin; - b: - if (allColumnsInAggregate && newAggCalls.isEmpty() && - RelOptUtil.areRowTypesEqual(r.getRowType(), aggregate.getRowType(), false)) { - // no need to aggregate - } else { - r = RelOptUtil.createProject(r, projects, null, true, - relBuilderFactory.create(aggregate.getCluster(), null)); - if (allColumnsInAggregate) { - // let's see if we can convert - List projects2 = new ArrayList<>(); - for (int key : Mappings.apply(mapping, aggregate.getGroupSet())) { - projects2.add(rexBuilder.makeInputRef(r, key)); - } - for (AggregateCall newAggCall : newAggCalls) { - final SqlSplittableAggFunction splitter = - newAggCall.getAggregation() - .unwrap(SqlSplittableAggFunction.class); - if (splitter != null) { - projects2.add( - splitter.singleton(rexBuilder, r.getRowType(), newAggCall)); - } - } - if (projects2.size() - == aggregate.getGroupSet().cardinality() + newAggCalls.size()) { - // We successfully converted agg calls into projects. - r = RelOptUtil.createProject(r, projects2, null, true, - relBuilderFactory.create(aggregate.getCluster(), null)); - break b; + + relBuilder.project(projects); + + boolean aggConvertedToProjects = false; + if (allColumnsInAggregate) { + // let's see if we can convert aggregate into projects + List projects2 = new ArrayList<>(); + for (int key : Mappings.apply(mapping, aggregate.getGroupSet())) { + projects2.add(relBuilder.field(key)); + } + for (AggregateCall newAggCall : newAggCalls) { + final SqlSplittableAggFunction splitter = + newAggCall.getAggregation().unwrap(SqlSplittableAggFunction.class); + if (splitter != null) { + final RelDataType rowType = relBuilder.peek().getRowType(); + projects2.add(splitter.singleton(rexBuilder, rowType, newAggCall)); } } - r = aggregateFactory.createAggregate(r, aggregate.indicator, - Mappings.apply(mapping, aggregate.getGroupSet()), - Mappings.apply2(mapping, aggregate.getGroupSets()), newAggCalls); + if (projects2.size() + == aggregate.getGroupSet().cardinality() + newAggCalls.size()) { + // We successfully converted agg calls into projects. + relBuilder.project(projects2); + aggConvertedToProjects = true; + } + } + + if (!aggConvertedToProjects) { + relBuilder.aggregate( + relBuilder.groupKey(Mappings.apply(mapping, aggregate.getGroupSet()), + Mappings.apply2(mapping, aggregate.getGroupSets())), + newAggCalls); } // Make a cost based decision to pick cheaper plan + RelNode r = relBuilder.build(); RelOptCost afterCost = mq.getCumulativeCost(r); RelOptCost beforeCost = mq.getCumulativeCost(aggregate); if (afterCost.isLt(beforeCost)) { @@ -316,8 +307,8 @@ public Integer apply(Integer a0) { private static ImmutableBitSet keyColumns(ImmutableBitSet aggregateColumns, ImmutableList predicates) { SortedMap equivalence = new TreeMap<>(); - for (RexNode pred : predicates) { - populateEquivalences(equivalence, pred); + for (RexNode predicate : predicates) { + populateEquivalences(equivalence, predicate); } ImmutableBitSet keyColumns = aggregateColumns; for (Integer aggregateColumn : aggregateColumns) { @@ -358,10 +349,9 @@ private static void populateEquivalence(Map equivalence, /** Creates a {@link org.apache.calcite.sql.SqlSplittableAggFunction.Registry} * that is a view of a list. */ - private static SqlSplittableAggFunction.Registry - registry(final List list) { + private static SqlSplittableAggFunction.Registry registry( + final List list) { return new SqlSplittableAggFunction.Registry() { - @Override public int register(E e) { int i = list.indexOf(e); if (i < 0) { diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateProjectMergeRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateProjectMergeRule.java index 7c3b4b0..b5f1775 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateProjectMergeRule.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateProjectMergeRule.java @@ -20,16 +20,17 @@ import java.util.List; import java.util.Map; -import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.core.Aggregate.Group; import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.rules.AggregateProjectMergeRule; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.ImmutableBitSet; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; -import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveGroupingID; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; @@ -48,15 +49,13 @@ *

In some cases, this rule has the effect of trimming: the aggregate will * use fewer columns than the project did. */ -public class HiveAggregateProjectMergeRule extends RelOptRule { +public class HiveAggregateProjectMergeRule extends AggregateProjectMergeRule { public static final HiveAggregateProjectMergeRule INSTANCE = new HiveAggregateProjectMergeRule(); /** Private constructor. */ private HiveAggregateProjectMergeRule() { - super( - operand(HiveAggregate.class, - operand(HiveProject.class, any()))); + super(HiveAggregate.class, HiveProject.class, HiveRelFactories.HIVE_BUILDER); } @Override @@ -72,17 +71,16 @@ public boolean matches(RelOptRuleCall call) { return super.matches(call); } - @Override public void onMatch(RelOptRuleCall call) { final HiveAggregate aggregate = call.rel(0); final HiveProject project = call.rel(1); - RelNode x = apply(aggregate, project); + RelNode x = apply(call, aggregate, project); if (x != null) { call.transformTo(x); } } - public static RelNode apply(HiveAggregate aggregate, + public static RelNode apply(RelOptRuleCall call, HiveAggregate aggregate, HiveProject project) { final List newKeys = Lists.newArrayList(); final Map map = new HashMap<>(); @@ -100,7 +98,7 @@ public static RelNode apply(HiveAggregate aggregate, final ImmutableBitSet newGroupSet = aggregate.getGroupSet().permute(map); ImmutableList newGroupingSets = null; - if (aggregate.indicator) { + if (aggregate.getGroupType() != Group.SIMPLE) { newGroupingSets = ImmutableBitSet.ORDERING.immutableSortedCopy( ImmutableBitSet.permute(aggregate.getGroupSets(), map)); @@ -139,7 +137,8 @@ public static RelNode apply(HiveAggregate aggregate, // Add a project if the group set is not in the same order or // contains duplicates. - RelNode rel = newAggregate; + final RelBuilder relBuilder = call.builder(); + relBuilder.push(newAggregate); if (!newKeys.equals(newGroupSet.asList())) { final List posList = Lists.newArrayList(); for (int newKey : newKeys) { @@ -155,14 +154,12 @@ public static RelNode apply(HiveAggregate aggregate, i < newAggregate.getRowType().getFieldCount(); i++) { posList.add(i); } - rel = HiveRelOptUtil.createProject( - HiveRelFactories.HIVE_BUILDER.create(aggregate.getCluster(), null), - rel, posList); - + relBuilder.project(relBuilder.fields(posList)); } - return rel; + return relBuilder.build(); } + } -// End AggregateProjectMergeRule.java +// End HiveAggregateProjectMergeRule.java diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExceptRewriteRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExceptRewriteRule.java index b63ea02..5b72dbd 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExceptRewriteRule.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExceptRewriteRule.java @@ -27,7 +27,6 @@ import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; @@ -35,17 +34,13 @@ import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; -import org.apache.calcite.sql.SqlAggFunction; -import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.SqlOperator; -import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.calcite.util.ImmutableBitSet; import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveExcept; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; -import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveIntersect; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRelNode; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableFunctionScan; @@ -53,12 +48,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.translator.SqlFunctionConverter; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.TypeConverter; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; -import org.apache.calcite.tools.RelBuilder; -import org.apache.calcite.util.ImmutableBitSet; -import org.apache.calcite.util.Util; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -169,7 +159,7 @@ public void onMatch(RelOptRuleCall call) { final ImmutableBitSet groupSet = ImmutableBitSet.of(groupSetPositions); HiveRelNode aggregateRel = new HiveAggregate(cluster, - cluster.traitSetOf(HiveRelNode.CONVENTION), gbInputRel, false, groupSet, null, + cluster.traitSetOf(HiveRelNode.CONVENTION), gbInputRel, groupSet, null, aggregateCalls); // the schema after GB is like this @@ -270,7 +260,7 @@ private RelNode createFirstGB(RelNode input, boolean left, RelOptCluster cluster TypeInfoFactory.longTypeInfo, input.getRowType().getFieldList().size(), aggFnRetType); aggregateCalls.add(aggregateCall); return new HiveAggregate(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), gbInputRel, - false, groupSet, null, aggregateCalls); + groupSet, null, aggregateCalls); } private RexNode multiply(RexNode r1, RexNode r2, RelOptCluster cluster, RexBuilder rexBuilder) diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java index 81bb730..89c5c23 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java @@ -268,7 +268,7 @@ public RexNode apply(RelDataTypeField input) { aggregateCalls.add(aggregateCall); } Aggregate aggregate = new HiveAggregate(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), gbInputRel, - false, ImmutableBitSet.of(), null, aggregateCalls); + ImmutableBitSet.of(), null, aggregateCalls); // create the project after GB. For those repeated values, e.g., select // count(distinct x, y), count(distinct y, x), we find the correct mapping. @@ -330,7 +330,7 @@ private Aggregate createGroupingSets(Aggregate aggregate, List> ar .createSqlType(SqlTypeName.INTEGER), HiveGroupingID.INSTANCE.getName()); aggregateCalls.add(aggCall); return new HiveAggregate(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), - aggregate.getInput(), true, groupSet, origGroupSets, aggregateCalls); + aggregate.getInput(), groupSet, origGroupSets, aggregateCalls); } /** diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveIntersectRewriteRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveIntersectRewriteRule.java index 5b0a7d7..a5d950a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveIntersectRewriteRule.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveIntersectRewriteRule.java @@ -27,18 +27,14 @@ import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.AggregateCall; -import org.apache.calcite.rel.core.Intersect; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; -import org.apache.calcite.sql.SqlAggFunction; -import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.util.ImmutableBitSet; import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil; @@ -52,16 +48,10 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.translator.SqlFunctionConverter; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.TypeConverter; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; -import org.apache.calcite.tools.RelBuilder; -import org.apache.calcite.util.ImmutableBitSet; -import org.apache.calcite.util.Util; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.esotericsoftware.minlog.Log; import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList.Builder; @@ -140,7 +130,7 @@ public void onMatch(RelOptRuleCall call) { aggregateCalls.add(aggregateCall); HiveRelNode aggregateRel = new HiveAggregate(cluster, - cluster.traitSetOf(HiveRelNode.CONVENTION), gbInputRel, false, groupSet, null, + cluster.traitSetOf(HiveRelNode.CONVENTION), gbInputRel, groupSet, null, aggregateCalls); bldr.add(aggregateRel); } @@ -173,7 +163,7 @@ public void onMatch(RelOptRuleCall call) { final ImmutableBitSet groupSet = ImmutableBitSet.of(groupSetPositions); HiveRelNode aggregateRel = new HiveAggregate(cluster, - cluster.traitSetOf(HiveRelNode.CONVENTION), union, false, groupSet, null, aggregateCalls); + cluster.traitSetOf(HiveRelNode.CONVENTION), union, groupSet, null, aggregateCalls); // add a filter count(c) = #branches int countInd = cInd; diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelColumnsAlignment.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelColumnsAlignment.java index f35bf2f..4a35234 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelColumnsAlignment.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelColumnsAlignment.java @@ -35,6 +35,7 @@ import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.SetOp; import org.apache.calcite.rel.core.Sort; +import org.apache.calcite.rel.core.Aggregate.Group; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; @@ -96,7 +97,7 @@ public RelNode align(Aggregate rel, List collations) { // sort them so they respect it LinkedHashSet aggregateColumnsOrder = new LinkedHashSet<>(); ImmutableList.Builder propagateCollations = ImmutableList.builder(); - if (!rel.indicator && !collations.isEmpty()) { + if (rel.getGroupType() == Group.SIMPLE && !collations.isEmpty()) { for (RelFieldCollation c : collations) { if (c.getFieldIndex() < rel.getGroupCount()) { // Group column found diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java index b922faa..62125f0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java @@ -17,6 +17,23 @@ */ package org.apache.hadoop.hive.ql.optimizer.calcite.rules; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Objects; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.TreeSet; + +import javax.annotation.Nonnull; + import org.apache.calcite.linq4j.Ord; import org.apache.calcite.linq4j.function.Function2; import org.apache.calcite.plan.Context; @@ -48,7 +65,6 @@ import org.apache.calcite.rel.logical.LogicalProject; import org.apache.calcite.rel.metadata.RelMdUtil; import org.apache.calcite.rel.metadata.RelMetadataQuery; -import org.apache.calcite.rel.rules.FilterCorrelateRule; import org.apache.calcite.rel.rules.FilterJoinRule; import org.apache.calcite.rel.rules.FilterProjectTransposeRule; import org.apache.calcite.rel.type.RelDataType; @@ -83,12 +99,13 @@ import org.apache.calcite.util.Stacks; import org.apache.calcite.util.Util; import org.apache.calcite.util.mapping.Mappings; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelShuttleImpl; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit; -import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; +import org.apache.hadoop.hive.ql.parse.SemanticException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -104,24 +121,6 @@ import com.google.common.collect.Multimaps; import com.google.common.collect.Sets; import com.google.common.collect.SortedSetMultimap; -import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelShuttleImpl; - -import java.math.BigDecimal; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.NavigableMap; -import java.util.Objects; -import java.util.Set; -import java.util.SortedMap; -import java.util.TreeMap; -import java.util.TreeSet; -import javax.annotation.Nonnull; /** * NOTE: this whole logic is replicated from Calcite's RelDecorrelator @@ -824,7 +823,7 @@ public Frame decorrelateRel(HiveAggregate rel) throws SemanticException{ } relBuilder.push( - new HiveAggregate(rel.getCluster(), rel.getTraitSet(), newProject, false, newGroupSet, null, newAggCalls) ); + new HiveAggregate(rel.getCluster(), rel.getTraitSet(), newProject, newGroupSet, null, newAggCalls) ); if (!omittedConstants.isEmpty()) { final List postProjects = new ArrayList<>(relBuilder.fields()); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java index eab8f6f..67e1482 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java @@ -182,6 +182,7 @@ public static ASTNode literal(RexLiteral literal, boolean useTypeQualInLiteral) case DATE: case TIME: case TIMESTAMP: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: case INTERVAL_DAY: case INTERVAL_DAY_HOUR: case INTERVAL_DAY_MINUTE: @@ -280,6 +281,10 @@ public static ASTNode literal(RexLiteral literal, boolean useTypeQualInLiteral) val = "'" + literal.getValueAs(TimestampString.class).toString() + "'"; type = HiveParser.TOK_TIMESTAMPLITERAL; break; + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + val = "'" + literal.getValueAs(TimestampString.class).toString() + " UTC'"; + type = HiveParser.TOK_TIMESTAMPLOCALTZLITERAL; + break; case INTERVAL_YEAR: case INTERVAL_MONTH: case INTERVAL_YEAR_MONTH: { diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java index d6b978f..f01cf21 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java @@ -63,8 +63,6 @@ import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; -import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveExtractDate; -import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFloorDate; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveGroupingID; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableFunctionScan; @@ -134,22 +132,23 @@ private ASTNode convert() throws CalciteSemanticException { if (groupBy != null) { ASTBuilder b; boolean groupingSetsExpression = false; - if (groupBy.indicator) { - Group aggregateType = Aggregate.Group.induce(groupBy.getGroupSet(), - groupBy.getGroupSets()); - if (aggregateType == Group.ROLLUP) { + Group aggregateType = groupBy.getGroupType(); + switch (aggregateType) { + case SIMPLE: + b = ASTBuilder.construct(HiveParser.TOK_GROUPBY, "TOK_GROUPBY"); + break; + case ROLLUP: b = ASTBuilder.construct(HiveParser.TOK_ROLLUP_GROUPBY, "TOK_ROLLUP_GROUPBY"); - } - else if (aggregateType == Group.CUBE) { + break; + case CUBE: b = ASTBuilder.construct(HiveParser.TOK_CUBE_GROUPBY, "TOK_CUBE_GROUPBY"); - } - else { + break; + case OTHER: b = ASTBuilder.construct(HiveParser.TOK_GROUPING_SETS, "TOK_GROUPING_SETS"); groupingSetsExpression = true; - } - } - else { - b = ASTBuilder.construct(HiveParser.TOK_GROUPBY, "TOK_GROUPBY"); + break; + default: + throw new CalciteSemanticException("Group type not recognized"); } HiveAggregate hiveAgg = (HiveAggregate) groupBy; @@ -201,14 +200,15 @@ else if (aggregateType == Group.CUBE) { ASTBuilder b = ASTBuilder.construct(HiveParser.TOK_SELECT, "TOK_SELECT"); if (select instanceof Project) { - if (select.getChildExps().isEmpty()) { + List childExps = ((Project) select).getChildExps(); + if (childExps.isEmpty()) { RexLiteral r = select.getCluster().getRexBuilder().makeExactLiteral(new BigDecimal(1)); ASTNode selectExpr = ASTBuilder.selectExpr(ASTBuilder.literal(r), "1"); b.add(selectExpr); } else { int i = 0; - for (RexNode r : select.getChildExps()) { + for (RexNode r : childExps) { ASTNode expr = r.accept(new RexVisitor(schema, r instanceof RexLiteral, select.getCluster().getRexBuilder())); String alias = select.getRowType().getFieldNames().get(i++); @@ -770,15 +770,6 @@ public QueryBlockInfo(Schema schema, ASTNode ast) { ColumnInfo cI = src.get(i); add(cI); } - // If we are using grouping sets, we add the - // fields again, these correspond to the boolean - // grouping in Calcite. They are not used by Hive. - if(gBy.indicator) { - for (int i : gBy.getGroupSet()) { - ColumnInfo cI = src.get(i); - add(cI); - } - } List aggs = gBy.getAggCallList(); for (AggregateCall agg : aggs) { if (agg.getAggregation() == HiveGroupingID.INSTANCE) { diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java index 3dcceab..ebbb650 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java @@ -45,13 +45,12 @@ import org.apache.calcite.util.DateString; import org.apache.calcite.util.TimeString; import org.apache.calcite.util.TimestampString; +import org.apache.calcite.util.TimestampWithTimeZoneString; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; -import org.apache.hadoop.hive.common.type.TimestampTZ; import org.apache.hadoop.hive.ql.exec.UDFArgumentException; import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcFactory; -import org.apache.hadoop.hive.ql.optimizer.calcite.HiveType; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter.RexVisitor; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter.Schema; import org.apache.hadoop.hive.ql.parse.ASTNode; @@ -244,6 +243,8 @@ public ExprNodeDesc visitLiteral(RexLiteral literal) { case TIME: case TIMESTAMP: return new ExprNodeConstantDesc(TypeInfoFactory.timestampTypeInfo, null); + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return new ExprNodeConstantDesc(TypeInfoFactory.timestampLocalTZTypeInfo, null); case BINARY: return new ExprNodeConstantDesc(TypeInfoFactory.binaryTypeInfo, null); case DECIMAL: @@ -267,12 +268,7 @@ public ExprNodeDesc visitLiteral(RexLiteral literal) { case INTERVAL_MINUTE_SECOND: case INTERVAL_SECOND: return new ExprNodeConstantDesc(TypeInfoFactory.intervalDayTimeTypeInfo, null); - case NULL: - case OTHER: default: - if (lType instanceof HiveType && ((HiveType) lType).getTypeClass() == TimestampTZ.class) { - return new ExprNodeConstantDesc(TypeInfoFactory.timestampLocalTZTypeInfo, null); - } return new ExprNodeConstantDesc(TypeInfoFactory.voidTypeInfo, null); } } else { @@ -308,6 +304,9 @@ public ExprNodeDesc visitLiteral(RexLiteral literal) { case TIMESTAMP: return new ExprNodeConstantDesc(TypeInfoFactory.timestampTypeInfo, Timestamp.valueOf(literal.getValueAs(TimestampString.class).toString())); + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return new ExprNodeConstantDesc(TypeInfoFactory.timestampLocalTZTypeInfo, + literal.getValueAs(TimestampString.class).toString() + " UTC"); case BINARY: return new ExprNodeConstantDesc(TypeInfoFactory.binaryTypeInfo, literal.getValue3()); case DECIMAL: @@ -340,12 +339,7 @@ public ExprNodeDesc visitLiteral(RexLiteral literal) { return new ExprNodeConstantDesc(TypeInfoFactory.intervalDayTimeTypeInfo, new HiveIntervalDayTime(secsBd)); } - case NULL: - case OTHER: default: - if (lType instanceof HiveType && ((HiveType) lType).getTypeClass() == TimestampTZ.class) { - return new ExprNodeConstantDesc(TypeInfoFactory.timestampLocalTZTypeInfo, literal.getValue3()); - } return new ExprNodeConstantDesc(TypeInfoFactory.voidTypeInfo, literal.getValue3()); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveGBOpConvUtil.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveGBOpConvUtil.java index 0f6c5b5..6f4188c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveGBOpConvUtil.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveGBOpConvUtil.java @@ -29,6 +29,7 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.core.Aggregate.Group; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.util.ImmutableBitSet; import org.apache.hadoop.hive.conf.HiveConf; @@ -173,7 +174,7 @@ private static GBInfo getGBInfo(HiveAggregate aggRel, OpAttr inputOpAf, HiveConf } // 2. Collect Grouping Set info - if (aggRel.indicator) { + if (aggRel.getGroupType() != Group.SIMPLE) { // 2.1 Translate Grouping set col bitset ImmutableList lstGrpSet = aggRel.getGroupSets(); int bitmap = 0; diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java index 471675b..1885869 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java @@ -19,8 +19,6 @@ package org.apache.hadoop.hive.ql.optimizer.calcite.translator; -import org.apache.hadoop.hive.ql.parse.*; - import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -40,7 +38,6 @@ import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.SemiJoin; -import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; @@ -62,7 +59,6 @@ import org.apache.hadoop.hive.ql.io.AcidUtils.Operation; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; -import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; @@ -74,8 +70,19 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveUnion; +import org.apache.hadoop.hive.ql.parse.JoinCond; +import org.apache.hadoop.hive.ql.parse.JoinType; +import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec; import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderExpression; import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitionExpression; +import org.apache.hadoop.hive.ql.parse.PTFTranslator; +import org.apache.hadoop.hive.ql.parse.ParseUtils; +import org.apache.hadoop.hive.ql.parse.RowResolver; +import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.parse.UnparseTranslator; +import org.apache.hadoop.hive.ql.parse.WindowingComponentizer; +import org.apache.hadoop.hive.ql.parse.WindowingSpec; import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFunctionSpec; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java index f8a959d..e72fb30 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java @@ -58,6 +58,7 @@ import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.hive.common.type.TimestampTZ; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException; @@ -436,7 +437,7 @@ private RexNode handleExplicitCast(ExprNodeGenericFuncDesc func, List c // Calcite always needs the else clause to be defined explicitly if (newChildRexNodeLst.size() % 2 == 0) { newChildRexNodeLst.add(cluster.getRexBuilder().makeNullLiteral( - newChildRexNodeLst.get(newChildRexNodeLst.size()-1).getType().getSqlTypeName())); + newChildRexNodeLst.get(newChildRexNodeLst.size()-1).getType())); } return newChildRexNodeLst; } @@ -687,6 +688,18 @@ protected RexNode convert(ExprNodeConstantDesc literal) throws CalciteSemanticEx rexBuilder.getTypeFactory().getTypeSystem().getDefaultPrecision(SqlTypeName.TIMESTAMP)), false); break; + case TIMESTAMPLOCALTZ: + final TimestampString tsLocalTZString; + if (value == null) { + tsLocalTZString = null; + } else { + tsLocalTZString = TimestampString.fromMillisSinceEpoch( + ((TimestampTZ)value).getZonedDateTime().toInstant().toEpochMilli()); + } + calciteLiteral = rexBuilder.makeTimestampWithLocalTimeZoneLiteral( + tsLocalTZString, + rexBuilder.getTypeFactory().getTypeSystem().getDefaultPrecision(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE)); + break; case INTERVAL_YEAR_MONTH: // Calcite year-month literal value is months as BigDecimal BigDecimal totalMonths = BigDecimal.valueOf(((HiveIntervalYearMonth) value).getTotalMonths()); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java index 31a088b..51714cf 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java @@ -335,7 +335,7 @@ private static String getName(GenericUDF hiveUDF) { registerFunction("-", SqlStdOperatorTable.MINUS, hToken(HiveParser.MINUS, "-")); registerFunction("*", SqlStdOperatorTable.MULTIPLY, hToken(HiveParser.STAR, "*")); registerFunction("/", SqlStdOperatorTable.DIVIDE, hToken(HiveParser.DIVIDE, "/")); - registerFunction("%", SqlStdOperatorTable.MOD, hToken(HiveParser.Identifier, "%")); + registerFunction("%", SqlStdOperatorTable.MOD, hToken(HiveParser.MOD, "%")); registerFunction("and", SqlStdOperatorTable.AND, hToken(HiveParser.KW_AND, "and")); registerFunction("or", SqlStdOperatorTable.OR, hToken(HiveParser.KW_OR, "or")); registerFunction("=", SqlStdOperatorTable.EQUALS, hToken(HiveParser.EQUAL, "=")); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java index 34886f3..90e0377 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java @@ -36,12 +36,10 @@ import org.apache.calcite.util.ConversionUtil; import org.apache.hadoop.hive.common.type.HiveChar; import org.apache.hadoop.hive.common.type.HiveVarchar; -import org.apache.hadoop.hive.common.type.TimestampTZ; import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.RowSchema; import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException; import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature; -import org.apache.hadoop.hive.ql.optimizer.calcite.HiveType; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.SqlFunctionConverter.HiveToken; import org.apache.hadoop.hive.ql.parse.HiveParser; import org.apache.hadoop.hive.ql.parse.RowResolver; @@ -65,7 +63,6 @@ private static final Map calciteToHiveTypeNameMap; - // TODO: Handling of char[], varchar[], string... static { Builder b = ImmutableMap. builder(); b.put(SqlTypeName.BOOLEAN.getName(), new HiveToken(HiveParser.TOK_BOOLEAN, "TOK_BOOLEAN")); @@ -203,7 +200,7 @@ public static RelDataType convert(PrimitiveTypeInfo type, RelDataTypeFactory dtF convertedType = dtFactory.createSqlType(SqlTypeName.TIMESTAMP); break; case TIMESTAMPLOCALTZ: - convertedType = new HiveType(TimestampTZ.class); + convertedType = dtFactory.createSqlType(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE); break; case INTERVAL_YEAR_MONTH: convertedType = dtFactory.createSqlIntervalType( @@ -330,6 +327,8 @@ public static TypeInfo convertPrimitiveType(RelDataType rType) { return TypeInfoFactory.dateTypeInfo; case TIMESTAMP: return TypeInfoFactory.timestampTypeInfo; + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return TypeInfoFactory.timestampLocalTZTypeInfo; case INTERVAL_YEAR: case INTERVAL_MONTH: case INTERVAL_YEAR_MONTH: @@ -361,13 +360,7 @@ public static TypeInfo convertPrimitiveType(RelDataType rType) { return TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME); else return TypeInfoFactory.getCharTypeInfo(charLength); - case NULL: - case OTHER: default: - if (rType instanceof HiveType && ((HiveType) rType).getTypeClass() == TimestampTZ.class) { - // TODO: This block should be removed when we upgrade Calcite to use local time-zone - return TypeInfoFactory.timestampLocalTZTypeInfo; - } return TypeInfoFactory.voidTypeInfo; } @@ -396,12 +389,10 @@ public static HiveToken hiveToken(RelDataType calciteType) { .getPrecision()), String.valueOf(calciteType.getScale())); } break; - case NULL: - if (calciteType instanceof HiveType && ((HiveType) calciteType).getTypeClass() == TimestampTZ.class) { - ht = new HiveToken(HiveParser.TOK_TIMESTAMPLOCALTZ, "TOK_TIMESTAMPLOCALTZ"); - break; - } - // fall-through + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: { + ht = new HiveToken(HiveParser.TOK_TIMESTAMPLOCALTZ, "TOK_TIMESTAMPLOCALTZ"); + } + break; default: ht = calciteToHiveTypeNameMap.get(calciteType.getSqlTypeName().getName()); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 6555269..b311bb5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -231,8 +231,6 @@ import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitionExpression; import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitionSpec; import org.apache.hadoop.hive.ql.parse.QBExpr.Opcode; -import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.PlannerContext; -import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.PlannerContextFactory; import org.apache.hadoop.hive.ql.parse.WindowingSpec.BoundarySpec; import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowExpressionSpec; import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFunctionSpec; @@ -1357,6 +1355,9 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu HiveRulesRegistry registry = new HiveRulesRegistry(); Properties calciteConfigProperties = new Properties(); calciteConfigProperties.setProperty( + CalciteConnectionProperty.TIME_ZONE.camelName(), + conf.getLocalTimeZone().getId()); + calciteConfigProperties.setProperty( CalciteConnectionProperty.MATERIALIZATIONS_ENABLED.camelName(), Boolean.FALSE.toString()); CalciteConnectionConfig calciteConfig = new CalciteConnectionConfigImpl(calciteConfigProperties); @@ -1500,7 +1501,8 @@ public RelOptMaterialization apply(RelOptMaterialization materialization) { } else { newViewScan = copyNodeScan(viewScan); } - return new RelOptMaterialization(newViewScan, materialization.queryRel, null); + return new RelOptMaterialization(newViewScan, materialization.queryRel, null, + materialization.qualifiedTableName); } private RelNode copyNodeScan(RelNode scan) { @@ -2402,7 +2404,8 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc List intervals = Arrays.asList(DruidTable.DEFAULT_INTERVAL); DruidTable druidTable = new DruidTable(new DruidSchema(address, address, false), - dataSource, RelDataTypeImpl.proto(rowType), metrics, DruidTable.DEFAULT_TIMESTAMP_COLUMN, intervals); + dataSource, RelDataTypeImpl.proto(rowType), metrics, DruidTable.DEFAULT_TIMESTAMP_COLUMN, + intervals, null, null); final TableScan scan = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), optTable, null == tableAlias ? tabMetaData.getTableName() : tableAlias, getAliasId(tableAlias, qb), HiveConf.getBoolVar(conf, @@ -2821,8 +2824,8 @@ private RelNode genGBRelNode(List gbExprs, List aggInfoLs } if (hasGroupSets) { // Create GroupingID column - AggregateCall aggCall = new AggregateCall(HiveGroupingID.INSTANCE, - false, new ImmutableList.Builder().build(), + AggregateCall aggCall = AggregateCall.create(HiveGroupingID.INSTANCE, + false, new ImmutableList.Builder().build(), -1, this.cluster.getTypeFactory().createSqlType(SqlTypeName.INTEGER), HiveGroupingID.INSTANCE.getName()); aggregateCalls.add(aggCall); @@ -2836,8 +2839,7 @@ private RelNode genGBRelNode(List gbExprs, List aggInfoLs RelNode gbInputRel = HiveProject.create(srcRel, gbChildProjLst, null); HiveRelNode aggregateRel = new HiveAggregate(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), - gbInputRel, (transformedGroupSets!=null ? true:false), groupSet, - transformedGroupSets, aggregateCalls); + gbInputRel, groupSet, transformedGroupSets, aggregateCalls); return aggregateRel; } @@ -3095,19 +3097,6 @@ private RelNode genGBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException } else if (qbp.getDestGroupingSets().contains(detsClauseName)) { groupingSets = getGroupingSets(grpByAstExprs, qbp, detsClauseName); } - - final int limit = groupingColsSize * 2; - while (groupingColsSize < limit) { - String field = getColumnInternalName(groupingColsSize); - outputColumnNames.add(field); - groupByOutputRowResolver.put(null, field, - new ColumnInfo( - field, - TypeInfoFactory.booleanTypeInfo, - null, - false)); - groupingColsSize++; - } } // 6. Construct aggregation function Info @@ -4061,7 +4050,7 @@ public RexNode apply(RelDataTypeField input) { if (selForWindow != null && selExprList.getToken().getType() == HiveParser.TOK_SELECTDI) { ImmutableBitSet groupSet = ImmutableBitSet.range(outputRel.getRowType().getFieldList().size()); outputRel = new HiveAggregate(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), - outputRel, false, groupSet, null, new ArrayList()); + outputRel, groupSet, null, new ArrayList()); RowResolver groupByOutputRowResolver = new RowResolver(); for (int i = 0; i < out_rwsch.getColumnInfos().size(); i++) { ColumnInfo colInfo = out_rwsch.getColumnInfos().get(i); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 020a300..c2d8904 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -127,6 +127,7 @@ TOK_DATETIME; TOK_TIMESTAMP; TOK_TIMESTAMPLITERAL; TOK_TIMESTAMPLOCALTZ; +TOK_TIMESTAMPLOCALTZLITERAL; TOK_INTERVAL_YEAR_MONTH; TOK_INTERVAL_YEAR_MONTH_LITERAL; TOK_INTERVAL_DAY_TIME; diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g index 0591fbf..0a7b7b2 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g @@ -324,6 +324,7 @@ constant | Number | dateLiteral | timestampLiteral + | timestampLocalTZLiteral | StringLiteral | stringLiteralSequence | IntegralLiteral @@ -367,6 +368,14 @@ timestampLiteral KW_CURRENT_TIMESTAMP -> ^(TOK_FUNCTION KW_CURRENT_TIMESTAMP) ; +timestampLocalTZLiteral + : + KW_TIMESTAMPLOCALTZ StringLiteral -> + { + adaptor.create(TOK_TIMESTAMPLOCALTZLITERAL, $StringLiteral.text) + } + ; + intervalValue : StringLiteral | Number diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java index 391ee08..1203b35 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; +import org.apache.hadoop.hive.common.type.TimestampTZUtil; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.ColumnInfo; @@ -207,7 +208,8 @@ public static ExprNodeDesc processGByExpr(Node nd, Object procCtx) opRules.put(new RuleRegExp("R4", HiveParser.KW_TRUE + "%|" + HiveParser.KW_FALSE + "%"), tf.getBoolExprProcessor()); opRules.put(new RuleRegExp("R5", HiveParser.TOK_DATELITERAL + "%|" - + HiveParser.TOK_TIMESTAMPLITERAL + "%"), tf.getDateTimeExprProcessor()); + + HiveParser.TOK_TIMESTAMPLITERAL + "%|" + + HiveParser.TOK_TIMESTAMPLOCALTZLITERAL + "%"), tf.getDateTimeExprProcessor()); opRules.put(new RuleRegExp("R6", HiveParser.TOK_INTERVAL_YEAR_MONTH_LITERAL + "%|" + HiveParser.TOK_INTERVAL_DAY_TIME_LITERAL + "%|" + HiveParser.TOK_INTERVAL_YEAR_LITERAL + "%|" @@ -510,6 +512,16 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, return new ExprNodeConstantDesc(TypeInfoFactory.timestampTypeInfo, Timestamp.valueOf(timeString)); } + if (expr.getType() == HiveParser.TOK_TIMESTAMPLOCALTZLITERAL) { + HiveConf conf; + try { + conf = Hive.get().getConf(); + } catch (HiveException e) { + throw new SemanticException(e); + } + return new ExprNodeConstantDesc(TypeInfoFactory.timestampLocalTZTypeInfo, + TimestampTZUtil.parse(timeString, conf.getLocalTimeZone())); + } throw new IllegalArgumentException("Invalid time literal type " + expr.getType()); } catch (Exception err) { throw new SemanticException( diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateFloor.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateFloor.java index 126ed44..f623636 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateFloor.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateFloor.java @@ -17,11 +17,17 @@ */ package org.apache.hadoop.hive.ql.udf; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.util.Iterator; import java.util.Map; import java.util.NoSuchElementException; +import org.apache.hadoop.hive.common.type.TimestampTZ; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.TimestampLocalTZWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.joda.time.Chronology; import org.joda.time.DateTime; @@ -40,11 +46,13 @@ public abstract class UDFDateFloor extends UDF { private final QueryGranularity granularity; - private final TimestampWritable result; + private final TimestampWritable resultTS; + private final TimestampLocalTZWritable resultTSLTZ; public UDFDateFloor(String granularity) { this.granularity = QueryGranularity.fromString(granularity); - this.result = new TimestampWritable(); + this.resultTS = new TimestampWritable(); + this.resultTSLTZ = new TimestampLocalTZWritable(); } public TimestampWritable evaluate(TimestampWritable t) { @@ -57,8 +65,23 @@ public TimestampWritable evaluate(TimestampWritable t) { final long newTimestampUTC = granularity.truncate(originalTimestampUTC); // utc final long newTimestamp = new DateTime(newTimestampUTC, DateTimeZone.UTC) .withZoneRetainFields(DateTimeZone.getDefault()).getMillis(); // utc -> default - result.setTime(newTimestamp); - return result; + resultTS.setTime(newTimestamp); + return resultTS; + } + + public TimestampLocalTZWritable evaluate(TimestampLocalTZWritable t) { + if (t == null) { + return null; + } + final ZonedDateTime localZDT = t.getTimestampTZ().getZonedDateTime(); // default + final long originalTimestampUTC = localZDT.withZoneSameLocal(ZoneOffset.UTC) + .toInstant().toEpochMilli(); // default -> utc + final long newTimestampUTC = granularity.truncate(originalTimestampUTC); // utc + final ZonedDateTime newLocalZDT = ZonedDateTime.of( + LocalDateTime.ofInstant(Instant.ofEpochMilli(newTimestampUTC), ZoneOffset.UTC), + localZDT.getZone()); // utc -> default + resultTSLTZ.set(new TimestampTZ(newLocalZDT)); + return resultTSLTZ; } /* diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEpochMilli.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEpochMilli.java new file mode 100644 index 0000000..1fe8ad4 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEpochMilli.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.udf.generic; + +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampLocalTZObjectInspector; +import org.apache.hadoop.io.LongWritable; + +/** + * GenericUDFEpochMilli. + */ +public class GenericUDFEpochMilli extends GenericUDF { + + private transient final LongWritable result = new LongWritable(); + private transient TimestampLocalTZObjectInspector oi; + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + if (arguments.length != 1) { + throw new UDFArgumentLengthException( + "The operator GenericUDFEpochMilli only accepts 1 argument."); + } + oi = (TimestampLocalTZObjectInspector) arguments[0]; + return PrimitiveObjectInspectorFactory.writableLongObjectInspector; + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + Object a0 = arguments[0].get(); + if (a0 == null) { + return null; + } + result.set(oi.getPrimitiveJavaObject(a0).getZonedDateTime().toInstant().toEpochMilli()); + return result; + } + + @Override + public String getDisplayString(String[] children) { + assert (children.length == 1); + return "GenericUDFEpochMilli(" + children[0] + ")"; + } + +} diff --git ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFDateFormatGranularity.java ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFDateFormatGranularity.java index 03b2d42..ccccd32 100644 --- ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFDateFormatGranularity.java +++ ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFDateFormatGranularity.java @@ -18,7 +18,12 @@ package org.apache.hadoop.hive.ql.udf; import java.sql.Timestamp; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import org.apache.hadoop.hive.common.type.TimestampTZ; +import org.apache.hadoop.hive.serde2.io.TimestampLocalTZWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.junit.Test; @@ -84,4 +89,77 @@ public void testTimestampToTimestampWithGranularity() throws Exception { assertEquals(494243222000L, i8.getTimestamp().getTime()); } + @Test + public void testTimestampWithLocalTZGranularity() throws Exception { + // Running example + // Friday 30th August 1985 02:47:02 AM + final TimestampLocalTZWritable t = new TimestampLocalTZWritable( + new TimestampTZ(Instant.ofEpochMilli(494243222000L).atZone(ZoneId.of("America/Los_Angeles")))); + UDFDateFloor g; + + // Year granularity + // Tuesday 1st January 1985 12:00:00 AM + g = new UDFDateFloorYear(); + TimestampLocalTZWritable i1 = g.evaluate(t); + assertEquals( + new TimestampTZ(Instant.ofEpochMilli(473414400000L).atZone(ZoneId.of("America/Los_Angeles"))), + i1.getTimestampTZ()); + + // Quarter granularity + // Monday 1st July 1985 12:00:00 AM + g = new UDFDateFloorQuarter(); + TimestampLocalTZWritable i2 = g.evaluate(t); + assertEquals( + new TimestampTZ(Instant.ofEpochMilli(489049200000L).atZone(ZoneId.of("America/Los_Angeles"))), + i2.getTimestampTZ()); + + // Month granularity + // Thursday 1st August 1985 12:00:00 AM + g = new UDFDateFloorMonth(); + TimestampLocalTZWritable i3 = g.evaluate(t); + assertEquals( + new TimestampTZ(Instant.ofEpochMilli(491727600000L).atZone(ZoneId.of("America/Los_Angeles"))), + i3.getTimestampTZ()); + + // Week granularity + // Monday 26th August 1985 12:00:00 AM + g = new UDFDateFloorWeek(); + TimestampLocalTZWritable i4 = g.evaluate(t); + assertEquals( + new TimestampTZ(Instant.ofEpochMilli(493887600000L).atZone(ZoneId.of("America/Los_Angeles"))), + i4.getTimestampTZ()); + + // Day granularity + // Friday 30th August 1985 12:00:00 AM + g = new UDFDateFloorDay(); + TimestampLocalTZWritable i5 = g.evaluate(t); + assertEquals( + new TimestampTZ(Instant.ofEpochMilli(494233200000L).atZone(ZoneId.of("America/Los_Angeles"))), + i5.getTimestampTZ()); + + // Hour granularity + // Friday 30th August 1985 02:00:00 AM + g = new UDFDateFloorHour(); + TimestampLocalTZWritable i6 = g.evaluate(t); + assertEquals( + new TimestampTZ(Instant.ofEpochMilli(494240400000L).atZone(ZoneId.of("America/Los_Angeles"))), + i6.getTimestampTZ()); + + // Minute granularity + // Friday 30th August 1985 02:47:00 AM + g = new UDFDateFloorMinute(); + TimestampLocalTZWritable i7 = g.evaluate(t); + assertEquals( + new TimestampTZ(Instant.ofEpochMilli(494243220000L).atZone(ZoneId.of("America/Los_Angeles"))), + i7.getTimestampTZ()); + + // Second granularity + // Friday 30th August 1985 02:47:02 AM + g = new UDFDateFloorSecond(); + TimestampLocalTZWritable i8 = g.evaluate(t); + assertEquals( + new TimestampTZ(Instant.ofEpochMilli(494243222000L).atZone(ZoneId.of("America/Los_Angeles"))), + i8.getTimestampTZ()); + } + } diff --git ql/src/test/queries/clientpositive/druid_timeseries.q ql/src/test/queries/clientpositive/druid_timeseries.q index c0ad60f..f784f26 100644 --- ql/src/test/queries/clientpositive/druid_timeseries.q +++ ql/src/test/queries/clientpositive/druid_timeseries.q @@ -77,8 +77,8 @@ EXPLAIN SELECT floor_hour(`__time`), max(added), sum(variation) FROM druid_table_1 WHERE floor_hour(`__time`) - BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP) - AND CAST('2014-01-01 00:00:00' AS TIMESTAMP) + BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) + AND CAST('2014-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) GROUP BY floor_hour(`__time`); -- WITH FILTER ON TIME @@ -90,5 +90,5 @@ FROM FROM druid_table_1 GROUP BY floor_hour(`__time`) ) subq -WHERE subq.h BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP) - AND CAST('2014-01-01 00:00:00' AS TIMESTAMP); +WHERE subq.h BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) + AND CAST('2014-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE); diff --git ql/src/test/queries/clientpositive/druid_topn.q ql/src/test/queries/clientpositive/druid_topn.q index b121b7e..24d5051 100644 --- ql/src/test/queries/clientpositive/druid_topn.q +++ ql/src/test/queries/clientpositive/druid_topn.q @@ -68,8 +68,8 @@ EXPLAIN SELECT robot, floor_hour(`__time`), max(added) as m, sum(variation) FROM druid_table_1 WHERE floor_hour(`__time`) - BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP) - AND CAST('2014-01-01 00:00:00' AS TIMESTAMP) + BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) + AND CAST('2014-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) GROUP BY robot, floor_hour(`__time`) ORDER BY m LIMIT 100; diff --git ql/src/test/queries/clientpositive/timestamptz_3.q ql/src/test/queries/clientpositive/timestamptz_3.q new file mode 100644 index 0000000..7c55a50 --- /dev/null +++ ql/src/test/queries/clientpositive/timestamptz_3.q @@ -0,0 +1,15 @@ +set hive.fetch.task.conversion=more; + +drop table tstz1; + +create table tstz1(t timestamp with local time zone); + +insert overwrite table tstz1 select cast('2016-01-03 12:26:34 America/Los_Angeles' as timestamp with local time zone); + +select cast(t as timestamp) from tstz1; +select cast(to_epoch_milli(t) as timestamp) from tstz1; + +set time zone UTC; + +select cast(t as timestamp) from tstz1; +select cast(to_epoch_milli(t) as timestamp) from tstz1; diff --git ql/src/test/results/clientpositive/cbo_rp_annotate_stats_groupby.q.out ql/src/test/results/clientpositive/cbo_rp_annotate_stats_groupby.q.out index a603cc6..a4c9a6a 100644 --- ql/src/test/results/clientpositive/cbo_rp_annotate_stats_groupby.q.out +++ ql/src/test/results/clientpositive/cbo_rp_annotate_stats_groupby.q.out @@ -315,17 +315,13 @@ STAGE PLANS: mode: mergepartial outputColumnNames: state, locid Statistics: Num rows: 32 Data size: 2880 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: state (type: string), locid (type: int) - outputColumnNames: state, locid + File Output Operator + compressed: false Statistics: Num rows: 32 Data size: 2880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 32 Data size: 2880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -368,17 +364,13 @@ STAGE PLANS: mode: mergepartial outputColumnNames: state, locid Statistics: Num rows: 24 Data size: 2160 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: state (type: string), locid (type: int) - outputColumnNames: state, locid + File Output Operator + compressed: false Statistics: Num rows: 24 Data size: 2160 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 24 Data size: 2160 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -421,17 +413,13 @@ STAGE PLANS: mode: mergepartial outputColumnNames: state, locid Statistics: Num rows: 24 Data size: 2160 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: state (type: string), locid (type: int) - outputColumnNames: state, locid + File Output Operator + compressed: false Statistics: Num rows: 24 Data size: 2160 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 24 Data size: 2160 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -474,17 +462,13 @@ STAGE PLANS: mode: mergepartial outputColumnNames: state, locid Statistics: Num rows: 8 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: state (type: string), locid (type: int) - outputColumnNames: state, locid + File Output Operator + compressed: false Statistics: Num rows: 8 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 8 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -527,17 +511,13 @@ STAGE PLANS: mode: mergepartial outputColumnNames: state, locid Statistics: Num rows: 16 Data size: 1440 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: state (type: string), locid (type: int) - outputColumnNames: state, locid + File Output Operator + compressed: false Statistics: Num rows: 16 Data size: 1440 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 16 Data size: 1440 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -580,17 +560,13 @@ STAGE PLANS: mode: mergepartial outputColumnNames: state, locid Statistics: Num rows: 24 Data size: 2160 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: state (type: string), locid (type: int) - outputColumnNames: state, locid + File Output Operator + compressed: false Statistics: Num rows: 24 Data size: 2160 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 24 Data size: 2160 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -633,17 +609,13 @@ STAGE PLANS: mode: mergepartial outputColumnNames: state, locid Statistics: Num rows: 32 Data size: 2880 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: state (type: string), locid (type: int) - outputColumnNames: state, locid + File Output Operator + compressed: false Statistics: Num rows: 32 Data size: 2880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 32 Data size: 2880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -735,17 +707,13 @@ STAGE PLANS: mode: mergepartial outputColumnNames: state, locid Statistics: Num rows: 16 Data size: 1440 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: state (type: string), locid (type: int) - outputColumnNames: state, locid + File Output Operator + compressed: false Statistics: Num rows: 16 Data size: 1440 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 16 Data size: 1440 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -837,17 +805,13 @@ STAGE PLANS: mode: mergepartial outputColumnNames: state, locid Statistics: Num rows: 16 Data size: 1592 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: state (type: string), locid (type: int) - outputColumnNames: state, locid + File Output Operator + compressed: false Statistics: Num rows: 16 Data size: 1592 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 16 Data size: 1592 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -890,17 +854,13 @@ STAGE PLANS: mode: mergepartial outputColumnNames: state, locid Statistics: Num rows: 12 Data size: 1194 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: state (type: string), locid (type: int) - outputColumnNames: state, locid + File Output Operator + compressed: false Statistics: Num rows: 12 Data size: 1194 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1194 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -943,17 +903,13 @@ STAGE PLANS: mode: mergepartial outputColumnNames: state, locid Statistics: Num rows: 12 Data size: 1194 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: state (type: string), locid (type: int) - outputColumnNames: state, locid + File Output Operator + compressed: false Statistics: Num rows: 12 Data size: 1194 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1194 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -996,17 +952,13 @@ STAGE PLANS: mode: mergepartial outputColumnNames: state, locid Statistics: Num rows: 4 Data size: 398 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: state (type: string), locid (type: int) - outputColumnNames: state, locid + File Output Operator + compressed: false Statistics: Num rows: 4 Data size: 398 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 398 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1049,17 +1001,13 @@ STAGE PLANS: mode: mergepartial outputColumnNames: state, locid Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: state (type: string), locid (type: int) - outputColumnNames: state, locid + File Output Operator + compressed: false Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1102,17 +1050,13 @@ STAGE PLANS: mode: mergepartial outputColumnNames: state, locid Statistics: Num rows: 12 Data size: 1194 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: state (type: string), locid (type: int) - outputColumnNames: state, locid + File Output Operator + compressed: false Statistics: Num rows: 12 Data size: 1194 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1194 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1155,17 +1099,13 @@ STAGE PLANS: mode: mergepartial outputColumnNames: state, locid Statistics: Num rows: 16 Data size: 1592 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: state (type: string), locid (type: int) - outputColumnNames: state, locid + File Output Operator + compressed: false Statistics: Num rows: 16 Data size: 1592 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 16 Data size: 1592 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1257,17 +1197,13 @@ STAGE PLANS: mode: mergepartial outputColumnNames: state, locid Statistics: Num rows: 16 Data size: 1592 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: state (type: string), locid (type: int) - outputColumnNames: state, locid + File Output Operator + compressed: false Statistics: Num rows: 16 Data size: 1592 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 16 Data size: 1592 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/druid_basic1.q.out ql/src/test/results/clientpositive/druid_basic1.q.out index 91f79bb..25ec0ca 100644 --- ql/src/test/results/clientpositive/druid_basic1.q.out +++ ql/src/test/results/clientpositive/druid_basic1.q.out @@ -17,7 +17,7 @@ POSTHOOK: query: DESCRIBE FORMATTED druid_table_1 POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@druid_table_1 # col_name data_type comment -__time timestamp from deserializer +__time timestamp with local time zone from deserializer robot string from deserializer namespace string from deserializer anonymous string from deserializer @@ -78,7 +78,7 @@ POSTHOOK: query: DESCRIBE FORMATTED druid_table_2 POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@druid_table_2 # col_name data_type comment -__time timestamp from deserializer +__time timestamp with local time zone from deserializer robot string from deserializer namespace string from deserializer anonymous string from deserializer diff --git ql/src/test/results/clientpositive/druid_basic2.q.out ql/src/test/results/clientpositive/druid_basic2.q.out index 5d6862c..753ccb4 100644 --- ql/src/test/results/clientpositive/druid_basic2.q.out +++ ql/src/test/results/clientpositive/druid_basic2.q.out @@ -17,7 +17,7 @@ POSTHOOK: query: DESCRIBE FORMATTED druid_table_1 POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@druid_table_1 # col_name data_type comment -__time timestamp from deserializer +__time timestamp with local time zone from deserializer robot string from deserializer namespace string from deserializer anonymous string from deserializer @@ -271,7 +271,7 @@ STAGE PLANS: column.name.delimiter , columns __time,robot,namespace,anonymous,unpatrolled,page,language,newpage,user,count,added,delta,variation,deleted columns.comments 'from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer' - columns.types timestamp:string:string:string:string:string:string:string:string:float:float:float:float:float + columns.types timestamp with local time zone:string:string:string:string:string:string:string:string:float:float:float:float:float druid.datasource wikipedia druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"],"dimensions":["robot","language"],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}} druid.query.type select @@ -280,7 +280,7 @@ STAGE PLANS: numFiles 0 numRows 0 rawDataSize 0 - serialization.ddl struct druid_table_1 { timestamp __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted} + serialization.ddl struct druid_table_1 { timestamp with local time zone __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted} serialization.format 1 serialization.lib org.apache.hadoop.hive.druid.QTestDruidSerDe storage_handler org.apache.hadoop.hive.druid.QTestDruidStorageHandler @@ -297,7 +297,7 @@ STAGE PLANS: column.name.delimiter , columns __time,robot,namespace,anonymous,unpatrolled,page,language,newpage,user,count,added,delta,variation,deleted columns.comments 'from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer' - columns.types timestamp:string:string:string:string:string:string:string:string:float:float:float:float:float + columns.types timestamp with local time zone:string:string:string:string:string:string:string:string:float:float:float:float:float druid.datasource wikipedia druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"],"dimensions":["language"],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}} druid.query.type select @@ -306,7 +306,7 @@ STAGE PLANS: numFiles 0 numRows 0 rawDataSize 0 - serialization.ddl struct druid_table_1 { timestamp __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted} + serialization.ddl struct druid_table_1 { timestamp with local time zone __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted} serialization.format 1 serialization.lib org.apache.hadoop.hive.druid.QTestDruidSerDe storage_handler org.apache.hadoop.hive.druid.QTestDruidStorageHandler @@ -439,7 +439,7 @@ STAGE PLANS: column.name.delimiter , columns __time,robot,namespace,anonymous,unpatrolled,page,language,newpage,user,count,added,delta,variation,deleted columns.comments 'from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer' - columns.types timestamp:string:string:string:string:string:string:string:string:float:float:float:float:float + columns.types timestamp with local time zone:string:string:string:string:string:string:string:string:float:float:float:float:float druid.datasource wikipedia druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"],"filter":{"type":"selector","dimension":"language","value":"en"},"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}} druid.query.type select @@ -448,7 +448,7 @@ STAGE PLANS: numFiles 0 numRows 0 rawDataSize 0 - serialization.ddl struct druid_table_1 { timestamp __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted} + serialization.ddl struct druid_table_1 { timestamp with local time zone __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted} serialization.format 1 serialization.lib org.apache.hadoop.hive.druid.QTestDruidSerDe storage_handler org.apache.hadoop.hive.druid.QTestDruidStorageHandler @@ -465,7 +465,7 @@ STAGE PLANS: column.name.delimiter , columns __time,robot,namespace,anonymous,unpatrolled,page,language,newpage,user,count,added,delta,variation,deleted columns.comments 'from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer' - columns.types timestamp:string:string:string:string:string:string:string:string:float:float:float:float:float + columns.types timestamp with local time zone:string:string:string:string:string:string:string:string:float:float:float:float:float druid.datasource wikipedia druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"],"filter":{"type":"selector","dimension":"language","value":"en"},"dimensions":["robot"],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}} druid.query.type select @@ -474,7 +474,7 @@ STAGE PLANS: numFiles 0 numRows 0 rawDataSize 0 - serialization.ddl struct druid_table_1 { timestamp __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted} + serialization.ddl struct druid_table_1 { timestamp with local time zone __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted} serialization.format 1 serialization.lib org.apache.hadoop.hive.druid.QTestDruidSerDe storage_handler org.apache.hadoop.hive.druid.QTestDruidStorageHandler @@ -553,12 +553,12 @@ STAGE PLANS: TableScan alias: druid_table_1 properties: - druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"default","dimension":"language"},{"type":"extraction","dimension":"__time","outputName":"floor_day","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"day","timeZone":"UTC","locale":"en-US"}}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleMax","name":"$f3","fieldName":"added"},{"type":"doubleSum","name":"$f4","fieldName":"delta"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]} + druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"default","dimension":"language"},{"type":"extraction","dimension":"__time","outputName":"floor_day","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"day","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleMax","name":"$f3","fieldName":"added"},{"type":"doubleSum","name":"$f4","fieldName":"delta"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]} druid.query.type groupBy Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE GatherStats: false Select Operator - expressions: robot (type: string), floor_day (type: timestamp), $f3 (type: float), $f4 (type: float), UDFToInteger(robot) (type: int) + expressions: robot (type: string), floor_day (type: timestamp with local time zone), $f3 (type: float), $f4 (type: float), UDFToInteger(robot) (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col5 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator @@ -569,7 +569,7 @@ STAGE PLANS: tag: -1 TopN: 10 TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string), _col1 (type: timestamp), _col3 (type: float) + value expressions: _col0 (type: string), _col1 (type: timestamp with local time zone), _col3 (type: float) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -586,16 +586,16 @@ STAGE PLANS: column.name.delimiter , columns __time,robot,namespace,anonymous,unpatrolled,page,language,newpage,user,count,added,delta,variation,deleted columns.comments 'from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer' - columns.types timestamp:string:string:string:string:string:string:string:string:float:float:float:float:float + columns.types timestamp with local time zone:string:string:string:string:string:string:string:string:float:float:float:float:float druid.datasource wikipedia - druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"default","dimension":"language"},{"type":"extraction","dimension":"__time","outputName":"floor_day","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"day","timeZone":"UTC","locale":"en-US"}}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleMax","name":"$f3","fieldName":"added"},{"type":"doubleSum","name":"$f4","fieldName":"delta"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]} + druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"default","dimension":"language"},{"type":"extraction","dimension":"__time","outputName":"floor_day","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"day","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleMax","name":"$f3","fieldName":"added"},{"type":"doubleSum","name":"$f4","fieldName":"delta"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]} druid.query.type groupBy #### A masked pattern was here #### name default.druid_table_1 numFiles 0 numRows 0 rawDataSize 0 - serialization.ddl struct druid_table_1 { timestamp __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted} + serialization.ddl struct druid_table_1 { timestamp with local time zone __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted} serialization.format 1 serialization.lib org.apache.hadoop.hive.druid.QTestDruidSerDe storage_handler org.apache.hadoop.hive.druid.QTestDruidStorageHandler @@ -612,16 +612,16 @@ STAGE PLANS: column.name.delimiter , columns __time,robot,namespace,anonymous,unpatrolled,page,language,newpage,user,count,added,delta,variation,deleted columns.comments 'from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer' - columns.types timestamp:string:string:string:string:string:string:string:string:float:float:float:float:float + columns.types timestamp with local time zone:string:string:string:string:string:string:string:string:float:float:float:float:float druid.datasource wikipedia - druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"default","dimension":"language"},{"type":"extraction","dimension":"__time","outputName":"floor_day","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"day","timeZone":"UTC","locale":"en-US"}}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleMax","name":"$f3","fieldName":"added"},{"type":"doubleSum","name":"$f4","fieldName":"delta"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]} + druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"default","dimension":"language"},{"type":"extraction","dimension":"__time","outputName":"floor_day","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"day","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleMax","name":"$f3","fieldName":"added"},{"type":"doubleSum","name":"$f4","fieldName":"delta"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]} druid.query.type groupBy #### A masked pattern was here #### name default.druid_table_1 numFiles 0 numRows 0 rawDataSize 0 - serialization.ddl struct druid_table_1 { timestamp __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted} + serialization.ddl struct druid_table_1 { timestamp with local time zone __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted} serialization.format 1 serialization.lib org.apache.hadoop.hive.druid.QTestDruidSerDe storage_handler org.apache.hadoop.hive.druid.QTestDruidStorageHandler @@ -635,7 +635,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: timestamp), KEY.reducesinkkey1 (type: float), VALUE._col2 (type: float) + expressions: VALUE._col0 (type: string), VALUE._col1 (type: timestamp with local time zone), KEY.reducesinkkey1 (type: float), VALUE._col2 (type: float) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Limit @@ -653,7 +653,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: columns _col0,_col1,_col2,_col3 - columns.types string:timestamp:float:float + columns.types string:timestamp with local time zone:float:float escape.delim \ hive.serialization.extend.additional.nesting.levels true serialization.escape.crlf true @@ -737,32 +737,32 @@ STAGE PLANS: Map Operator Tree: TableScan alias: druid_table_1 - filterExpr: floor_day(__time) BETWEEN 1999-11-01 00:00:00.0 AND 1999-11-10 00:00:00.0 (type: boolean) + filterExpr: floor_day(__time) BETWEEN 1999-11-01 00:00:00.0 US/Pacific AND 1999-11-10 00:00:00.0 US/Pacific (type: boolean) properties: druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"],"dimensions":["robot"],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}} druid.query.type select Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: floor_day(__time) BETWEEN 1999-11-01 00:00:00.0 AND 1999-11-10 00:00:00.0 (type: boolean) + predicate: floor_day(__time) BETWEEN 1999-11-01 00:00:00.0 US/Pacific AND 1999-11-10 00:00:00.0 US/Pacific (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: robot (type: string), floor_day(__time) (type: timestamp) + expressions: robot (type: string), floor_day(__time) (type: timestamp with local time zone) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Group By Operator - keys: _col0 (type: string), _col1 (type: timestamp) + keys: _col0 (type: string), _col1 (type: timestamp with local time zone) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: timestamp) + key expressions: _col0 (type: string), _col1 (type: timestamp with local time zone) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: timestamp) + Map-reduce partition columns: _col0 (type: string), _col1 (type: timestamp with local time zone) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE TopN Hash Memory Usage: 0.1 Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: timestamp) + keys: KEY._col0 (type: string), KEY._col1 (type: timestamp with local time zone) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE @@ -782,10 +782,10 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE TopN Hash Memory Usage: 0.1 - value expressions: _col1 (type: timestamp) + value expressions: _col1 (type: timestamp with local time zone) Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: timestamp) + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: timestamp with local time zone) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Limit @@ -831,16 +831,16 @@ STAGE PLANS: Map Operator Tree: TableScan alias: druid_table_1 - filterExpr: floor_day(extract) BETWEEN 1999-11-01 00:00:00.0 AND 1999-11-10 00:00:00.0 (type: boolean) + filterExpr: floor_day(extract) BETWEEN 1999-11-01 00:00:00.0 US/Pacific AND 1999-11-10 00:00:00.0 US/Pacific (type: boolean) properties: - druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"UTC"}},{"type":"default","dimension":"robot"}],"limitSpec":{"type":"default"},"aggregations":[{"type":"longSum","name":"dummy_agg","fieldName":"dummy_agg"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]} + druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"US/Pacific"}},{"type":"default","dimension":"robot"}],"limitSpec":{"type":"default"},"aggregations":[{"type":"longSum","name":"dummy_agg","fieldName":"dummy_agg"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]} druid.query.type groupBy Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: floor_day(extract) BETWEEN 1999-11-01 00:00:00.0 AND 1999-11-10 00:00:00.0 (type: boolean) + predicate: floor_day(extract) BETWEEN 1999-11-01 00:00:00.0 US/Pacific AND 1999-11-10 00:00:00.0 US/Pacific (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: robot (type: string), extract (type: timestamp) + expressions: robot (type: string), extract (type: timestamp with local time zone) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator @@ -848,10 +848,10 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE TopN Hash Memory Usage: 0.1 - value expressions: _col1 (type: timestamp) + value expressions: _col1 (type: timestamp with local time zone) Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: timestamp) + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: timestamp with local time zone) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Limit @@ -898,11 +898,11 @@ STAGE PLANS: TableScan alias: druid_table_1 properties: - druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"extraction","dimension":"__time","outputName":"floor_day","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"day","timeZone":"UTC","locale":"en-US"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"robot","direction":"ascending","dimensionOrder":"alphanumeric"}]},"aggregations":[{"type":"longSum","name":"dummy_agg","fieldName":"dummy_agg"}],"intervals":["1999-11-01T00:00:00.000/1999-11-10T00:00:00.001"]} + druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"extraction","dimension":"__time","outputName":"floor_day","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"day","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"robot","direction":"ascending","dimensionOrder":"alphanumeric"}]},"aggregations":[{"type":"longSum","name":"dummy_agg","fieldName":"dummy_agg"}],"intervals":["1999-11-01T08:00:00.000/1999-11-10T08:00:00.001"]} druid.query.type groupBy Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: robot (type: string), floor_day (type: timestamp) + expressions: robot (type: string), floor_day (type: timestamp with local time zone) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE ListSink @@ -935,20 +935,20 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE GatherStats: false Select Operator - expressions: __time (type: timestamp), robot (type: string), language (type: string), added (type: float), delta (type: float) + expressions: __time (type: timestamp with local time zone), robot (type: string), language (type: string), added (type: float), delta (type: float) outputColumnNames: __time, robot, language, added, delta Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Group By Operator aggregations: max(added), sum(delta) - keys: robot (type: string), language (type: string), floor_day(__time) (type: timestamp) + keys: robot (type: string), language (type: string), floor_day(__time) (type: timestamp with local time zone) mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp) + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp with local time zone) null sort order: aaa sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp) + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp with local time zone) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE tag: -1 value expressions: _col3 (type: float), _col4 (type: double) @@ -968,14 +968,14 @@ STAGE PLANS: column.name.delimiter , columns __time,robot,namespace,anonymous,unpatrolled,page,language,newpage,user,count,added,delta,variation,deleted columns.comments 'from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer' - columns.types timestamp:string:string:string:string:string:string:string:string:float:float:float:float:float + columns.types timestamp with local time zone:string:string:string:string:string:string:string:string:float:float:float:float:float druid.datasource wikipedia #### A masked pattern was here #### name default.druid_table_1 numFiles 0 numRows 0 rawDataSize 0 - serialization.ddl struct druid_table_1 { timestamp __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted} + serialization.ddl struct druid_table_1 { timestamp with local time zone __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted} serialization.format 1 serialization.lib org.apache.hadoop.hive.druid.QTestDruidSerDe storage_handler org.apache.hadoop.hive.druid.QTestDruidStorageHandler @@ -992,14 +992,14 @@ STAGE PLANS: column.name.delimiter , columns __time,robot,namespace,anonymous,unpatrolled,page,language,newpage,user,count,added,delta,variation,deleted columns.comments 'from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer' - columns.types timestamp:string:string:string:string:string:string:string:string:float:float:float:float:float + columns.types timestamp with local time zone:string:string:string:string:string:string:string:string:float:float:float:float:float druid.datasource wikipedia #### A masked pattern was here #### name default.druid_table_1 numFiles 0 numRows 0 rawDataSize 0 - serialization.ddl struct druid_table_1 { timestamp __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted} + serialization.ddl struct druid_table_1 { timestamp with local time zone __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted} serialization.format 1 serialization.lib org.apache.hadoop.hive.druid.QTestDruidSerDe storage_handler org.apache.hadoop.hive.druid.QTestDruidStorageHandler @@ -1014,12 +1014,12 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), sum(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: timestamp) + keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: timestamp with local time zone) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col0 (type: string), _col2 (type: timestamp), _col3 (type: float), _col4 (type: double) + expressions: _col0 (type: string), _col2 (type: timestamp with local time zone), _col3 (type: float), _col4 (type: double) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE File Output Operator @@ -1033,7 +1033,7 @@ STAGE PLANS: properties: column.name.delimiter , columns _col0,_col1,_col2,_col3 - columns.types string,timestamp,float,double + columns.types string,timestamp with local time zone,float,double escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -1054,7 +1054,7 @@ STAGE PLANS: tag: -1 TopN: 10 TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string), _col1 (type: timestamp), _col3 (type: double) + value expressions: _col0 (type: string), _col1 (type: timestamp with local time zone), _col3 (type: double) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -1067,7 +1067,7 @@ STAGE PLANS: properties: column.name.delimiter , columns _col0,_col1,_col2,_col3 - columns.types string,timestamp,float,double + columns.types string,timestamp with local time zone,float,double escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -1077,7 +1077,7 @@ STAGE PLANS: properties: column.name.delimiter , columns _col0,_col1,_col2,_col3 - columns.types string,timestamp,float,double + columns.types string,timestamp with local time zone,float,double escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -1086,7 +1086,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: timestamp), KEY.reducesinkkey1 (type: float), VALUE._col2 (type: double) + expressions: VALUE._col0 (type: string), VALUE._col1 (type: timestamp with local time zone), KEY.reducesinkkey1 (type: float), VALUE._col2 (type: double) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Limit @@ -1104,7 +1104,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: columns _col0,_col1,_col2,_col3 - columns.types string:timestamp:float:double + columns.types string:timestamp with local time zone:float:double escape.delim \ hive.serialization.extend.additional.nesting.levels true serialization.escape.crlf true diff --git ql/src/test/results/clientpositive/druid_intervals.q.out ql/src/test/results/clientpositive/druid_intervals.q.out index 63f28d5..3cd28b5 100644 --- ql/src/test/results/clientpositive/druid_intervals.q.out +++ ql/src/test/results/clientpositive/druid_intervals.q.out @@ -17,7 +17,7 @@ POSTHOOK: query: DESCRIBE FORMATTED druid_table_1 POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@druid_table_1 # col_name data_type comment -__time timestamp from deserializer +__time timestamp with local time zone from deserializer robot string from deserializer namespace string from deserializer anonymous string from deserializer @@ -82,7 +82,7 @@ STAGE PLANS: druid.query.type select Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: __time (type: timestamp) + expressions: __time (type: timestamp with local time zone) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE ListSink @@ -108,11 +108,11 @@ STAGE PLANS: TableScan alias: druid_table_1 properties: - druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["1900-01-01T00:00:00.000/2012-03-01T00:00:00.000"],"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}} + druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["1900-01-01T00:00:00.000/2012-03-01T08:00:00.000"],"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}} druid.query.type select Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: __time (type: timestamp) + expressions: __time (type: timestamp with local time zone) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE ListSink @@ -138,11 +138,11 @@ STAGE PLANS: TableScan alias: druid_table_1 properties: - druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["2010-01-01T00:00:00.000/2012-03-01T00:00:00.001"],"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}} + druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["2010-01-01T08:00:00.000/2012-03-01T08:00:00.001"],"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}} druid.query.type select Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: __time (type: timestamp) + expressions: __time (type: timestamp with local time zone) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE ListSink @@ -170,11 +170,11 @@ STAGE PLANS: TableScan alias: druid_table_1 properties: - druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["2010-01-01T00:00:00.000/2011-01-01T00:00:00.000"],"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}} + druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["2010-01-01T08:00:00.000/2011-01-01T08:00:00.000"],"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}} druid.query.type select Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: __time (type: timestamp) + expressions: __time (type: timestamp with local time zone) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE ListSink @@ -200,11 +200,11 @@ STAGE PLANS: TableScan alias: druid_table_1 properties: - druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["2010-01-01T00:00:00.000/2011-01-01T00:00:00.001"],"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}} + druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["2010-01-01T08:00:00.000/2011-01-01T08:00:00.001"],"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}} druid.query.type select Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: __time (type: timestamp) + expressions: __time (type: timestamp with local time zone) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE ListSink @@ -232,11 +232,11 @@ STAGE PLANS: TableScan alias: druid_table_1 properties: - druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["2010-01-01T00:00:00.000/2011-01-01T00:00:00.001","2012-01-01T00:00:00.000/2013-01-01T00:00:00.001"],"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}} + druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["2010-01-01T08:00:00.000/2011-01-01T08:00:00.001","2012-01-01T08:00:00.000/2013-01-01T08:00:00.001"],"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}} druid.query.type select Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: __time (type: timestamp) + expressions: __time (type: timestamp with local time zone) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE ListSink @@ -264,11 +264,11 @@ STAGE PLANS: TableScan alias: druid_table_1 properties: - druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["2010-01-01T00:00:00.000/2012-01-01T00:00:00.001"],"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}} + druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["2010-01-01T08:00:00.000/2012-01-01T08:00:00.001"],"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}} druid.query.type select Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: __time (type: timestamp) + expressions: __time (type: timestamp with local time zone) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE ListSink @@ -294,11 +294,11 @@ STAGE PLANS: TableScan alias: druid_table_1 properties: - druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["2010-01-01T00:00:00.000/2010-01-01T00:00:00.001","2011-01-01T00:00:00.000/2011-01-01T00:00:00.001"],"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}} + druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["2010-01-01T08:00:00.000/2010-01-01T08:00:00.001","2011-01-01T08:00:00.000/2011-01-01T08:00:00.001"],"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}} druid.query.type select Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: __time (type: timestamp) + expressions: __time (type: timestamp with local time zone) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE ListSink @@ -324,11 +324,11 @@ STAGE PLANS: TableScan alias: druid_table_1 properties: - druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["2010-01-01T00:00:00.000/2010-01-01T00:00:00.001","2011-01-01T00:00:00.000/2011-01-01T00:00:00.001"],"filter":{"type":"selector","dimension":"robot","value":"user1"},"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}} + druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["2010-01-01T08:00:00.000/2010-01-01T08:00:00.001","2011-01-01T08:00:00.000/2011-01-01T08:00:00.001"],"filter":{"type":"selector","dimension":"robot","value":"user1"},"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}} druid.query.type select Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: __time (type: timestamp), 'user1' (type: string) + expressions: __time (type: timestamp with local time zone), 'user1' (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE ListSink @@ -353,16 +353,16 @@ STAGE PLANS: Map Operator Tree: TableScan alias: druid_table_1 - filterExpr: ((__time) IN (2010-01-01 00:00:00.0, 2011-01-01 00:00:00.0) or (robot = 'user1')) (type: boolean) + filterExpr: ((__time) IN (2010-01-01 00:00:00.0 US/Pacific, 2011-01-01 00:00:00.0 US/Pacific) or (robot = 'user1')) (type: boolean) properties: druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"],"dimensions":["robot"],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}} druid.query.type select Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((__time) IN (2010-01-01 00:00:00.0, 2011-01-01 00:00:00.0) or (robot = 'user1')) (type: boolean) + predicate: ((__time) IN (2010-01-01 00:00:00.0 US/Pacific, 2011-01-01 00:00:00.0 US/Pacific) or (robot = 'user1')) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: __time (type: timestamp), robot (type: string) + expressions: __time (type: timestamp with local time zone), robot (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/druid_timeseries.q.out ql/src/test/results/clientpositive/druid_timeseries.q.out index 04db1e0..330c068 100644 --- ql/src/test/results/clientpositive/druid_timeseries.q.out +++ ql/src/test/results/clientpositive/druid_timeseries.q.out @@ -17,7 +17,7 @@ POSTHOOK: query: DESCRIBE FORMATTED druid_table_1 POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@druid_table_1 # col_name data_type comment -__time timestamp from deserializer +__time timestamp with local time zone from deserializer robot string from deserializer namespace string from deserializer anonymous string from deserializer @@ -108,11 +108,11 @@ STAGE PLANS: TableScan alias: druid_table_1 properties: - druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"UTC"}}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleMax","name":"$f1","fieldName":"added"},{"type":"doubleSum","name":"$f2","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]} + druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"US/Pacific"}}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleMax","name":"$f1","fieldName":"added"},{"type":"doubleSum","name":"$f2","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]} druid.query.type groupBy Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: extract (type: timestamp), $f1 (type: float), $f2 (type: float) + expressions: extract (type: timestamp with local time zone), $f1 (type: float), $f2 (type: float) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE ListSink @@ -142,7 +142,7 @@ STAGE PLANS: druid.query.type timeseries Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: __time (type: timestamp), $f1 (type: float), $f2 (type: float) + expressions: __time (type: timestamp with local time zone), $f1 (type: float), $f2 (type: float) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE ListSink @@ -172,7 +172,7 @@ STAGE PLANS: druid.query.type timeseries Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: __time (type: timestamp), $f1 (type: float), $f2 (type: float) + expressions: __time (type: timestamp with local time zone), $f1 (type: float), $f2 (type: float) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE ListSink @@ -202,7 +202,7 @@ STAGE PLANS: druid.query.type timeseries Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: __time (type: timestamp), $f1 (type: float), $f2 (type: float) + expressions: __time (type: timestamp with local time zone), $f1 (type: float), $f2 (type: float) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE ListSink @@ -232,7 +232,7 @@ STAGE PLANS: druid.query.type timeseries Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: __time (type: timestamp), $f1 (type: float), $f2 (type: float) + expressions: __time (type: timestamp with local time zone), $f1 (type: float), $f2 (type: float) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE ListSink @@ -262,7 +262,7 @@ STAGE PLANS: druid.query.type timeseries Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: __time (type: timestamp), $f1 (type: float), $f2 (type: float) + expressions: __time (type: timestamp with local time zone), $f1 (type: float), $f2 (type: float) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE ListSink @@ -292,7 +292,7 @@ STAGE PLANS: druid.query.type timeseries Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: __time (type: timestamp), $f1 (type: float), $f2 (type: float) + expressions: __time (type: timestamp with local time zone), $f1 (type: float), $f2 (type: float) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE ListSink @@ -322,7 +322,7 @@ STAGE PLANS: druid.query.type timeseries Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: __time (type: timestamp), $f1 (type: float), $f2 (type: float) + expressions: __time (type: timestamp with local time zone), $f1 (type: float), $f2 (type: float) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE ListSink @@ -352,7 +352,7 @@ STAGE PLANS: druid.query.type timeseries Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: __time (type: timestamp), $f1 (type: float), $f2 (type: float) + expressions: __time (type: timestamp with local time zone), $f1 (type: float), $f2 (type: float) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE ListSink @@ -384,7 +384,7 @@ STAGE PLANS: druid.query.type timeseries Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: __time (type: timestamp), $f1 (type: float), $f2 (type: float) + expressions: __time (type: timestamp with local time zone), $f1 (type: float), $f2 (type: float) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE ListSink @@ -393,16 +393,16 @@ PREHOOK: query: EXPLAIN SELECT floor_hour(`__time`), max(added), sum(variation) FROM druid_table_1 WHERE floor_hour(`__time`) - BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP) - AND CAST('2014-01-01 00:00:00' AS TIMESTAMP) + BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) + AND CAST('2014-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) GROUP BY floor_hour(`__time`) PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN SELECT floor_hour(`__time`), max(added), sum(variation) FROM druid_table_1 WHERE floor_hour(`__time`) - BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP) - AND CAST('2014-01-01 00:00:00' AS TIMESTAMP) + BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) + AND CAST('2014-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) GROUP BY floor_hour(`__time`) POSTHOOK: type: QUERY STAGE DEPENDENCIES: @@ -415,34 +415,34 @@ STAGE PLANS: Map Operator Tree: TableScan alias: druid_table_1 - filterExpr: floor_hour(__time) BETWEEN 2010-01-01 00:00:00.0 AND 2014-01-01 00:00:00.0 (type: boolean) + filterExpr: floor_hour(__time) BETWEEN 2010-01-01 00:00:00.0 US/Pacific AND 2014-01-01 00:00:00.0 US/Pacific (type: boolean) properties: druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"],"dimensions":[],"metrics":["added","variation"],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}} druid.query.type select Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: floor_hour(__time) BETWEEN 2010-01-01 00:00:00.0 AND 2014-01-01 00:00:00.0 (type: boolean) + predicate: floor_hour(__time) BETWEEN 2010-01-01 00:00:00.0 US/Pacific AND 2014-01-01 00:00:00.0 US/Pacific (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: floor_hour(__time) (type: timestamp), added (type: float), variation (type: float) + expressions: floor_hour(__time) (type: timestamp with local time zone), added (type: float), variation (type: float) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Group By Operator aggregations: max(_col1), sum(_col2) - keys: _col0 (type: timestamp) + keys: _col0 (type: timestamp with local time zone) mode: hash outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: timestamp) + key expressions: _col0 (type: timestamp with local time zone) sort order: + - Map-reduce partition columns: _col0 (type: timestamp) + Map-reduce partition columns: _col0 (type: timestamp with local time zone) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE value expressions: _col1 (type: float), _col2 (type: double) Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), sum(VALUE._col1) - keys: KEY._col0 (type: timestamp) + keys: KEY._col0 (type: timestamp with local time zone) mode: mergepartial outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE @@ -468,8 +468,8 @@ FROM FROM druid_table_1 GROUP BY floor_hour(`__time`) ) subq -WHERE subq.h BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP) - AND CAST('2014-01-01 00:00:00' AS TIMESTAMP) +WHERE subq.h BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) + AND CAST('2014-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN SELECT subq.h, subq.m, subq.s @@ -479,8 +479,8 @@ FROM FROM druid_table_1 GROUP BY floor_hour(`__time`) ) subq -WHERE subq.h BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP) - AND CAST('2014-01-01 00:00:00' AS TIMESTAMP) +WHERE subq.h BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) + AND CAST('2014-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -492,34 +492,34 @@ STAGE PLANS: Map Operator Tree: TableScan alias: druid_table_1 - filterExpr: floor_hour(__time) BETWEEN 2010-01-01 00:00:00.0 AND 2014-01-01 00:00:00.0 (type: boolean) + filterExpr: floor_hour(__time) BETWEEN 2010-01-01 00:00:00.0 US/Pacific AND 2014-01-01 00:00:00.0 US/Pacific (type: boolean) properties: druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"],"dimensions":[],"metrics":["added","variation"],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}} druid.query.type select Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: floor_hour(__time) BETWEEN 2010-01-01 00:00:00.0 AND 2014-01-01 00:00:00.0 (type: boolean) + predicate: floor_hour(__time) BETWEEN 2010-01-01 00:00:00.0 US/Pacific AND 2014-01-01 00:00:00.0 US/Pacific (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: floor_hour(__time) (type: timestamp), added (type: float), variation (type: float) + expressions: floor_hour(__time) (type: timestamp with local time zone), added (type: float), variation (type: float) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Group By Operator aggregations: max(_col1), sum(_col2) - keys: _col0 (type: timestamp) + keys: _col0 (type: timestamp with local time zone) mode: hash outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: timestamp) + key expressions: _col0 (type: timestamp with local time zone) sort order: + - Map-reduce partition columns: _col0 (type: timestamp) + Map-reduce partition columns: _col0 (type: timestamp with local time zone) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE value expressions: _col1 (type: float), _col2 (type: double) Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), sum(VALUE._col1) - keys: KEY._col0 (type: timestamp) + keys: KEY._col0 (type: timestamp with local time zone) mode: mergepartial outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE diff --git ql/src/test/results/clientpositive/druid_topn.q.out ql/src/test/results/clientpositive/druid_topn.q.out index 5c31f85..3e2b477 100644 --- ql/src/test/results/clientpositive/druid_topn.q.out +++ ql/src/test/results/clientpositive/druid_topn.q.out @@ -17,7 +17,7 @@ POSTHOOK: query: DESCRIBE FORMATTED druid_table_1 POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@druid_table_1 # col_name data_type comment -__time timestamp from deserializer +__time timestamp with local time zone from deserializer robot string from deserializer namespace string from deserializer anonymous string from deserializer @@ -118,11 +118,11 @@ STAGE PLANS: TableScan alias: druid_table_1 properties: - druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"UTC"}},{"type":"default","dimension":"robot"}],"limitSpec":{"type":"default","limit":100,"columns":[{"dimension":"$f3","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f2","fieldName":"added"},{"type":"doubleSum","name":"$f3","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]} + druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"US/Pacific"}},{"type":"default","dimension":"robot"}],"limitSpec":{"type":"default","limit":100,"columns":[{"dimension":"$f3","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f2","fieldName":"added"},{"type":"doubleSum","name":"$f3","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]} druid.query.type groupBy Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: robot (type: string), extract (type: timestamp), $f2 (type: float), $f3 (type: float) + expressions: robot (type: string), extract (type: timestamp with local time zone), $f2 (type: float), $f3 (type: float) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE ListSink @@ -152,11 +152,11 @@ STAGE PLANS: TableScan alias: druid_table_1 properties: - druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"extraction","dimension":"__time","outputName":"floor_year","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"year","timeZone":"UTC","locale":"en-US"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"$f3","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f2","fieldName":"added"},{"type":"doubleSum","name":"$f3","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]} + druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"extraction","dimension":"__time","outputName":"floor_year","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"year","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"$f3","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f2","fieldName":"added"},{"type":"doubleSum","name":"$f3","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]} druid.query.type groupBy Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: robot (type: string), floor_year (type: timestamp), $f2 (type: float), $f3 (type: float) + expressions: robot (type: string), floor_year (type: timestamp with local time zone), $f2 (type: float), $f3 (type: float) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE ListSink @@ -186,11 +186,11 @@ STAGE PLANS: TableScan alias: druid_table_1 properties: - druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"extraction","dimension":"__time","outputName":"floor_month","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"month","timeZone":"UTC","locale":"en-US"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"$f3","direction":"ascending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f2","fieldName":"added"},{"type":"doubleSum","name":"$f3","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]} + druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"extraction","dimension":"__time","outputName":"floor_month","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"month","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"$f3","direction":"ascending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f2","fieldName":"added"},{"type":"doubleSum","name":"$f3","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]} druid.query.type groupBy Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: robot (type: string), floor_month (type: timestamp), $f2 (type: float), $f3 (type: float) + expressions: robot (type: string), floor_month (type: timestamp with local time zone), $f2 (type: float), $f3 (type: float) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE ListSink @@ -220,11 +220,11 @@ STAGE PLANS: TableScan alias: druid_table_1 properties: - druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"default","dimension":"namespace"},{"type":"extraction","dimension":"__time","outputName":"floor_month","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"month","timeZone":"UTC","locale":"en-US"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"$f4","direction":"descending","dimensionOrder":"numeric"},{"dimension":"$f3","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f3","fieldName":"added"},{"type":"doubleSum","name":"$f4","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]} + druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"default","dimension":"namespace"},{"type":"extraction","dimension":"__time","outputName":"floor_month","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"month","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"$f4","direction":"descending","dimensionOrder":"numeric"},{"dimension":"$f3","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f3","fieldName":"added"},{"type":"doubleSum","name":"$f4","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]} druid.query.type groupBy Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: robot (type: string), floor_month (type: timestamp), $f3 (type: float), $f4 (type: float) + expressions: robot (type: string), floor_month (type: timestamp with local time zone), $f3 (type: float), $f4 (type: float) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE ListSink @@ -254,11 +254,11 @@ STAGE PLANS: TableScan alias: druid_table_1 properties: - druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"default","dimension":"namespace"},{"type":"extraction","dimension":"__time","outputName":"floor_month","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"month","timeZone":"UTC","locale":"en-US"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"robot","direction":"ascending","dimensionOrder":"alphanumeric"},{"dimension":"$f3","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f3","fieldName":"added"},{"type":"doubleSum","name":"$f4","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]} + druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"default","dimension":"namespace"},{"type":"extraction","dimension":"__time","outputName":"floor_month","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"month","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"robot","direction":"ascending","dimensionOrder":"alphanumeric"},{"dimension":"$f3","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f3","fieldName":"added"},{"type":"doubleSum","name":"$f4","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]} druid.query.type groupBy Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: robot (type: string), floor_month (type: timestamp), $f3 (type: float), $f4 (type: float) + expressions: robot (type: string), floor_month (type: timestamp with local time zone), $f3 (type: float), $f4 (type: float) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE ListSink @@ -294,7 +294,7 @@ STAGE PLANS: druid.query.type timeseries Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: __time (type: timestamp), $f1_0 (type: float), $f2 (type: float) + expressions: __time (type: timestamp with local time zone), $f1_0 (type: float), $f2 (type: float) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator @@ -302,17 +302,17 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: timestamp), _col1 (type: float) + value expressions: _col0 (type: timestamp with local time zone), _col1 (type: float) Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: timestamp), VALUE._col1 (type: float), KEY.reducesinkkey0 (type: float) + expressions: VALUE._col0 (type: timestamp with local time zone), VALUE._col1 (type: float), KEY.reducesinkkey0 (type: float) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Limit Number of rows: 10 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: '1' (type: string), _col0 (type: timestamp), _col1 (type: float), _col2 (type: float) + expressions: '1' (type: string), _col0 (type: timestamp with local time zone), _col1 (type: float), _col2 (type: float) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE File Output Operator @@ -333,8 +333,8 @@ PREHOOK: query: EXPLAIN SELECT robot, floor_hour(`__time`), max(added) as m, sum(variation) FROM druid_table_1 WHERE floor_hour(`__time`) - BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP) - AND CAST('2014-01-01 00:00:00' AS TIMESTAMP) + BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) + AND CAST('2014-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) GROUP BY robot, floor_hour(`__time`) ORDER BY m LIMIT 100 @@ -343,8 +343,8 @@ POSTHOOK: query: EXPLAIN SELECT robot, floor_hour(`__time`), max(added) as m, sum(variation) FROM druid_table_1 WHERE floor_hour(`__time`) - BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP) - AND CAST('2014-01-01 00:00:00' AS TIMESTAMP) + BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) + AND CAST('2014-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) GROUP BY robot, floor_hour(`__time`) ORDER BY m LIMIT 100 @@ -360,34 +360,34 @@ STAGE PLANS: Map Operator Tree: TableScan alias: druid_table_1 - filterExpr: floor_hour(__time) BETWEEN 2010-01-01 00:00:00.0 AND 2014-01-01 00:00:00.0 (type: boolean) + filterExpr: floor_hour(__time) BETWEEN 2010-01-01 00:00:00.0 US/Pacific AND 2014-01-01 00:00:00.0 US/Pacific (type: boolean) properties: druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"],"dimensions":["robot"],"metrics":["added","variation"],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}} druid.query.type select Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: floor_hour(__time) BETWEEN 2010-01-01 00:00:00.0 AND 2014-01-01 00:00:00.0 (type: boolean) + predicate: floor_hour(__time) BETWEEN 2010-01-01 00:00:00.0 US/Pacific AND 2014-01-01 00:00:00.0 US/Pacific (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: robot (type: string), floor_hour(__time) (type: timestamp), added (type: float), variation (type: float) + expressions: robot (type: string), floor_hour(__time) (type: timestamp with local time zone), added (type: float), variation (type: float) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Group By Operator aggregations: max(_col2), sum(_col3) - keys: _col0 (type: string), _col1 (type: timestamp) + keys: _col0 (type: string), _col1 (type: timestamp with local time zone) mode: hash outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: timestamp) + key expressions: _col0 (type: string), _col1 (type: timestamp with local time zone) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: timestamp) + Map-reduce partition columns: _col0 (type: string), _col1 (type: timestamp with local time zone) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE value expressions: _col2 (type: float), _col3 (type: double) Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), sum(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: timestamp) + keys: KEY._col0 (type: string), KEY._col1 (type: timestamp with local time zone) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE @@ -407,10 +407,10 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string), _col1 (type: timestamp), _col3 (type: double) + value expressions: _col0 (type: string), _col1 (type: timestamp with local time zone), _col3 (type: double) Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: timestamp), KEY.reducesinkkey0 (type: float), VALUE._col2 (type: double) + expressions: VALUE._col0 (type: string), VALUE._col1 (type: timestamp with local time zone), KEY.reducesinkkey0 (type: float), VALUE._col2 (type: double) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Limit diff --git ql/src/test/results/clientpositive/groupby_grouping_sets1.q.out ql/src/test/results/clientpositive/groupby_grouping_sets1.q.out index 13eac20..fcf203e 100644 --- ql/src/test/results/clientpositive/groupby_grouping_sets1.q.out +++ ql/src/test/results/clientpositive/groupby_grouping_sets1.q.out @@ -464,22 +464,21 @@ STAGE PLANS: outputColumnNames: a Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: a (type: string), 0 (type: int) + keys: a (type: string) mode: hash - outputColumnNames: _col0, _col1 + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int) + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: int) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE - pruneGroupingSetId: true File Output Operator compressed: false Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE @@ -532,35 +531,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() - keys: _col0 (type: double), 0 (type: int) + keys: _col0 (type: double) mode: hash - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: double), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: double), _col1 (type: int) + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: double), KEY._col1 (type: int) + keys: KEY._col0 (type: double) mode: mergepartial - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE - pruneGroupingSetId: true - Select Operator - expressions: _col0 (type: double), _col2 (type: bigint) - outputColumnNames: _col0, _col1 + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/groupby_grouping_sets_limit.q.out ql/src/test/results/clientpositive/groupby_grouping_sets_limit.q.out index 02636eb..1c45853 100644 --- ql/src/test/results/clientpositive/groupby_grouping_sets_limit.q.out +++ ql/src/test/results/clientpositive/groupby_grouping_sets_limit.q.out @@ -357,23 +357,22 @@ STAGE PLANS: outputColumnNames: a Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: a (type: string), 0 (type: int) + keys: a (type: string) mode: hash - outputColumnNames: _col0, _col1 + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int) + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: int) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE - pruneGroupingSetId: true Limit Number of rows: 10 Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE @@ -427,39 +426,34 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() - keys: _col0 (type: double), 0 (type: int) + keys: _col0 (type: double) mode: hash - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: double), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: double), _col1 (type: int) + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 - value expressions: _col2 (type: bigint) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: double), KEY._col1 (type: int) + keys: KEY._col0 (type: double) mode: mergepartial - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE - pruneGroupingSetId: true - Select Operator - expressions: _col0 (type: double), _col2 (type: bigint) - outputColumnNames: _col0, _col1 + Limit + Number of rows: 10 Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/vector_groupby_grouping_id2.q.out ql/src/test/results/clientpositive/llap/vector_groupby_grouping_id2.q.out index f514448..43e5958 100644 --- ql/src/test/results/clientpositive/llap/vector_groupby_grouping_id2.q.out +++ ql/src/test/results/clientpositive/llap/vector_groupby_grouping_id2.q.out @@ -600,7 +600,7 @@ STAGE PLANS: Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: int) - outputColumnNames: _col4 + outputColumnNames: _col2 Select Vectorization: className: VectorSelectOperator native: true @@ -617,7 +617,7 @@ STAGE PLANS: native: false vectorProcessingMode: HASH projectedOutputColumns: [0] - keys: _col4 (type: int) + keys: _col2 (type: int) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE @@ -912,7 +912,7 @@ STAGE PLANS: Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: int) - outputColumnNames: _col4 + outputColumnNames: _col2 Select Vectorization: className: VectorSelectOperator native: true @@ -929,7 +929,7 @@ STAGE PLANS: native: false vectorProcessingMode: HASH projectedOutputColumns: [0] - keys: _col4 (type: int) + keys: _col2 (type: int) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE @@ -2084,7 +2084,7 @@ STAGE PLANS: Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: int) - outputColumnNames: _col4 + outputColumnNames: _col2 Select Vectorization: className: VectorSelectOperator native: true @@ -2101,7 +2101,7 @@ STAGE PLANS: native: false vectorProcessingMode: HASH projectedOutputColumns: [0] - keys: _col4 (type: int) + keys: _col2 (type: int) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE diff --git ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets1.q.out ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets1.q.out index c759170..cc9a67d 100644 --- ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets1.q.out +++ ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets1.q.out @@ -534,14 +534,14 @@ STAGE PLANS: outputColumnNames: a Statistics: Num rows: 6 Data size: 1104 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: a (type: string), 0 (type: int) + keys: a (type: string) mode: hash - outputColumnNames: _col0, _col1 + outputColumnNames: _col0 Statistics: Num rows: 6 Data size: 1104 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int) + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 1104 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs @@ -549,11 +549,10 @@ STAGE PLANS: Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: int) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE - pruneGroupingSetId: true File Output Operator compressed: false Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE @@ -612,16 +611,16 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 2208 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() - keys: _col0 (type: double), 0 (type: int) + keys: _col0 (type: double) mode: hash - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 6 Data size: 2208 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: double), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: double), _col1 (type: int) + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 6 Data size: 2208 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) + value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -629,22 +628,17 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: double), KEY._col1 (type: int) + keys: KEY._col0 (type: double) mode: mergepartial - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 3 Data size: 1104 Basic stats: COMPLETE Column stats: NONE - pruneGroupingSetId: true - Select Operator - expressions: _col0 (type: double), _col2 (type: bigint) - outputColumnNames: _col0, _col1 + File Output Operator + compressed: false Statistics: Num rows: 3 Data size: 1104 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 1104 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_limit.q.out ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_limit.q.out index 15190b8..b268de8 100644 --- ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_limit.q.out +++ ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_limit.q.out @@ -482,14 +482,14 @@ STAGE PLANS: outputColumnNames: a Statistics: Num rows: 6 Data size: 1104 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: a (type: string), 0 (type: int) + keys: a (type: string) mode: hash - outputColumnNames: _col0, _col1 + outputColumnNames: _col0 Statistics: Num rows: 6 Data size: 1104 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int) + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 1104 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap @@ -498,11 +498,10 @@ STAGE PLANS: Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: int) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE - pruneGroupingSetId: true Reduce Output Operator key expressions: _col0 (type: string) sort order: + @@ -577,17 +576,17 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 2208 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() - keys: _col0 (type: double), 0 (type: int) + keys: _col0 (type: double) mode: hash - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 6 Data size: 2208 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: double), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: double), _col1 (type: int) + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 6 Data size: 2208 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 - value expressions: _col2 (type: bigint) + value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -595,21 +594,16 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: double), KEY._col1 (type: int) + keys: KEY._col0 (type: double) mode: mergepartial - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 3 Data size: 1104 Basic stats: COMPLETE Column stats: NONE - pruneGroupingSetId: true - Select Operator - expressions: _col0 (type: double), _col2 (type: bigint) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + Statistics: Num rows: 3 Data size: 1104 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: double) - sort order: + - Statistics: Num rows: 3 Data size: 1104 Basic stats: COMPLETE Column stats: NONE - TopN Hash Memory Usage: 0.1 - value expressions: _col1 (type: bigint) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: bigint) Reducer 3 Execution mode: vectorized, llap Reduce Operator Tree: diff --git ql/src/test/results/clientpositive/timestamptz_3.q.out ql/src/test/results/clientpositive/timestamptz_3.q.out new file mode 100644 index 0000000..196c584 --- /dev/null +++ ql/src/test/results/clientpositive/timestamptz_3.q.out @@ -0,0 +1,57 @@ +PREHOOK: query: drop table tstz1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table tstz1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table tstz1(t timestamp with local time zone) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tstz1 +POSTHOOK: query: create table tstz1(t timestamp with local time zone) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tstz1 +PREHOOK: query: insert overwrite table tstz1 select cast('2016-01-03 12:26:34 America/Los_Angeles' as timestamp with local time zone) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@tstz1 +POSTHOOK: query: insert overwrite table tstz1 select cast('2016-01-03 12:26:34 America/Los_Angeles' as timestamp with local time zone) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@tstz1 +POSTHOOK: Lineage: tstz1.t SIMPLE [] +PREHOOK: query: select cast(t as timestamp) from tstz1 +PREHOOK: type: QUERY +PREHOOK: Input: default@tstz1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as timestamp) from tstz1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstz1 +#### A masked pattern was here #### +2016-01-03 12:26:34 +PREHOOK: query: select cast(to_epoch_milli(t) as timestamp) from tstz1 +PREHOOK: type: QUERY +PREHOOK: Input: default@tstz1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(to_epoch_milli(t) as timestamp) from tstz1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstz1 +#### A masked pattern was here #### +2016-01-03 12:26:34 +PREHOOK: query: select cast(t as timestamp) from tstz1 +PREHOOK: type: QUERY +PREHOOK: Input: default@tstz1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as timestamp) from tstz1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstz1 +#### A masked pattern was here #### +2016-01-03 20:26:34 +PREHOOK: query: select cast(to_epoch_milli(t) as timestamp) from tstz1 +PREHOOK: type: QUERY +PREHOOK: Input: default@tstz1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(to_epoch_milli(t) as timestamp) from tstz1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstz1 +#### A masked pattern was here #### +2016-01-03 12:26:34