From a20b8fe89aa44cccd9006aa226b924ca44c42cc9 Mon Sep 17 00:00:00 2001 From: Nishant Date: Sat, 6 Oct 2018 02:46:25 +0530 Subject: [PATCH] [HIVE-20700] Add config to diable rollup for druid tables --- .../apache/hadoop/hive/conf/Constants.java | 1 + .../org/apache/hadoop/hive/conf/HiveConf.java | 1 + .../hive/druid/DruidStorageHandlerUtils.java | 4 +++ .../clientpositive/druidmini_test_insert.q | 9 +++++++ .../druid/druidmini_test_insert.q.out | 25 +++++++++++++++++++ 5 files changed, 40 insertions(+) diff --git a/common/src/java/org/apache/hadoop/hive/conf/Constants.java b/common/src/java/org/apache/hadoop/hive/conf/Constants.java index 807d6bc5cc..a5da2d6424 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/Constants.java +++ b/common/src/java/org/apache/hadoop/hive/conf/Constants.java @@ -32,6 +32,7 @@ "org.apache.hadoop.hive.druid.io.DruidOutputFormat"; public static final String DRUID_DATA_SOURCE = "druid.datasource"; public static final String DRUID_SEGMENT_GRANULARITY = "druid.segment.granularity"; + public static final String DRUID_ROLLUP = "druid.rollup"; public static final String DRUID_QUERY_GRANULARITY = "druid.query.granularity"; public static final String DRUID_TARGET_SHARDS_PER_GRANULARITY = "druid.segment.targetShardsPerGranularity"; diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 58951efc12..2336f68162 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2748,6 +2748,7 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal HIVE_DRUID_BASE_PERSIST_DIRECTORY("hive.druid.basePersistDirectory", "", "Local temporary directory used to persist intermediate indexing state, will default to JVM system property java.io.tmpdir." ), + HIVE_DRUID_ROLLUP("hive.druid.rollup", true, "Whether to rollup druid rows or not."), DRUID_SEGMENT_DIRECTORY("hive.druid.storage.storageDirectory", "/druid/segments" , "druid deep storage location."), DRUID_METADATA_BASE("hive.druid.metadata.base", "druid", "Default prefix for metadata tables"), diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java index b9eb367f0f..5042351699 100644 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java +++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java @@ -826,12 +826,16 @@ public static GranularitySpec getGranularitySpec(Configuration configuration, Pr tableProperties.getProperty(Constants.DRUID_SEGMENT_GRANULARITY) != null ? tableProperties.getProperty(Constants.DRUID_SEGMENT_GRANULARITY) : HiveConf.getVar(configuration, HiveConf.ConfVars.HIVE_DRUID_INDEXING_GRANULARITY); + final boolean rollup = tableProperties.getProperty(Constants.DRUID_ROLLUP) != null ? + Boolean.parseBoolean(tableProperties.getProperty(Constants.DRUID_SEGMENT_GRANULARITY)): + HiveConf.getBoolVar(configuration, HiveConf.ConfVars.HIVE_DRUID_ROLLUP); return new UniformGranularitySpec( Granularity.fromString(segmentGranularity), Granularity.fromString( tableProperties.getProperty(Constants.DRUID_QUERY_GRANULARITY) == null ? "NONE" : tableProperties.getProperty(Constants.DRUID_QUERY_GRANULARITY)), + rollup, null ); } diff --git a/ql/src/test/queries/clientpositive/druidmini_test_insert.q b/ql/src/test/queries/clientpositive/druidmini_test_insert.q index 09e4a192b4..1c47b1acf4 100644 --- a/ql/src/test/queries/clientpositive/druidmini_test_insert.q +++ b/ql/src/test/queries/clientpositive/druidmini_test_insert.q @@ -139,6 +139,15 @@ EXPLAIN select * from druid_test_table_n9 where `__time` = cast('2015-03-09 23:5 EXPLAIN select * from druid_test_table_n9 where `__time` = cast('2015-03-10 00:00:00' as timestamp ); EXPLAIN select * from druid_test_table_n9 where `__time` = cast('2015-03-10 23:59:59' as timestamp ); +-- disable rollup and insert identical rows +SET hive.druid.rollup=false; +insert into druid_test_table_n9 values +('2015-03-12 00:00:00', 'i4-1', 4), +('2015-03-12 00:00:00', 'i4-1', 2), +('2015-03-12 00:00:00', 'i4-1', 1); + +select * from druid_test_table_n9 where `__time` = cast('2015-03-12 00:00:00' as timestamp ); + DROP TABLE test_base_table; DROP TABLE druid_test_table_n9; diff --git a/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out b/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out index a4a5594e5d..153285721e 100644 --- a/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out +++ b/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out @@ -814,6 +814,31 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 ListSink +PREHOOK: query: insert into druid_test_table_n9 values +('2015-03-12 00:00:00', 'i4-1', 4), +('2015-03-12 00:00:00', 'i4-1', 2), +('2015-03-12 00:00:00', 'i4-1', 1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: druid_test_dst@druid_test_table_n9 +POSTHOOK: query: insert into druid_test_table_n9 values +('2015-03-12 00:00:00', 'i4-1', 4), +('2015-03-12 00:00:00', 'i4-1', 2), +('2015-03-12 00:00:00', 'i4-1', 1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: druid_test_dst@druid_test_table_n9 +PREHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-12 00:00:00' as timestamp ) +PREHOOK: type: QUERY +PREHOOK: Input: druid_test_dst@druid_test_table_n9 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-12 00:00:00' as timestamp ) +POSTHOOK: type: QUERY +POSTHOOK: Input: druid_test_dst@druid_test_table_n9 +POSTHOOK: Output: hdfs://### HDFS PATH ### +2015-03-12 00:00:00.0 US/Pacific i4-1 4.0 +2015-03-12 00:00:00.0 US/Pacific i4-1 2.0 +2015-03-12 00:00:00.0 US/Pacific i4-1 1.0 PREHOOK: query: DROP TABLE test_base_table PREHOOK: type: DROPTABLE PREHOOK: Input: druid_test_dst@test_base_table -- 2.17.1 (Apple Git-112)