diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 808ec661b1..0332fdb178 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2958,6 +2958,8 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal HIVE_INSERT_INTO_MULTILEVEL_DIRS("hive.insert.into.multilevel.dirs", false, "Where to insert into multilevel directories like\n" + "\"insert directory '/HIVEFT25686/chinna/' from table\""), + HIVE_CTAS_EXTERNAL_TABLES("hive.ctas.external.tables", true, + "whether CTAS for external tables is allowed"), HIVE_INSERT_INTO_EXTERNAL_TABLES("hive.insert.into.external.tables", true, "whether insert into external tables is allowed"), HIVE_TEMPORARY_TABLE_STORAGE( diff --git druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java index 57e4800bad..53d93e1b74 100644 --- druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java +++ druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java @@ -215,12 +215,10 @@ public void configureInputJobCredentials(TableDesc tableDesc, Map existingDataSources = DruidStorageHandlerUtils .getAllDataSourceNames(getConnector(), getDruidMetadataStorageTablesConfig()); LOG.debug("pre-create data source with name {}", dataSourceName); + // Check for existence of for the datasource we are going to create in druid_segments table. if (existingDataSources.contains(dataSourceName)) { throw new MetaException(String.format("Data source [%s] already existing", dataSourceName)); } @@ -263,38 +254,17 @@ public void preCreateTable(Table table) throws MetaException { @Override public void rollbackCreateTable(Table table) { - if (MetaStoreUtils.isExternalTable(table)) { - return; - } - final Path segmentDescriptorDir = getSegmentDescriptorDir(); - try { - List dataSegmentList = DruidStorageHandlerUtils - .getCreatedSegments(segmentDescriptorDir, getConf()); - for (DataSegment dataSegment : dataSegmentList) { - try { - deleteSegment(dataSegment); - } catch (SegmentLoadingException e) { - LOG.error(String.format("Error while trying to clean the segment [%s]", dataSegment), e); - } - } - } catch (IOException e) { - LOG.error("Exception while rollback", e); - throw Throwables.propagate(e); - } finally { - cleanWorkingDir(); - } + cleanWorkingDir(); } @Override public void commitCreateTable(Table table) throws MetaException { - if (MetaStoreUtils.isExternalTable(table)) { - // For external tables, we do not need to do anything else - return; - } if(isKafkaStreamingTable(table)){ updateKafkaIngestion(table); } - this.commitInsertTable(table, true); + // For CTAS queries when user has explicitly specified the datasource. + // We will append the data to existing druid datasource. + this.commitInsertTable(table, false); } private void updateKafkaIngestion(Table table){ @@ -762,9 +732,6 @@ public void rollbackDropTable(Table table) { @Override public void commitDropTable(Table table, boolean deleteData) { - if (MetaStoreUtils.isExternalTable(table)) { - return; - } if(isKafkaStreamingTable(table)) { // Stop Kafka Ingestion first final String overlordAddress = Preconditions.checkNotNull(HiveConf @@ -775,12 +742,15 @@ public void commitDropTable(Table table, boolean deleteData) { "Druid Datasource name is null"); stopKafkaIngestion(overlordAddress, dataSourceName); } + String dataSourceName = Preconditions .checkNotNull(table.getParameters().get(Constants.DRUID_DATA_SOURCE), "DataSource name is null !" ); - - if (deleteData == true) { + // TODO: Move MetaStoreUtils.isExternalTablePurge(table) calls to a common place for all StorageHandlers + // deleteData flag passed down to StorageHandler should be true only if + // MetaStoreUtils.isExternalTablePurge(table) returns true. + if (deleteData == true && MetaStoreUtils.isExternalTablePurge(table)) { LOG.info("Dropping with purge all the data for data source {}", dataSourceName); List dataSegmentList = DruidStorageHandlerUtils .getDataSegmentList(getConnector(), getDruidMetadataStorageTablesConfig(), dataSourceName); @@ -806,9 +776,6 @@ public void commitDropTable(Table table, boolean deleteData) { public void commitInsertTable(Table table, boolean overwrite) throws MetaException { LOG.debug("commit insert into table {} overwrite {}", table.getTableName(), overwrite); - if (MetaStoreUtils.isExternalTable(table)) { - throw new MetaException("Cannot insert data into external table backed by Druid"); - } try { // Check if there segments to load final Path segmentDescriptorDir = getSegmentDescriptorDir(); diff --git druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java index f0e12a22fe..df9049ea02 100644 --- druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java +++ druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java @@ -483,7 +483,6 @@ protected SegmentAnalysis submitMetadataRequest(String address, SegmentMetadataQ } @Override public boolean shouldStoreFieldsInMetastore(Map tableParams) { - // If Druid table is not an external table store the schema in metadata store. - return !MetaStoreUtils.isExternal(tableParams); + return true; } } diff --git druid-handler/src/test/org/apache/hadoop/hive/druid/TestDruidStorageHandler.java druid-handler/src/test/org/apache/hadoop/hive/druid/TestDruidStorageHandler.java index b96a13fada..510330d5d0 100644 --- druid-handler/src/test/org/apache/hadoop/hive/druid/TestDruidStorageHandler.java +++ druid-handler/src/test/org/apache/hadoop/hive/druid/TestDruidStorageHandler.java @@ -106,7 +106,9 @@ private DataSegment createSegment(String location, Interval interval, String ver public void before() throws Throwable { tableWorkingPath = temporaryFolder.newFolder().getAbsolutePath(); segmentsTable = derbyConnectorRule.metadataTablesConfigSupplier().get().getSegmentsTable(); - Mockito.when(tableMock.getParameters()).thenReturn(new HashMap<>()); + HashMap params = new HashMap<>(); + params.put("external.table.purge", "TRUE"); + Mockito.when(tableMock.getParameters()).thenReturn(params); Mockito.when(tableMock.getPartitionKeysSize()).thenReturn(0); StorageDescriptor storageDes = Mockito.mock(StorageDescriptor.class); Mockito.when(storageDes.getBucketColsSize()).thenReturn(0); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 644ac1f3a0..c84ccda4db 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -13046,7 +13046,7 @@ ASTNode analyzeCreateTable( throw new SemanticException(ErrorMsg.CTAS_PARCOL_COEXISTENCE.getMsg()); } } - if (isExt) { + if (!conf.getBoolVar(ConfVars.HIVE_CTAS_EXTERNAL_TABLES) && isExt) { throw new SemanticException(ErrorMsg.CTAS_EXTTBL_COEXISTENCE.getMsg()); } command_type = CTAS; diff --git ql/src/test/queries/clientnegative/ctas.q ql/src/test/queries/clientnegative/ctas.q index 507a7a76b1..1d0afaf376 100644 --- ql/src/test/queries/clientnegative/ctas.q +++ ql/src/test/queries/clientnegative/ctas.q @@ -1,5 +1,5 @@ - +SET hive.ctas.external.tables=false; create external table nzhang_ctas4 as select key, value from src; diff --git ql/src/test/queries/clientnegative/druid_datasource2.q ql/src/test/queries/clientnegative/druid_datasource2.q index cc20931043..2e7400f47b 100644 --- ql/src/test/queries/clientnegative/druid_datasource2.q +++ ql/src/test/queries/clientnegative/druid_datasource2.q @@ -1,3 +1,4 @@ +SET metastore.strict.managed.tables=true; CREATE TABLE druid_table_1 STORED BY 'org.apache.hadoop.hive.druid.QTestDruidStorageHandler' TBLPROPERTIES ("property" = "localhost", "druid.datasource" = "mydatasource"); diff --git ql/src/test/queries/clientpositive/druid_timestamptz.q ql/src/test/queries/clientpositive/druid_timestamptz.q index 483004402f..605d240ae2 100644 --- ql/src/test/queries/clientpositive/druid_timestamptz.q +++ ql/src/test/queries/clientpositive/druid_timestamptz.q @@ -1,9 +1,10 @@ set hive.fetch.task.conversion=more; - +SET hive.ctas.external.tables=true; +SET hive.external.table.purge.default = true; drop table tstz1_n0; -create table tstz1_n0(`__time` timestamp with local time zone, n string, v integer) +create external table tstz1_n0(`__time` timestamp with local time zone, n string, v integer) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR"); diff --git ql/src/test/queries/clientpositive/druid_timestamptz2.q ql/src/test/queries/clientpositive/druid_timestamptz2.q index 8f573c8c9b..8b2c092994 100644 --- ql/src/test/queries/clientpositive/druid_timestamptz2.q +++ ql/src/test/queries/clientpositive/druid_timestamptz2.q @@ -1,9 +1,9 @@ +SET hive.ctas.external.tables=true; +SET hive.external.table.purge.default = true; CREATE database druid_test_dst; use druid_test_dst; - - -create table test_base_table(`timecolumn` timestamp, `interval_marker` string, `num_l` double); +create external table test_base_table(`timecolumn` timestamp, `interval_marker` string, `num_l` double); insert into test_base_table values ('2015-03-08 00:00:00', 'i1-start', 4); insert into test_base_table values ('2015-03-08 23:59:59', 'i1-end', 1); insert into test_base_table values ('2015-03-09 00:00:00', 'i2-start', 4); @@ -11,7 +11,7 @@ insert into test_base_table values ('2015-03-09 23:59:59', 'i2-end', 1); insert into test_base_table values ('2015-03-10 00:00:00', 'i3-start', 2); insert into test_base_table values ('2015-03-10 23:59:59', 'i3-end', 2); -CREATE TABLE druid_test_table_1 +CREATE EXTERNAL TABLE druid_test_table_1 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") AS @@ -20,7 +20,7 @@ FROM druid_test_dst.test_base_table; select * FROM druid_test_table_1; -CREATE TABLE druid_test_table_2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) +CREATE EXTERNAL TABLE druid_test_table_2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY"); @@ -36,7 +36,7 @@ select * FROM druid_test_table_2; SET TIME ZONE UTC; -CREATE TABLE druid_test_table_utc +CREATE EXTERNAL TABLE druid_test_table_utc STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") AS @@ -45,7 +45,7 @@ FROM druid_test_dst.test_base_table; select * FROM druid_test_table_utc; -CREATE TABLE druid_test_table_utc2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) +CREATE EXTERNAL TABLE druid_test_table_utc2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY"); diff --git ql/src/test/queries/clientpositive/druidkafkamini_basic.q ql/src/test/queries/clientpositive/druidkafkamini_basic.q index 814890a64f..4cdabf2d7d 100644 --- ql/src/test/queries/clientpositive/druidkafkamini_basic.q +++ ql/src/test/queries/clientpositive/druidkafkamini_basic.q @@ -1,5 +1,5 @@ SET hive.vectorized.execution.enabled=false; -CREATE TABLE druid_kafka_test(`__time` timestamp, page string, `user` string, language string, added int, deleted int) +CREATE EXTERNAL TABLE druid_kafka_test(`__time` timestamp, page string, `user` string, language string, added int, deleted int) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "MONTH", diff --git ql/src/test/queries/clientpositive/druidmini_dynamic_partition.q ql/src/test/queries/clientpositive/druidmini_dynamic_partition.q index 4443af2a57..de42900e53 100644 --- ql/src/test/queries/clientpositive/druidmini_dynamic_partition.q +++ ql/src/test/queries/clientpositive/druidmini_dynamic_partition.q @@ -1,5 +1,7 @@ SET hive.vectorized.execution.enabled=false; -CREATE TABLE druid_partitioned_table_0 +SET hive.ctas.external.tables=true; +SET hive.external.table.purge.default = true; +CREATE EXTERNAL TABLE druid_partitioned_table_0 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "HOUR", @@ -20,7 +22,7 @@ CREATE TABLE druid_partitioned_table_0 cboolean2 FROM alltypesorc where ctimestamp1 IS NOT NULL; -EXPLAIN CREATE TABLE druid_partitioned_table +EXPLAIN CREATE EXTERNAL TABLE druid_partitioned_table STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "HOUR", @@ -43,7 +45,7 @@ EXPLAIN CREATE TABLE druid_partitioned_table -CREATE TABLE druid_partitioned_table +CREATE EXTERNAL TABLE druid_partitioned_table STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "HOUR", @@ -154,7 +156,7 @@ SELECT sum(cint), sum(cbigint) FROM druid_partitioned_table ; set hive.druid.indexer.partition.size.max=10; -CREATE TABLE druid_max_size_partition +CREATE EXTERNAL TABLE druid_max_size_partition STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "HOUR", diff --git ql/src/test/queries/clientpositive/druidmini_expressions.q ql/src/test/queries/clientpositive/druidmini_expressions.q index fad8f73520..7857973611 100644 --- ql/src/test/queries/clientpositive/druidmini_expressions.q +++ ql/src/test/queries/clientpositive/druidmini_expressions.q @@ -1,7 +1,8 @@ --! qt:dataset:alltypesorc +SET hive.ctas.external.tables=true; SET hive.vectorized.execution.enabled=false; -CREATE TABLE druid_table_n0 +CREATE EXTERNAL TABLE druid_table_n0 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS diff --git ql/src/test/queries/clientpositive/druidmini_extractTime.q ql/src/test/queries/clientpositive/druidmini_extractTime.q index 95413612ea..0dbcd25e1c 100644 --- ql/src/test/queries/clientpositive/druidmini_extractTime.q +++ ql/src/test/queries/clientpositive/druidmini_extractTime.q @@ -1,7 +1,9 @@ --! qt:dataset:alltypesorc SET hive.vectorized.execution.enabled=false; -CREATE TABLE druid_table +SET hive.ctas.external.tables=true; +SET hive.external.table.purge.default = true; +CREATE EXTERNAL TABLE druid_table STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS @@ -181,7 +183,7 @@ SELECT CAST(`__time` AS DATE) AS `x_date` FROM druid_table ORDER BY `x_date` LIM create table test_extract_from_string_base_table(`timecolumn` timestamp, `date_c` string, `timestamp_c` string, `metric_c` double); insert into test_extract_from_string_base_table values ('2015-03-08 00:00:00', '2015-03-10', '2015-03-08 05:30:20', 5.0); -CREATE TABLE druid_test_extract_from_string_table +CREATE EXTERNAL TABLE druid_test_extract_from_string_table STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") AS select diff --git ql/src/test/queries/clientpositive/druidmini_floorTime.q ql/src/test/queries/clientpositive/druidmini_floorTime.q index a526413dfd..3b5334fbe0 100644 --- ql/src/test/queries/clientpositive/druidmini_floorTime.q +++ ql/src/test/queries/clientpositive/druidmini_floorTime.q @@ -1,7 +1,9 @@ --! qt:dataset:alltypesorc SET hive.vectorized.execution.enabled=false; -CREATE TABLE druid_table_n2 +SET hive.ctas.external.tables=true; +SET hive.external.table.purge.default = true; +CREATE EXTERNAL TABLE druid_table_n2 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS diff --git ql/src/test/queries/clientpositive/druidmini_joins.q ql/src/test/queries/clientpositive/druidmini_joins.q index 720127ed3f..1f92a0d1f0 100644 --- ql/src/test/queries/clientpositive/druidmini_joins.q +++ ql/src/test/queries/clientpositive/druidmini_joins.q @@ -1,11 +1,12 @@ SET hive.vectorized.execution.enabled=false; SET hive.explain.user=false; - +SET hive.ctas.external.tables=true; +SET hive.external.table.purge.default = true; --SET hive.execution.mode=llap; DROP TABLE druid_table_with_nulls; -CREATE TABLE druid_table_with_nulls +CREATE EXTERNAL TABLE druid_table_with_nulls STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR") AS diff --git ql/src/test/queries/clientpositive/druidmini_masking.q ql/src/test/queries/clientpositive/druidmini_masking.q index f0f2c0cbf6..824611f275 100644 --- ql/src/test/queries/clientpositive/druidmini_masking.q +++ ql/src/test/queries/clientpositive/druidmini_masking.q @@ -1,7 +1,9 @@ set hive.mapred.mode=nonstrict; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; +SET hive.ctas.external.tables=true; +SET hive.external.table.purge.default = true; -CREATE TABLE masking_test_druid +CREATE EXTERNAL TABLE masking_test_druid STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR") AS diff --git ql/src/test/queries/clientpositive/druidmini_test1.q ql/src/test/queries/clientpositive/druidmini_test1.q index f93665ed63..30abf3cea0 100644 --- ql/src/test/queries/clientpositive/druidmini_test1.q +++ ql/src/test/queries/clientpositive/druidmini_test1.q @@ -1,5 +1,8 @@ --! qt:dataset:alltypesorc -CREATE TABLE druid_table_n3 +SET hive.ctas.external.tables=true; +SET hive.external.table.purge.default = true; + +CREATE EXTERNAL TABLE druid_table_n3 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS diff --git ql/src/test/queries/clientpositive/druidmini_test_alter.q ql/src/test/queries/clientpositive/druidmini_test_alter.q index e19a00637b..0d8623d90d 100644 --- ql/src/test/queries/clientpositive/druidmini_test_alter.q +++ ql/src/test/queries/clientpositive/druidmini_test_alter.q @@ -1,4 +1,7 @@ -CREATE TABLE druid_alltypesorc_n0 +--! qt:dataset:alltypesorc +SET hive.ctas.external.tables=true; +SET hive.external.table.purge.default = true; +CREATE EXTERNAL TABLE druid_alltypesorc_n0 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS diff --git ql/src/test/queries/clientpositive/druidmini_test_insert.q ql/src/test/queries/clientpositive/druidmini_test_insert.q index cdd4c23ac8..dd1ba1a1e2 100644 --- ql/src/test/queries/clientpositive/druidmini_test_insert.q +++ ql/src/test/queries/clientpositive/druidmini_test_insert.q @@ -1,4 +1,6 @@ -CREATE TABLE druid_alltypesorc +SET hive.ctas.external.tables=true; +SET hive.external.table.purge.default = true; +CREATE EXTERNAL TABLE druid_alltypesorc STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS @@ -62,7 +64,7 @@ DROP TABLE druid_alltypesorc; insert into test_table_n9 values ('2015-01-08 00:00:00', 'i1-start', 4); insert into test_table_n9 values ('2015-01-08 23:59:59', 'i1-end', 1); - CREATE TABLE druid_table_n1 (`__time` timestamp with local time zone, `userid` string, `num_l` float) + CREATE EXTERNAL TABLE druid_table_n1 (`__time` timestamp with local time zone, `userid` string, `num_l` float) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY"); @@ -89,7 +91,7 @@ insert into test_base_table values ('2015-03-09 23:59:59', 'i2-end', 1); insert into test_base_table values ('2015-03-10 00:00:00', 'i3-start', 2); insert into test_base_table values ('2015-03-10 23:59:59', 'i3-end', 2); -CREATE TABLE druid_test_table_n9 +CREATE EXTERNAL TABLE druid_test_table_n9 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") AS diff --git ql/src/test/results/clientnegative/druid_datasource2.q.out ql/src/test/results/clientnegative/druid_datasource2.q.out index 1b74b0609c..2f783fe39a 100644 --- ql/src/test/results/clientnegative/druid_datasource2.q.out +++ ql/src/test/results/clientnegative/druid_datasource2.q.out @@ -4,4 +4,4 @@ TBLPROPERTIES ("property" = "localhost", "druid.datasource" = "mydatasource") PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@druid_table_1 -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Datasource name cannot be specified using [druid.datasource] for managed tables using Druid) +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Table default.druid_table_1 failed strict managed table checks due to the following reason: Table is marked as a managed table but is not transactional.) diff --git ql/src/test/results/clientpositive/druid/druid_timestamptz.q.out ql/src/test/results/clientpositive/druid/druid_timestamptz.q.out index ed1a6eae2e..fa9583a8e7 100644 --- ql/src/test/results/clientpositive/druid/druid_timestamptz.q.out +++ ql/src/test/results/clientpositive/druid/druid_timestamptz.q.out @@ -2,13 +2,13 @@ PREHOOK: query: drop table tstz1_n0 PREHOOK: type: DROPTABLE POSTHOOK: query: drop table tstz1_n0 POSTHOOK: type: DROPTABLE -PREHOOK: query: create table tstz1_n0(`__time` timestamp with local time zone, n string, v integer) +PREHOOK: query: create external table tstz1_n0(`__time` timestamp with local time zone, n string, v integer) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR") PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@tstz1_n0 -POSTHOOK: query: create table tstz1_n0(`__time` timestamp with local time zone, n string, v integer) +POSTHOOK: query: create external table tstz1_n0(`__time` timestamp with local time zone, n string, v integer) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR") POSTHOOK: type: CREATETABLE diff --git ql/src/test/results/clientpositive/druid/druid_timestamptz2.q.out ql/src/test/results/clientpositive/druid/druid_timestamptz2.q.out index c71a435fee..b62095aad6 100644 --- ql/src/test/results/clientpositive/druid/druid_timestamptz2.q.out +++ ql/src/test/results/clientpositive/druid/druid_timestamptz2.q.out @@ -10,11 +10,11 @@ PREHOOK: Input: database:druid_test_dst POSTHOOK: query: use druid_test_dst POSTHOOK: type: SWITCHDATABASE POSTHOOK: Input: database:druid_test_dst -PREHOOK: query: create table test_base_table(`timecolumn` timestamp, `interval_marker` string, `num_l` double) +PREHOOK: query: create external table test_base_table(`timecolumn` timestamp, `interval_marker` string, `num_l` double) PREHOOK: type: CREATETABLE PREHOOK: Output: database:druid_test_dst PREHOOK: Output: druid_test_dst@test_base_table -POSTHOOK: query: create table test_base_table(`timecolumn` timestamp, `interval_marker` string, `num_l` double) +POSTHOOK: query: create external table test_base_table(`timecolumn` timestamp, `interval_marker` string, `num_l` double) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:druid_test_dst POSTHOOK: Output: druid_test_dst@test_base_table @@ -84,7 +84,7 @@ POSTHOOK: Output: druid_test_dst@test_base_table POSTHOOK: Lineage: test_base_table.interval_marker SCRIPT [] POSTHOOK: Lineage: test_base_table.num_l SCRIPT [] POSTHOOK: Lineage: test_base_table.timecolumn SCRIPT [] -PREHOOK: query: CREATE TABLE druid_test_table_1 +PREHOOK: query: CREATE EXTERNAL TABLE druid_test_table_1 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") AS @@ -94,7 +94,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: druid_test_dst@test_base_table PREHOOK: Output: database:druid_test_dst PREHOOK: Output: druid_test_dst@druid_test_table_1 -POSTHOOK: query: CREATE TABLE druid_test_table_1 +POSTHOOK: query: CREATE EXTERNAL TABLE druid_test_table_1 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") AS @@ -121,13 +121,13 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 2015-03-09 23:59:59.0 US/Pacific i2-end 1.0 2015-03-10 00:00:00.0 US/Pacific i3-start 2.0 2015-03-10 23:59:59.0 US/Pacific i3-end 2.0 -PREHOOK: query: CREATE TABLE druid_test_table_2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) +PREHOOK: query: CREATE EXTERNAL TABLE druid_test_table_2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") PREHOOK: type: CREATETABLE PREHOOK: Output: database:druid_test_dst PREHOOK: Output: druid_test_dst@druid_test_table_2 -POSTHOOK: query: CREATE TABLE druid_test_table_2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) +POSTHOOK: query: CREATE EXTERNAL TABLE druid_test_table_2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") POSTHOOK: type: CREATETABLE @@ -195,7 +195,7 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 2015-03-09 23:59:59.0 US/Pacific i2-end 1.0 2015-03-10 00:00:00.0 US/Pacific i3-start 2.0 2015-03-10 23:59:59.0 US/Pacific i3-end 2.0 -PREHOOK: query: CREATE TABLE druid_test_table_utc +PREHOOK: query: CREATE EXTERNAL TABLE druid_test_table_utc STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") AS @@ -205,7 +205,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: druid_test_dst@test_base_table PREHOOK: Output: database:druid_test_dst PREHOOK: Output: druid_test_dst@druid_test_table_utc -POSTHOOK: query: CREATE TABLE druid_test_table_utc +POSTHOOK: query: CREATE EXTERNAL TABLE druid_test_table_utc STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") AS @@ -232,13 +232,13 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 2015-03-09 23:59:59.0 UTC i2-end 1.0 2015-03-10 00:00:00.0 UTC i3-start 2.0 2015-03-10 23:59:59.0 UTC i3-end 2.0 -PREHOOK: query: CREATE TABLE druid_test_table_utc2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) +PREHOOK: query: CREATE EXTERNAL TABLE druid_test_table_utc2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") PREHOOK: type: CREATETABLE PREHOOK: Output: database:druid_test_dst PREHOOK: Output: druid_test_dst@druid_test_table_utc2 -POSTHOOK: query: CREATE TABLE druid_test_table_utc2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) +POSTHOOK: query: CREATE EXTERNAL TABLE druid_test_table_utc2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") POSTHOOK: type: CREATETABLE diff --git ql/src/test/results/clientpositive/druid/druidkafkamini_basic.q.out ql/src/test/results/clientpositive/druid/druidkafkamini_basic.q.out index 07439743f0..ffd9279efe 100644 --- ql/src/test/results/clientpositive/druid/druidkafkamini_basic.q.out +++ ql/src/test/results/clientpositive/druid/druidkafkamini_basic.q.out @@ -1,4 +1,4 @@ -PREHOOK: query: CREATE TABLE druid_kafka_test(`__time` timestamp, page string, `user` string, language string, added int, deleted int) +PREHOOK: query: CREATE EXTERNAL TABLE druid_kafka_test(`__time` timestamp, page string, `user` string, language string, added int, deleted int) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "MONTH", @@ -15,7 +15,7 @@ PREHOOK: query: CREATE TABLE druid_kafka_test(`__time` timestamp, page string, ` PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@druid_kafka_test -POSTHOOK: query: CREATE TABLE druid_kafka_test(`__time` timestamp, page string, `user` string, language string, added int, deleted int) +POSTHOOK: query: CREATE EXTERNAL TABLE druid_kafka_test(`__time` timestamp, page string, `user` string, language string, added int, deleted int) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "MONTH", @@ -339,6 +339,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"__time":"true","added":"true","deleted":"true","language":"true","page":"true","user":"true"}} + EXTERNAL TRUE bucket_count -1 bucketing_version 2 column.name.delimiter , @@ -379,6 +380,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"__time":"true","added":"true","deleted":"true","language":"true","page":"true","user":"true"}} + EXTERNAL TRUE bucket_count -1 bucketing_version 2 column.name.delimiter , diff --git ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out index 28deb79821..8b1e44694a 100644 --- ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out +++ ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out @@ -1,4 +1,4 @@ -PREHOOK: query: CREATE TABLE druid_partitioned_table_0 +PREHOOK: query: CREATE EXTERNAL TABLE druid_partitioned_table_0 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "HOUR", @@ -22,7 +22,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc PREHOOK: Output: database:default PREHOOK: Output: default@druid_partitioned_table_0 -POSTHOOK: query: CREATE TABLE druid_partitioned_table_0 +POSTHOOK: query: CREATE EXTERNAL TABLE druid_partitioned_table_0 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "HOUR", @@ -57,7 +57,7 @@ POSTHOOK: Lineage: druid_partitioned_table_0.csmallint SIMPLE [(alltypesorc)allt POSTHOOK: Lineage: druid_partitioned_table_0.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: druid_partitioned_table_0.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] POSTHOOK: Lineage: druid_partitioned_table_0.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] -PREHOOK: query: EXPLAIN CREATE TABLE druid_partitioned_table +PREHOOK: query: EXPLAIN CREATE EXTERNAL TABLE druid_partitioned_table STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "HOUR", @@ -78,7 +78,7 @@ PREHOOK: query: EXPLAIN CREATE TABLE druid_partitioned_table cboolean2 FROM alltypesorc where ctimestamp1 IS NOT NULL PREHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: query: EXPLAIN CREATE TABLE druid_partitioned_table +POSTHOOK: query: EXPLAIN CREATE EXTERNAL TABLE druid_partitioned_table STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "HOUR", @@ -165,6 +165,8 @@ STAGE PLANS: druid.query.granularity MINUTE druid.segment.granularity HOUR druid.segment.targetShardsPerGranularity 6 + external.table.purge true + isExternal: true Stage: Stage-3 Stats Work @@ -176,7 +178,7 @@ STAGE PLANS: hdfs directory: true destination: hdfs://### HDFS PATH ### -PREHOOK: query: CREATE TABLE druid_partitioned_table +PREHOOK: query: CREATE EXTERNAL TABLE druid_partitioned_table STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "HOUR", @@ -200,7 +202,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc PREHOOK: Output: database:default PREHOOK: Output: default@druid_partitioned_table -POSTHOOK: query: CREATE TABLE druid_partitioned_table +POSTHOOK: query: CREATE EXTERNAL TABLE druid_partitioned_table STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "HOUR", @@ -567,7 +569,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@druid_partitioned_table POSTHOOK: Output: hdfs://### HDFS PATH ### 1408069801800 10992545287 -PREHOOK: query: CREATE TABLE druid_max_size_partition +PREHOOK: query: CREATE EXTERNAL TABLE druid_max_size_partition STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "HOUR", @@ -590,7 +592,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc PREHOOK: Output: database:default PREHOOK: Output: default@druid_max_size_partition -POSTHOOK: query: CREATE TABLE druid_max_size_partition +POSTHOOK: query: CREATE EXTERNAL TABLE druid_max_size_partition STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "HOUR", diff --git ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out index fd77a915d9..e322d8f91f 100644 --- ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out +++ ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out @@ -1,4 +1,4 @@ -PREHOOK: query: CREATE TABLE druid_table_n0 +PREHOOK: query: CREATE EXTERNAL TABLE druid_table_n0 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS @@ -18,7 +18,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc PREHOOK: Output: database:default PREHOOK: Output: default@druid_table_n0 -POSTHOOK: query: CREATE TABLE druid_table_n0 +POSTHOOK: query: CREATE EXTERNAL TABLE druid_table_n0 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS @@ -270,6 +270,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + EXTERNAL TRUE bucket_count -1 bucketing_version 2 column.name.delimiter , @@ -301,6 +302,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + EXTERNAL TRUE bucket_count -1 bucketing_version 2 column.name.delimiter , diff --git ql/src/test/results/clientpositive/druid/druidmini_extractTime.q.out ql/src/test/results/clientpositive/druid/druidmini_extractTime.q.out index 30e273bae8..116c3266f4 100644 --- ql/src/test/results/clientpositive/druid/druidmini_extractTime.q.out +++ ql/src/test/results/clientpositive/druid/druidmini_extractTime.q.out @@ -1,4 +1,4 @@ -PREHOOK: query: CREATE TABLE druid_table +PREHOOK: query: CREATE EXTERNAL TABLE druid_table STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS @@ -18,7 +18,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc PREHOOK: Output: database:default PREHOOK: Output: default@druid_table -POSTHOOK: query: CREATE TABLE druid_table +POSTHOOK: query: CREATE EXTERNAL TABLE druid_table STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS @@ -776,7 +776,7 @@ POSTHOOK: Lineage: test_extract_from_string_base_table.date_c SCRIPT [] POSTHOOK: Lineage: test_extract_from_string_base_table.metric_c SCRIPT [] POSTHOOK: Lineage: test_extract_from_string_base_table.timecolumn SCRIPT [] POSTHOOK: Lineage: test_extract_from_string_base_table.timestamp_c SCRIPT [] -PREHOOK: query: CREATE TABLE druid_test_extract_from_string_table +PREHOOK: query: CREATE EXTERNAL TABLE druid_test_extract_from_string_table STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") AS select @@ -786,7 +786,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@test_extract_from_string_base_table PREHOOK: Output: database:default PREHOOK: Output: default@druid_test_extract_from_string_table -POSTHOOK: query: CREATE TABLE druid_test_extract_from_string_table +POSTHOOK: query: CREATE EXTERNAL TABLE druid_test_extract_from_string_table STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") AS select diff --git ql/src/test/results/clientpositive/druid/druidmini_floorTime.q.out ql/src/test/results/clientpositive/druid/druidmini_floorTime.q.out index c54fd93a57..1c9e9c6718 100644 --- ql/src/test/results/clientpositive/druid/druidmini_floorTime.q.out +++ ql/src/test/results/clientpositive/druid/druidmini_floorTime.q.out @@ -1,4 +1,4 @@ -PREHOOK: query: CREATE TABLE druid_table_n2 +PREHOOK: query: CREATE EXTERNAL TABLE druid_table_n2 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS @@ -18,7 +18,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc PREHOOK: Output: database:default PREHOOK: Output: default@druid_table_n2 -POSTHOOK: query: CREATE TABLE druid_table_n2 +POSTHOOK: query: CREATE EXTERNAL TABLE druid_table_n2 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS diff --git ql/src/test/results/clientpositive/druid/druidmini_joins.q.out ql/src/test/results/clientpositive/druid/druidmini_joins.q.out index 73a3c9fad3..014c7b5406 100644 --- ql/src/test/results/clientpositive/druid/druidmini_joins.q.out +++ ql/src/test/results/clientpositive/druid/druidmini_joins.q.out @@ -2,7 +2,7 @@ PREHOOK: query: DROP TABLE druid_table_with_nulls PREHOOK: type: DROPTABLE POSTHOOK: query: DROP TABLE druid_table_with_nulls POSTHOOK: type: DROPTABLE -PREHOOK: query: CREATE TABLE druid_table_with_nulls +PREHOOK: query: CREATE EXTERNAL TABLE druid_table_with_nulls STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR") AS @@ -22,7 +22,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: database:default PREHOOK: Output: default@druid_table_with_nulls -POSTHOOK: query: CREATE TABLE druid_table_with_nulls +POSTHOOK: query: CREATE EXTERNAL TABLE druid_table_with_nulls STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR") AS diff --git ql/src/test/results/clientpositive/druid/druidmini_masking.q.out ql/src/test/results/clientpositive/druid/druidmini_masking.q.out index 1aad9677a0..e3b50a4224 100644 --- ql/src/test/results/clientpositive/druid/druidmini_masking.q.out +++ ql/src/test/results/clientpositive/druid/druidmini_masking.q.out @@ -1,4 +1,4 @@ -PREHOOK: query: CREATE TABLE masking_test_druid +PREHOOK: query: CREATE EXTERNAL TABLE masking_test_druid STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR") AS @@ -17,7 +17,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: database:default PREHOOK: Output: default@masking_test_druid -POSTHOOK: query: CREATE TABLE masking_test_druid +POSTHOOK: query: CREATE EXTERNAL TABLE masking_test_druid STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR") AS diff --git ql/src/test/results/clientpositive/druid/druidmini_test1.q.out ql/src/test/results/clientpositive/druid/druidmini_test1.q.out index 4e078aa41c..7f6c6b0aa3 100644 --- ql/src/test/results/clientpositive/druid/druidmini_test1.q.out +++ ql/src/test/results/clientpositive/druid/druidmini_test1.q.out @@ -1,4 +1,4 @@ -PREHOOK: query: CREATE TABLE druid_table_n3 +PREHOOK: query: CREATE EXTERNAL TABLE druid_table_n3 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS @@ -18,7 +18,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc PREHOOK: Output: database:default PREHOOK: Output: default@druid_table_n3 -POSTHOOK: query: CREATE TABLE druid_table_n3 +POSTHOOK: query: CREATE EXTERNAL TABLE druid_table_n3 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS diff --git ql/src/test/results/clientpositive/druid/druidmini_test_alter.q.out ql/src/test/results/clientpositive/druid/druidmini_test_alter.q.out index 439dc9c16c..a175f7650a 100644 --- ql/src/test/results/clientpositive/druid/druidmini_test_alter.q.out +++ ql/src/test/results/clientpositive/druid/druidmini_test_alter.q.out @@ -1,4 +1,4 @@ -PREHOOK: query: CREATE TABLE druid_alltypesorc_n0 +PREHOOK: query: CREATE EXTERNAL TABLE druid_alltypesorc_n0 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS @@ -16,7 +16,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc PREHOOK: Output: database:default PREHOOK: Output: default@druid_alltypesorc_n0 -POSTHOOK: query: CREATE TABLE druid_alltypesorc_n0 +POSTHOOK: query: CREATE EXTERNAL TABLE druid_alltypesorc_n0 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS diff --git ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out index c675aeb17d..0da76109ac 100644 --- ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out +++ ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out @@ -1,4 +1,4 @@ -PREHOOK: query: CREATE TABLE druid_alltypesorc +PREHOOK: query: CREATE EXTERNAL TABLE druid_alltypesorc STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS @@ -18,7 +18,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc PREHOOK: Output: database:default PREHOOK: Output: default@druid_alltypesorc -POSTHOOK: query: CREATE TABLE druid_alltypesorc +POSTHOOK: query: CREATE EXTERNAL TABLE druid_alltypesorc STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS @@ -194,13 +194,13 @@ POSTHOOK: Output: druid_test_create_then_insert@test_table_n9 POSTHOOK: Lineage: test_table_n9.num_l SCRIPT [] POSTHOOK: Lineage: test_table_n9.timecolumn SCRIPT [] POSTHOOK: Lineage: test_table_n9.userid SCRIPT [] -PREHOOK: query: CREATE TABLE druid_table_n1 (`__time` timestamp with local time zone, `userid` string, `num_l` float) +PREHOOK: query: CREATE EXTERNAL TABLE druid_table_n1 (`__time` timestamp with local time zone, `userid` string, `num_l` float) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") PREHOOK: type: CREATETABLE PREHOOK: Output: database:druid_test_create_then_insert PREHOOK: Output: druid_test_create_then_insert@druid_table_n1 -POSTHOOK: query: CREATE TABLE druid_table_n1 (`__time` timestamp with local time zone, `userid` string, `num_l` float) +POSTHOOK: query: CREATE EXTERNAL TABLE druid_table_n1 (`__time` timestamp with local time zone, `userid` string, `num_l` float) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") POSTHOOK: type: CREATETABLE @@ -335,7 +335,7 @@ POSTHOOK: Output: druid_test_dst@test_base_table POSTHOOK: Lineage: test_base_table.num_l SCRIPT [] POSTHOOK: Lineage: test_base_table.timecolumn SCRIPT [] POSTHOOK: Lineage: test_base_table.userid SCRIPT [] -PREHOOK: query: CREATE TABLE druid_test_table_n9 +PREHOOK: query: CREATE EXTERNAL TABLE druid_test_table_n9 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") AS @@ -344,7 +344,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: druid_test_dst@test_base_table PREHOOK: Output: database:druid_test_dst PREHOOK: Output: druid_test_dst@druid_test_table_n9 -POSTHOOK: query: CREATE TABLE druid_test_table_n9 +POSTHOOK: query: CREATE EXTERNAL TABLE druid_test_table_n9 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") AS diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java index 32850fdae7..0d4fc42ec7 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java @@ -566,6 +566,25 @@ public static boolean isExternalTable(Table table) { return isExternal(params); } + /** + * Determines whether an table needs to be purged or not. + * + * @param table table of interest + * + * @return true if external table needs to be purged + */ + public static boolean isExternalTablePurge(Table table) { + if (table == null) { + return false; + } + Map params = table.getParameters(); + if (params == null) { + return false; + } + + return isPropertyTrue(params, EXTERNAL_TABLE_PURGE); + } + public static boolean isExternal(Map tableParams){ return "TRUE".equalsIgnoreCase(tableParams.get("EXTERNAL")); }