diff --git ql/src/test/queries/clientpositive/avro_external_tbl_partitioned_native.q ql/src/test/queries/clientpositive/avro_external_tbl_partitioned_native.q new file mode 100644 index 0000000..50a3060 --- /dev/null +++ ql/src/test/queries/clientpositive/avro_external_tbl_partitioned_native.q @@ -0,0 +1,33 @@ +dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/avro_external_tbl_partitioned_native; +dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/avro_external_tbl_partitioned_native/part; + +CREATE TABLE episodes ( + title string COMMENT "episode title", + air_date string COMMENT "initial date", + doctor int COMMENT "main actor playing the Doctor in episode") +STORED AS AVRO; + +LOAD DATA LOCAL INPATH '../../data/files/episodes.avro' INTO TABLE episodes; + +CREATE EXTERNAL TABLE episodes_partitioned ( + title string COMMENT "episode title", + doctor int COMMENT "main actor playing the Doctor in episode") +PARTITIONED BY (air_date string) +STORED AS AVRO +LOCATION '${system:test.tmp.dir}/avro_external_tbl_partitioned_native'; + +ALTER TABLE episodes_partitioned ADD IF NOT EXISTS PARTITION (air_date = '14 May 2011') LOCATION +'${system:test.tmp.dir}/avro_external_tbl_partitioned_native/part'; + +SET hive.exec.dynamic.partition.mode=nonstrict; +INSERT OVERWRITE TABLE episodes_partitioned PARTITION (air_date) +SELECT title, doctor, air_date FROM episodes; + +SELECT * FROM episodes_partitioned WHERE doctor > 6; + +-- Verify that Fetch works in addition to Map +SELECT * FROM episodes_partitioned ORDER BY air_date LIMIT 5; +-- Fetch w/filter to specific partition +SELECT * FROM episodes_partitioned WHERE doctor = 6; +-- Fetch w/non-existent partition +SELECT * FROM episodes_partitioned WHERE doctor = 7 LIMIT 5; \ No newline at end of file diff --git ql/src/test/results/clientpositive/avro_external_tbl_partitioned_native.q.out ql/src/test/results/clientpositive/avro_external_tbl_partitioned_native.q.out new file mode 100644 index 0000000..2ab8ec9 --- /dev/null +++ ql/src/test/results/clientpositive/avro_external_tbl_partitioned_native.q.out @@ -0,0 +1,199 @@ +PREHOOK: query: CREATE TABLE episodes ( + title string COMMENT "episode title", + air_date string COMMENT "initial date", + doctor int COMMENT "main actor playing the Doctor in episode") +STORED AS AVRO +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@episodes +POSTHOOK: query: CREATE TABLE episodes ( + title string COMMENT "episode title", + air_date string COMMENT "initial date", + doctor int COMMENT "main actor playing the Doctor in episode") +STORED AS AVRO +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@episodes +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/episodes.avro' INTO TABLE episodes +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@episodes +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/episodes.avro' INTO TABLE episodes +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@episodes +PREHOOK: query: CREATE EXTERNAL TABLE episodes_partitioned ( + title string COMMENT "episode title", + doctor int COMMENT "main actor playing the Doctor in episode") +PARTITIONED BY (air_date string) +STORED AS AVRO +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@episodes_partitioned +POSTHOOK: query: CREATE EXTERNAL TABLE episodes_partitioned ( + title string COMMENT "episode title", + doctor int COMMENT "main actor playing the Doctor in episode") +PARTITIONED BY (air_date string) +STORED AS AVRO +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@episodes_partitioned +PREHOOK: query: ALTER TABLE episodes_partitioned ADD IF NOT EXISTS PARTITION (air_date = '14 May 2011') LOCATION +#### A masked pattern was here #### +PREHOOK: type: ALTERTABLE_ADDPARTS +#### A masked pattern was here #### +PREHOOK: Output: default@episodes_partitioned +POSTHOOK: query: ALTER TABLE episodes_partitioned ADD IF NOT EXISTS PARTITION (air_date = '14 May 2011') LOCATION +#### A masked pattern was here #### +POSTHOOK: type: ALTERTABLE_ADDPARTS +#### A masked pattern was here #### +POSTHOOK: Output: default@episodes_partitioned +POSTHOOK: Output: default@episodes_partitioned@air_date=14 May 2011 +PREHOOK: query: INSERT OVERWRITE TABLE episodes_partitioned PARTITION (air_date) +SELECT title, doctor, air_date FROM episodes +PREHOOK: type: QUERY +PREHOOK: Input: default@episodes +PREHOOK: Output: default@episodes_partitioned +POSTHOOK: query: INSERT OVERWRITE TABLE episodes_partitioned PARTITION (air_date) +SELECT title, doctor, air_date FROM episodes +POSTHOOK: type: QUERY +POSTHOOK: Input: default@episodes +POSTHOOK: Output: default@episodes_partitioned@air_date=14 May 2011 +POSTHOOK: Output: default@episodes_partitioned@air_date=23 November 1963 +POSTHOOK: Output: default@episodes_partitioned@air_date=26 March 2005 +POSTHOOK: Output: default@episodes_partitioned@air_date=3 April 2010 +POSTHOOK: Output: default@episodes_partitioned@air_date=3 September 1977 +POSTHOOK: Output: default@episodes_partitioned@air_date=4 January 1982 +POSTHOOK: Output: default@episodes_partitioned@air_date=5 November 1966 +POSTHOOK: Output: default@episodes_partitioned@air_date=6 September 1986 +POSTHOOK: Lineage: episodes_partitioned PARTITION(air_date=14 May 2011).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(air_date=14 May 2011).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(air_date=23 November 1963).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(air_date=23 November 1963).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(air_date=26 March 2005).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(air_date=26 March 2005).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(air_date=3 April 2010).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(air_date=3 April 2010).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(air_date=3 September 1977).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(air_date=3 September 1977).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(air_date=4 January 1982).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(air_date=4 January 1982).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(air_date=5 November 1966).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(air_date=5 November 1966).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(air_date=6 September 1986).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(air_date=6 September 1986).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] +PREHOOK: query: SELECT * FROM episodes_partitioned WHERE doctor > 6 +PREHOOK: type: QUERY +PREHOOK: Input: default@episodes_partitioned +PREHOOK: Input: default@episodes_partitioned@air_date=14 May 2011 +PREHOOK: Input: default@episodes_partitioned@air_date=23 November 1963 +PREHOOK: Input: default@episodes_partitioned@air_date=26 March 2005 +PREHOOK: Input: default@episodes_partitioned@air_date=3 April 2010 +PREHOOK: Input: default@episodes_partitioned@air_date=3 September 1977 +PREHOOK: Input: default@episodes_partitioned@air_date=4 January 1982 +PREHOOK: Input: default@episodes_partitioned@air_date=5 November 1966 +PREHOOK: Input: default@episodes_partitioned@air_date=6 September 1986 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM episodes_partitioned WHERE doctor > 6 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@episodes_partitioned +POSTHOOK: Input: default@episodes_partitioned@air_date=14 May 2011 +POSTHOOK: Input: default@episodes_partitioned@air_date=23 November 1963 +POSTHOOK: Input: default@episodes_partitioned@air_date=26 March 2005 +POSTHOOK: Input: default@episodes_partitioned@air_date=3 April 2010 +POSTHOOK: Input: default@episodes_partitioned@air_date=3 September 1977 +POSTHOOK: Input: default@episodes_partitioned@air_date=4 January 1982 +POSTHOOK: Input: default@episodes_partitioned@air_date=5 November 1966 +POSTHOOK: Input: default@episodes_partitioned@air_date=6 September 1986 +#### A masked pattern was here #### +The Doctor's Wife 11 14 May 2011 +Rose 9 26 March 2005 +The Eleventh Hour 11 3 April 2010 +PREHOOK: query: -- Verify that Fetch works in addition to Map +SELECT * FROM episodes_partitioned ORDER BY air_date LIMIT 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@episodes_partitioned +PREHOOK: Input: default@episodes_partitioned@air_date=14 May 2011 +PREHOOK: Input: default@episodes_partitioned@air_date=23 November 1963 +PREHOOK: Input: default@episodes_partitioned@air_date=26 March 2005 +PREHOOK: Input: default@episodes_partitioned@air_date=3 April 2010 +PREHOOK: Input: default@episodes_partitioned@air_date=3 September 1977 +PREHOOK: Input: default@episodes_partitioned@air_date=4 January 1982 +PREHOOK: Input: default@episodes_partitioned@air_date=5 November 1966 +PREHOOK: Input: default@episodes_partitioned@air_date=6 September 1986 +#### A masked pattern was here #### +POSTHOOK: query: -- Verify that Fetch works in addition to Map +SELECT * FROM episodes_partitioned ORDER BY air_date LIMIT 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@episodes_partitioned +POSTHOOK: Input: default@episodes_partitioned@air_date=14 May 2011 +POSTHOOK: Input: default@episodes_partitioned@air_date=23 November 1963 +POSTHOOK: Input: default@episodes_partitioned@air_date=26 March 2005 +POSTHOOK: Input: default@episodes_partitioned@air_date=3 April 2010 +POSTHOOK: Input: default@episodes_partitioned@air_date=3 September 1977 +POSTHOOK: Input: default@episodes_partitioned@air_date=4 January 1982 +POSTHOOK: Input: default@episodes_partitioned@air_date=5 November 1966 +POSTHOOK: Input: default@episodes_partitioned@air_date=6 September 1986 +#### A masked pattern was here #### +The Doctor's Wife 11 14 May 2011 +An Unearthly Child 1 23 November 1963 +Rose 9 26 March 2005 +The Eleventh Hour 11 3 April 2010 +Horror of Fang Rock 4 3 September 1977 +PREHOOK: query: -- Fetch w/filter to specific partition +SELECT * FROM episodes_partitioned WHERE doctor = 6 +PREHOOK: type: QUERY +PREHOOK: Input: default@episodes_partitioned +PREHOOK: Input: default@episodes_partitioned@air_date=14 May 2011 +PREHOOK: Input: default@episodes_partitioned@air_date=23 November 1963 +PREHOOK: Input: default@episodes_partitioned@air_date=26 March 2005 +PREHOOK: Input: default@episodes_partitioned@air_date=3 April 2010 +PREHOOK: Input: default@episodes_partitioned@air_date=3 September 1977 +PREHOOK: Input: default@episodes_partitioned@air_date=4 January 1982 +PREHOOK: Input: default@episodes_partitioned@air_date=5 November 1966 +PREHOOK: Input: default@episodes_partitioned@air_date=6 September 1986 +#### A masked pattern was here #### +POSTHOOK: query: -- Fetch w/filter to specific partition +SELECT * FROM episodes_partitioned WHERE doctor = 6 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@episodes_partitioned +POSTHOOK: Input: default@episodes_partitioned@air_date=14 May 2011 +POSTHOOK: Input: default@episodes_partitioned@air_date=23 November 1963 +POSTHOOK: Input: default@episodes_partitioned@air_date=26 March 2005 +POSTHOOK: Input: default@episodes_partitioned@air_date=3 April 2010 +POSTHOOK: Input: default@episodes_partitioned@air_date=3 September 1977 +POSTHOOK: Input: default@episodes_partitioned@air_date=4 January 1982 +POSTHOOK: Input: default@episodes_partitioned@air_date=5 November 1966 +POSTHOOK: Input: default@episodes_partitioned@air_date=6 September 1986 +#### A masked pattern was here #### +The Mysterious Planet 6 6 September 1986 +PREHOOK: query: -- Fetch w/non-existent partition +SELECT * FROM episodes_partitioned WHERE doctor = 7 LIMIT 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@episodes_partitioned +PREHOOK: Input: default@episodes_partitioned@air_date=14 May 2011 +PREHOOK: Input: default@episodes_partitioned@air_date=23 November 1963 +PREHOOK: Input: default@episodes_partitioned@air_date=26 March 2005 +PREHOOK: Input: default@episodes_partitioned@air_date=3 April 2010 +PREHOOK: Input: default@episodes_partitioned@air_date=3 September 1977 +PREHOOK: Input: default@episodes_partitioned@air_date=4 January 1982 +PREHOOK: Input: default@episodes_partitioned@air_date=5 November 1966 +PREHOOK: Input: default@episodes_partitioned@air_date=6 September 1986 +#### A masked pattern was here #### +POSTHOOK: query: -- Fetch w/non-existent partition +SELECT * FROM episodes_partitioned WHERE doctor = 7 LIMIT 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@episodes_partitioned +POSTHOOK: Input: default@episodes_partitioned@air_date=14 May 2011 +POSTHOOK: Input: default@episodes_partitioned@air_date=23 November 1963 +POSTHOOK: Input: default@episodes_partitioned@air_date=26 March 2005 +POSTHOOK: Input: default@episodes_partitioned@air_date=3 April 2010 +POSTHOOK: Input: default@episodes_partitioned@air_date=3 September 1977 +POSTHOOK: Input: default@episodes_partitioned@air_date=4 January 1982 +POSTHOOK: Input: default@episodes_partitioned@air_date=5 November 1966 +POSTHOOK: Input: default@episodes_partitioned@air_date=6 September 1986 +#### A masked pattern was here ####