diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 34df01e60e..7759c2aaf3 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2342,9 +2342,6 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "Currently it only works with Apache Tez. This should always be set to true. \n" + "Since it is a new feature, it has been made configurable."), - @Deprecated - HIVEOPTSORTDYNAMICPARTITION("hive.optimize.sort.dynamic.partition", false, - "Deprecated. Use hive.optimize.sort.dynamic.partition.threshold instead."), HIVEOPTSORTDYNAMICPARTITIONTHRESHOLD("hive.optimize.sort.dynamic.partition.threshold", 0, "When enabled dynamic partitioning column will be globally sorted.\n" + "This way we can keep only one record writer open for each partition value\n" + diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_dynamic_partitions_merge_move.q b/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_dynamic_partitions_merge_move.q index 84ca1c0c91..2e4578f718 100644 --- a/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_dynamic_partitions_merge_move.q +++ b/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_dynamic_partitions_merge_move.q @@ -2,7 +2,6 @@ set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat; SET hive.blobstore.optimizations.enabled=true; SET mapreduce.input.fileinputformat.split.maxsize=10; SET hive.merge.mapfiles=true; -set hive.optimize.sort.dynamic.partition=false; CREATE TABLE tmp_table_merge_move (id string, name string, dt string, pid int); diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_dynamic_partitions_merge_only.q b/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_dynamic_partitions_merge_only.q index c6e3a6dfc6..6321a614b8 100644 --- a/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_dynamic_partitions_merge_only.q +++ b/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_dynamic_partitions_merge_only.q @@ -2,7 +2,6 @@ set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat; SET hive.blobstore.optimizations.enabled=true; SET mapreduce.input.fileinputformat.split.maxsize=10; SET hive.merge.mapfiles=true; -set hive.optimize.sort.dynamic.partition=false; CREATE TABLE tmp_table_merge (id string, name string, dt string, pid int); diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_dynamic_partitions_move_only.q b/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_dynamic_partitions_move_only.q index a877362abc..0909661fe0 100644 --- a/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_dynamic_partitions_move_only.q +++ b/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_dynamic_partitions_move_only.q @@ -2,7 +2,6 @@ set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat; SET hive.blobstore.optimizations.enabled=true; SET mapreduce.input.fileinputformat.split.maxsize=10; SET hive.merge.mapfiles=true; -set hive.optimize.sort.dynamic.partition=false; CREATE TABLE tmp_table_move (id string, name string, dt string, pid int); diff --git a/itests/hive-blobstore/src/test/resources/hive-site.xml b/itests/hive-blobstore/src/test/resources/hive-site.xml index 91bef552a2..995487e0cf 100644 --- a/itests/hive-blobstore/src/test/resources/hive-site.xml +++ b/itests/hive-blobstore/src/test/resources/hive-site.xml @@ -239,11 +239,6 @@ false - - hive.optimize.sort.dynamic.partition - true - - hive.dummyparam.test.server.specific.config.override from.hive-site.xml diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java index 51b0fa336f..f0766cd087 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java @@ -257,7 +257,7 @@ public void testSdpoBucketed() throws Exception { private void testSdpoBucketed(boolean isVectorized, boolean isSdpo, int bucketing_version) throws Exception { hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, isVectorized); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVEOPTSORTDYNAMICPARTITION, isSdpo); + hiveConf.setIntVar(HiveConf.ConfVars.HIVEOPTSORTDYNAMICPARTITIONTHRESHOLD, (isSdpo? 1:-1)); runStatementOnDriver("drop table if exists acid_uap"); runStatementOnDriver("create transactional table acid_uap(a int, b varchar(128)) " + "partitioned by (ds string) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES " + diff --git a/ql/src/test/queries/clientpositive/acid_table_stats.q b/ql/src/test/queries/clientpositive/acid_table_stats.q index 8547a7ed34..27c746c130 100644 --- a/ql/src/test/queries/clientpositive/acid_table_stats.q +++ b/ql/src/test/queries/clientpositive/acid_table_stats.q @@ -4,7 +4,6 @@ set hive.support.concurrency=true; set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; -set hive.optimize.sort.dynamic.partition=true; -- single level partition, sorted dynamic partition enabled drop table acid; diff --git a/ql/src/test/queries/clientpositive/dynpart_merge.q b/ql/src/test/queries/clientpositive/dynpart_merge.q index b0337df72d..94e08a1b33 100644 --- a/ql/src/test/queries/clientpositive/dynpart_merge.q +++ b/ql/src/test/queries/clientpositive/dynpart_merge.q @@ -1,7 +1,6 @@ set hive.mapred.mode=nonstrict; set hive.exec.dynamic.partition=true; set hive.exec.dynamic.partition.mode=strict; -set hive.optimize.sort.dynamic.partition=false; set hive.merge.mapfiles=true; set hive.merge.mapredfiles=true; set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; diff --git a/ql/src/test/queries/clientpositive/dynpart_sort_opt_bucketing.q b/ql/src/test/queries/clientpositive/dynpart_sort_opt_bucketing.q index 5430ebdb48..22b02d5c3a 100644 --- a/ql/src/test/queries/clientpositive/dynpart_sort_opt_bucketing.q +++ b/ql/src/test/queries/clientpositive/dynpart_sort_opt_bucketing.q @@ -14,7 +14,6 @@ into 256 buckets stored as textfile; load data local inpath '../../data/files/sortdp/000000_0' overwrite into table t1_staging partition (e='epart'); -set hive.optimize.sort.dynamic.partition=true; @@ -42,7 +41,6 @@ dfs -cat ${hiveconf:hive.metastore.warehouse.dir}/t1_n147/e=epart/000006_0; select 'bucket_8'; dfs -cat ${hiveconf:hive.metastore.warehouse.dir}/t1_n147/e=epart/000008_0; -set hive.optimize.sort.dynamic.partition=false; diff --git a/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q b/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q index cbd66df690..86e0c11bb5 100644 --- a/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q +++ b/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q @@ -2,7 +2,6 @@ set hive.compute.query.using.stats=false; set hive.mapred.mode=nonstrict; set hive.explain.user=false; -set hive.optimize.sort.dynamic.partition=true; set hive.exec.dynamic.partition=true; set hive.exec.max.dynamic.partitions=1000; set hive.exec.max.dynamic.partitions.pernode=1000; diff --git a/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q b/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q index e6db525a67..2ad6d364ab 100644 --- a/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q +++ b/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q @@ -2,7 +2,6 @@ SET hive.vectorized.execution.enabled=false; set hive.compute.query.using.stats=false; set hive.mapred.mode=nonstrict; set hive.explain.user=false; -set hive.optimize.sort.dynamic.partition=true; set hive.exec.dynamic.partition=true; set hive.exec.max.dynamic.partitions=1000; set hive.exec.max.dynamic.partitions.pernode=1000; @@ -197,7 +196,6 @@ create table over1k_part3( b bigint, f float) partitioned by (s string, t tinyint, i int); -set hive.optimize.sort.dynamic.partition=false; insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where s="foo"; insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27; insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100; diff --git a/ql/src/test/queries/clientpositive/dynpart_sort_optimization2.q b/ql/src/test/queries/clientpositive/dynpart_sort_optimization2.q index 9d5681e47c..28a41d1712 100644 --- a/ql/src/test/queries/clientpositive/dynpart_sort_optimization2.q +++ b/ql/src/test/queries/clientpositive/dynpart_sort_optimization2.q @@ -1,7 +1,6 @@ --! qt:dataset:src set hive.mapred.mode=nonstrict; set hive.explain.user=false; -set hive.optimize.sort.dynamic.partition=true; set hive.exec.dynamic.partition=true; set hive.exec.max.dynamic.partitions=1000; set hive.exec.max.dynamic.partitions.pernode=1000; diff --git a/ql/src/test/queries/clientpositive/materialized_view_partitioned_3.q b/ql/src/test/queries/clientpositive/materialized_view_partitioned_3.q index db9944e245..a02d69bfb6 100644 --- a/ql/src/test/queries/clientpositive/materialized_view_partitioned_3.q +++ b/ql/src/test/queries/clientpositive/materialized_view_partitioned_3.q @@ -2,7 +2,6 @@ set hive.support.concurrency=true; set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; -set hive.optimize.sort.dynamic.partition=true; CREATE TABLE src_txn stored as orc TBLPROPERTIES ('transactional' = 'true') AS SELECT * FROM src; diff --git a/ql/src/test/queries/clientpositive/merge_dynamic_partition.q b/ql/src/test/queries/clientpositive/merge_dynamic_partition.q index 9c79b99521..883932788f 100644 --- a/ql/src/test/queries/clientpositive/merge_dynamic_partition.q +++ b/ql/src/test/queries/clientpositive/merge_dynamic_partition.q @@ -20,7 +20,6 @@ set hive.input.format=org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat; set hive.merge.mapfiles=false; set hive.merge.mapredfiles=false; set hive.merge.smallfiles.avgsize=1000000000; -set hive.optimize.sort.dynamic.partition=false; explain insert overwrite table merge_dynamic_part_n1 partition (ds='2008-04-08', hr) select key, value, hr from srcpart_merge_dp_n1 where ds='2008-04-08'; insert overwrite table merge_dynamic_part_n1 partition (ds='2008-04-08', hr) select key, value, hr from srcpart_merge_dp_n1 where ds='2008-04-08'; diff --git a/ql/src/test/queries/clientpositive/merge_dynamic_partition2.q b/ql/src/test/queries/clientpositive/merge_dynamic_partition2.q index 487b39057b..84ee13e883 100644 --- a/ql/src/test/queries/clientpositive/merge_dynamic_partition2.q +++ b/ql/src/test/queries/clientpositive/merge_dynamic_partition2.q @@ -21,7 +21,6 @@ set hive.merge.mapfiles=true; set hive.merge.mapredfiles=true; set hive.merge.smallfiles.avgsize=3000; set hive.exec.compress.output=false; -set hive.optimize.sort.dynamic.partition=false; explain insert overwrite table merge_dynamic_part_n0 partition (ds='2008-04-08', hr) select key, value, hr from srcpart_merge_dp_n0 where ds='2008-04-08'; diff --git a/ql/src/test/queries/clientpositive/mm_dp.q b/ql/src/test/queries/clientpositive/mm_dp.q index 212d48b584..b0b550927d 100644 --- a/ql/src/test/queries/clientpositive/mm_dp.q +++ b/ql/src/test/queries/clientpositive/mm_dp.q @@ -17,7 +17,6 @@ set hive.support.concurrency=true; set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; set hive.exec.max.dynamic.partitions=1000; set hive.exec.max.dynamic.partitions.pernode=1000; -set hive.optimize.sort.dynamic.partition=true; -- Force multiple writers when reading drop table dp_mm; diff --git a/ql/src/test/queries/clientpositive/orc_merge10.q b/ql/src/test/queries/clientpositive/orc_merge10.q index 9ea555c7b4..5ef66e7f01 100644 --- a/ql/src/test/queries/clientpositive/orc_merge10.q +++ b/ql/src/test/queries/clientpositive/orc_merge10.q @@ -7,7 +7,6 @@ set hive.mapred.mode=nonstrict; set hive.explain.user=false; set hive.merge.orcfile.stripe.level=false; set hive.exec.dynamic.partition=true; -set hive.optimize.sort.dynamic.partition=false; set mapred.min.split.size=1000; set mapred.max.split.size=2000; set tez.am.grouping.split-count=2; diff --git a/ql/src/test/queries/clientpositive/orc_merge7.q b/ql/src/test/queries/clientpositive/orc_merge7.q index 9e82c18366..959c28b435 100644 --- a/ql/src/test/queries/clientpositive/orc_merge7.q +++ b/ql/src/test/queries/clientpositive/orc_merge7.q @@ -22,7 +22,6 @@ set hive.compute.splits.in.am=true; set tez.grouping.min-size=1000; set tez.grouping.max-size=50000; set hive.exec.dynamic.partition=true; -set hive.optimize.sort.dynamic.partition=false; set hive.merge.sparkfiles=false; -- 3 mappers diff --git a/ql/src/test/queries/clientpositive/orc_merge_diff_fs.q b/ql/src/test/queries/clientpositive/orc_merge_diff_fs.q index 871f455047..41dde570bb 100644 --- a/ql/src/test/queries/clientpositive/orc_merge_diff_fs.q +++ b/ql/src/test/queries/clientpositive/orc_merge_diff_fs.q @@ -7,7 +7,6 @@ set hive.mapred.mode=nonstrict; set hive.explain.user=false; set hive.merge.orcfile.stripe.level=false; set hive.exec.dynamic.partition=true; -set hive.optimize.sort.dynamic.partition=false; set mapred.min.split.size=1000; set mapred.max.split.size=2000; set tez.grouping.min-size=1000; diff --git a/ql/src/test/queries/clientpositive/orc_merge_incompat2.q b/ql/src/test/queries/clientpositive/orc_merge_incompat2.q index e67eca3e69..03460fb319 100644 --- a/ql/src/test/queries/clientpositive/orc_merge_incompat2.q +++ b/ql/src/test/queries/clientpositive/orc_merge_incompat2.q @@ -23,7 +23,6 @@ set hive.compute.splits.in.am=true; set tez.am.grouping.min-size=1000; set tez.am.grouping.max-size=50000; set hive.exec.dynamic.partition=true; -set hive.optimize.sort.dynamic.partition=false; set hive.merge.sparkfiles=false; explain insert overwrite table orc_merge5a partition (st) select userid,string1,subtype,decimal1,ts,subtype from orc_merge5; diff --git a/ql/src/test/queries/clientpositive/pcr.q b/ql/src/test/queries/clientpositive/pcr.q index 70ea05ba99..b3e6b26d40 100644 --- a/ql/src/test/queries/clientpositive/pcr.q +++ b/ql/src/test/queries/clientpositive/pcr.q @@ -5,11 +5,11 @@ drop table pcr_t1; drop table pcr_t2; drop table pcr_t3; -create table pcr_t1 (key int, value string) partitioned by (ds string); +create table pcr_t1 (key int, value string) partitioned by (ds string) stored as orc TBLPROPERTIES ('transactional'='true'); -insert overwrite table pcr_t1 partition (ds='2000-04-08') select * from src where key < 20 order by key; -insert overwrite table pcr_t1 partition (ds='2000-04-09') select * from src where key < 20 order by key; -insert overwrite table pcr_t1 partition (ds='2000-04-10') select * from src where key < 20 order by key; +insert overwrite table pcr_t1 partition (ds='2000-04-08') select * from qtest.src where key < 20 order by key; +insert overwrite table pcr_t1 partition (ds='2000-04-09') select * from qtest.src where key < 20 order by key; +insert overwrite table pcr_t1 partition (ds='2000-04-10') select * from qtest.src where key < 20 order by key; explain extended select key, value, ds from pcr_t1 where ds<='2000-04-09' and key<5 order by key, ds; select key, value, ds from pcr_t1 where ds<='2000-04-09' and key<5 order by key, ds; diff --git a/ql/src/test/queries/clientpositive/schema_evol_orc_acid_part_update.q b/ql/src/test/queries/clientpositive/schema_evol_orc_acid_part_update.q index 1b6dab890b..fc6d36cf04 100644 --- a/ql/src/test/queries/clientpositive/schema_evol_orc_acid_part_update.q +++ b/ql/src/test/queries/clientpositive/schema_evol_orc_acid_part_update.q @@ -125,7 +125,6 @@ create table missing_ddl_2_n0(name string, age int); insert overwrite table missing_ddl_2_n0 select value, key from srcbucket; alter table missing_ddl_2_n0 add columns (gps double); -set hive.optimize.sort.dynamic.partition=true; DROP TABLE IF EXISTS all100kjson_textfile_orc_n0; CREATE TABLE all100kjson_textfile_orc_n0 ( diff --git a/ql/src/test/queries/clientpositive/schema_evol_orc_acid_part_update_llap_io.q b/ql/src/test/queries/clientpositive/schema_evol_orc_acid_part_update_llap_io.q index 732ea5765b..198c264bfc 100644 --- a/ql/src/test/queries/clientpositive/schema_evol_orc_acid_part_update_llap_io.q +++ b/ql/src/test/queries/clientpositive/schema_evol_orc_acid_part_update_llap_io.q @@ -126,7 +126,6 @@ create table missing_ddl_2(name string, age int); insert overwrite table missing_ddl_2 select value, key from srcbucket; alter table missing_ddl_2 add columns (gps double); -set hive.optimize.sort.dynamic.partition=true; DROP TABLE IF EXISTS all100kjson_textfile_orc; CREATE TABLE all100kjson_textfile_orc ( diff --git a/ql/src/test/queries/clientpositive/temp_table_merge_dynamic_partition.q b/ql/src/test/queries/clientpositive/temp_table_merge_dynamic_partition.q index c492dcffab..fbe36630c9 100644 --- a/ql/src/test/queries/clientpositive/temp_table_merge_dynamic_partition.q +++ b/ql/src/test/queries/clientpositive/temp_table_merge_dynamic_partition.q @@ -21,7 +21,6 @@ set hive.input.format=org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat; set hive.merge.mapfiles=false; set hive.merge.mapredfiles=false; set hive.merge.smallfiles.avgsize=1000000000; -set hive.optimize.sort.dynamic.partition=false; explain insert overwrite table merge_dynamic_part_n1_temp partition (ds='2008-04-08', hr) select key, value, hr from srcpart_merge_dp_n1_temp where ds='2008-04-08'; insert overwrite table merge_dynamic_part_n1_temp partition (ds='2008-04-08', hr) select key, value, hr from srcpart_merge_dp_n1_temp where ds='2008-04-08'; diff --git a/ql/src/test/queries/clientpositive/temp_table_merge_dynamic_partition2.q b/ql/src/test/queries/clientpositive/temp_table_merge_dynamic_partition2.q index 526669e521..c2ace4b5ea 100644 --- a/ql/src/test/queries/clientpositive/temp_table_merge_dynamic_partition2.q +++ b/ql/src/test/queries/clientpositive/temp_table_merge_dynamic_partition2.q @@ -22,7 +22,6 @@ set hive.merge.mapfiles=true; set hive.merge.mapredfiles=true; set hive.merge.smallfiles.avgsize=3000; set hive.exec.compress.output=false; -set hive.optimize.sort.dynamic.partition=false; explain insert overwrite table merge_dynamic_part_n0_temp partition (ds='2008-04-08', hr) select key, value, hr from srcpart_merge_dp_n0_temp where ds='2008-04-08'; diff --git a/ql/src/test/queries/clientpositive/vector_outer_join_constants.q b/ql/src/test/queries/clientpositive/vector_outer_join_constants.q index 253bb92a5e..0431f0ea80 100644 --- a/ql/src/test/queries/clientpositive/vector_outer_join_constants.q +++ b/ql/src/test/queries/clientpositive/vector_outer_join_constants.q @@ -11,10 +11,8 @@ set hive.optimize.constant.propagation=true; set hive.optimize.index.filter=true; set hive.optimize.bucketmapjoin=true; set hive.limit.optimize.enable=true; -set hive.optimize.sort.dynamic.partition=false; set hive.optimize.bucketmapjoin.sortedmerge=false; set hive.optimize.reducededuplication.min.reducer=1; -set hive.optimize.sort.dynamic.partition=false; set hive.vectorized.execution.reduce.enabled=true; set hive.auto.convert.join=true; set hive.vectorized.execution.mapjoin.native.enabled=true;