diff --git a/ql/src/test/queries/clientpositive/exim_hidden_files.q b/ql/src/test/queries/clientpositive/exim_hidden_files.q index f046411266..9375535507 100644 --- a/ql/src/test/queries/clientpositive/exim_hidden_files.q +++ b/ql/src/test/queries/clientpositive/exim_hidden_files.q @@ -6,18 +6,18 @@ set hive.test.mode.nosamplelist=exim_department,exim_employee_n6; create table exim_employee_n6 ( emp_id int) partitioned by (emp_country string); load data local inpath "../../data/files/test.dat" into table exim_employee_n6 partition (emp_country="in"); -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/exim_employee/emp_country=in/_logs; -dfs -touchz ${system:test.warehouse.dir}/exim_employee/emp_country=in/_logs/job.xml; -export table exim_employee_n6 to 'ql/test/data/exports/exim_employee'; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/exim_employee_n6/emp_country=in/_logs; +dfs -touchz ${system:test.warehouse.dir}/exim_employee_n6/emp_country=in/_logs/job.xml; +export table exim_employee_n6 to 'ql/test/data/exports/exim_employee_n6'; drop table exim_employee_n6; create database importer; use importer; -import from 'ql/test/data/exports/exim_employee'; +import from 'ql/test/data/exports/exim_employee_n6'; describe formatted exim_employee_n6; select * from exim_employee_n6; -dfs -rmr target/tmp/ql/test/data/exports/exim_employee; +dfs -rmr target/tmp/ql/test/data/exports/exim_employee_n6; drop table exim_employee_n6; drop database importer; use default; diff --git a/ql/src/test/queries/clientpositive/msck_repair_0.q b/ql/src/test/queries/clientpositive/msck_repair_0.q index aeb4820af5..17168e26b7 100644 --- a/ql/src/test/queries/clientpositive/msck_repair_0.q +++ b/ql/src/test/queries/clientpositive/msck_repair_0.q @@ -9,8 +9,8 @@ MSCK TABLE repairtable_n5; show partitions repairtable_n5; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=c/p2=a/p3=b; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=c/p2=a/p3=b/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n5/p1=c/p2=a/p3=b; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n5/p1=c/p2=a/p3=b/datafile; MSCK TABLE default.repairtable_n5; @@ -26,8 +26,8 @@ show partitions repairtable_n5; set hive.mapred.mode=strict; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=e/p2=f/p3=g; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=e/p2=f/p3=g/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n5/p1=e/p2=f/p3=g; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n5/p1=e/p2=f/p3=g/datafile; MSCK REPAIR TABLE default.repairtable_n5; diff --git a/ql/src/test/queries/clientpositive/msck_repair_2.q b/ql/src/test/queries/clientpositive/msck_repair_2.q index be745b2d60..2a0987f8a5 100644 --- a/ql/src/test/queries/clientpositive/msck_repair_2.q +++ b/ql/src/test/queries/clientpositive/msck_repair_2.q @@ -9,9 +9,9 @@ MSCK TABLE repairtable_n2; show partitions repairtable_n2; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=c/p2=a/p3=b; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=c/p2=a/p3=b/datafile; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=c/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n2/p1=c/p2=a/p3=b; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n2/p1=c/p2=a/p3=b/datafile; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n2/p1=c/datafile; MSCK TABLE default.repairtable_n2; show partitions repairtable_n2; diff --git a/ql/src/test/queries/clientpositive/msck_repair_3.q b/ql/src/test/queries/clientpositive/msck_repair_3.q index 140a6904dd..2e01f690e4 100644 --- a/ql/src/test/queries/clientpositive/msck_repair_3.q +++ b/ql/src/test/queries/clientpositive/msck_repair_3.q @@ -7,7 +7,7 @@ CREATE TABLE repairtable_n3(col STRING) PARTITIONED BY (p1 STRING, p2 STRING); MSCK TABLE repairtable_n3; show partitions repairtable_n3; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=c/p2=a/p3=b; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n3/p1=c/p2=a/p3=b; MSCK TABLE default.repairtable_n3; show partitions repairtable_n3; diff --git a/ql/src/test/queries/clientpositive/msck_repair_batchsize.q b/ql/src/test/queries/clientpositive/msck_repair_batchsize.q index 5a7afcca5b..10ffc8a12f 100644 --- a/ql/src/test/queries/clientpositive/msck_repair_batchsize.q +++ b/ql/src/test/queries/clientpositive/msck_repair_batchsize.q @@ -6,12 +6,12 @@ CREATE TABLE repairtable_n0(col STRING) PARTITIONED BY (p1 STRING, p2 STRING); MSCK TABLE repairtable_n0; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=a/p2=a; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=b/p2=a; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=c/p2=a; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=a/p2=a/datafile; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=b/p2=a/datafile; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=c/p2=a/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n0/p1=a/p2=a; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n0/p1=b/p2=a; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n0/p1=c/p2=a; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n0/p1=a/p2=a/datafile; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n0/p1=b/p2=a/datafile; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n0/p1=c/p2=a/datafile; MSCK TABLE default.repairtable_n0; show partitions default.repairtable_n0; @@ -25,10 +25,10 @@ show partitions repairtable_n0; DROP TABLE default.repairtable_n0; -dfs ${system:test.dfs.mkdir} -p ${system:test.tmp.dir}/apps/hive/warehouse/test.db/repairtable/p1=c/p2=a/p3=b; -CREATE TABLE `repairtable_n0`( `col` string) PARTITIONED BY ( `p1` string, `p2` string) location '${system:test.tmp.dir}/apps/hive/warehouse/test.db/repairtable/'; +dfs ${system:test.dfs.mkdir} -p ${system:test.tmp.dir}/apps/hive/warehouse/test.db/repairtable_n0/p1=c/p2=a/p3=b; +CREATE TABLE `repairtable_n0`( `col` string) PARTITIONED BY ( `p1` string, `p2` string) location '${system:test.tmp.dir}/apps/hive/warehouse/test.db/repairtable_n0/'; -dfs -touchz ${system:test.tmp.dir}/apps/hive/warehouse/test.db/repairtable/p1=c/p2=a/p3=b/datafile; +dfs -touchz ${system:test.tmp.dir}/apps/hive/warehouse/test.db/repairtable_n0/p1=c/p2=a/p3=b/datafile; set hive.mv.files.thread=1; MSCK TABLE repairtable_n0; show partitions repairtable_n0; diff --git a/ql/src/test/queries/clientpositive/msck_repair_drop.q b/ql/src/test/queries/clientpositive/msck_repair_drop.q index 9923fb50cb..407249e55a 100644 --- a/ql/src/test/queries/clientpositive/msck_repair_drop.q +++ b/ql/src/test/queries/clientpositive/msck_repair_drop.q @@ -11,31 +11,31 @@ CREATE TABLE repairtable_n1(col STRING) PARTITIONED BY (p1 STRING, p2 STRING); -- the same set of 10 partitions will be created between each drop attempts -- p1=3, p1=4 and p1=5 will be used to test keywords add, drop and sync -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=1/p2=11/p3=111; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=1/p2=11/p3=111/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=1/p2=12/p3=121; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=1/p2=12/p3=121/datafile; - -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=21/p3=211; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=21/p3=211/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=22/p3=221; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=22/p3=221/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=23/p3=231; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=23/p3=231/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=24/p3=241; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=24/p3=241/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=25/p3=251; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=25/p3=251/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=26/p3=261; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=26/p3=261/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=27/p3=271; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=27/p3=271/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=28/p3=281; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=28/p3=281/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=29/p3=291; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=29/p3=291/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=210/p3=2101; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=210/p3=2101/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=1/p2=11/p3=111; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=1/p2=11/p3=111/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=1/p2=12/p3=121; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=1/p2=12/p3=121/datafile; + +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=21/p3=211; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=21/p3=211/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=22/p3=221; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=22/p3=221/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=23/p3=231; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=23/p3=231/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=24/p3=241; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=24/p3=241/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=25/p3=251; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=25/p3=251/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=26/p3=261; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=26/p3=261/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=27/p3=271; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=27/p3=271/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=28/p3=281; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=28/p3=281/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=29/p3=291; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=29/p3=291/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=210/p3=2101; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=210/p3=2101/datafile; MSCK TABLE default.repairtable_n1; MSCK REPAIR TABLE default.repairtable_n1; @@ -44,7 +44,7 @@ MSCK REPAIR TABLE default.repairtable_n1; show partitions default.repairtable_n1; -- Remove all p1=2 partitions from file system -dfs -rmr ${system:test.warehouse.dir}/repairtable/p1=2; +dfs -rmr ${system:test.warehouse.dir}/repairtable_n1/p1=2; -- test 1: each partition is dropped individually set hive.msck.repair.batch.size=1; @@ -53,26 +53,26 @@ MSCK REPAIR TABLE default.repairtable_n1 DROP PARTITIONS; show partitions default.repairtable_n1; -- Recreate p1=2 partitions -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=21/p3=211; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=21/p3=211/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=22/p3=221; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=22/p3=221/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=23/p3=231; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=23/p3=231/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=24/p3=241; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=24/p3=241/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=25/p3=251; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=25/p3=251/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=26/p3=261; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=26/p3=261/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=27/p3=271; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=27/p3=271/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=28/p3=281; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=28/p3=281/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=29/p3=291; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=29/p3=291/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=210/p3=2101; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=210/p3=2101/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=21/p3=211; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=21/p3=211/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=22/p3=221; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=22/p3=221/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=23/p3=231; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=23/p3=231/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=24/p3=241; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=24/p3=241/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=25/p3=251; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=25/p3=251/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=26/p3=261; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=26/p3=261/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=27/p3=271; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=27/p3=271/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=28/p3=281; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=28/p3=281/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=29/p3=291; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=29/p3=291/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=210/p3=2101; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=210/p3=2101/datafile; MSCK TABLE default.repairtable_n1; MSCK REPAIR TABLE default.repairtable_n1; @@ -81,7 +81,7 @@ MSCK REPAIR TABLE default.repairtable_n1; show partitions default.repairtable_n1; -- Remove all p1=2 partitions from file system -dfs -rmr ${system:test.warehouse.dir}/repairtable/p1=2; +dfs -rmr ${system:test.warehouse.dir}/repairtable_n1/p1=2; -- test 2: partition are dropped in groups of 3 set hive.msck.repair.batch.size=3; @@ -90,26 +90,26 @@ MSCK REPAIR TABLE default.repairtable_n1 DROP PARTITIONS; show partitions default.repairtable_n1; -- Recreate p1=2 partitions -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=21/p3=211; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=21/p3=211/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=22/p3=221; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=22/p3=221/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=23/p3=231; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=23/p3=231/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=24/p3=241; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=24/p3=241/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=25/p3=251; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=25/p3=251/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=26/p3=261; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=26/p3=261/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=27/p3=271; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=27/p3=271/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=28/p3=281; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=28/p3=281/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=29/p3=291; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=29/p3=291/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=2/p2=210/p3=2101; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=2/p2=210/p3=2101/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=21/p3=211; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=21/p3=211/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=22/p3=221; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=22/p3=221/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=23/p3=231; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=23/p3=231/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=24/p3=241; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=24/p3=241/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=25/p3=251; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=25/p3=251/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=26/p3=261; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=26/p3=261/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=27/p3=271; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=27/p3=271/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=28/p3=281; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=28/p3=281/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=29/p3=291; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=29/p3=291/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=210/p3=2101; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=2/p2=210/p3=2101/datafile; MSCK TABLE default.repairtable_n1; MSCK REPAIR TABLE default.repairtable_n1; @@ -118,7 +118,7 @@ MSCK REPAIR TABLE default.repairtable_n1; show partitions default.repairtable_n1; -- Remove all p1=2 partitions from file system -dfs -rmr ${system:test.warehouse.dir}/repairtable/p1=2; +dfs -rmr ${system:test.warehouse.dir}/repairtable_n1/p1=2; -- test 3. all partitions are dropped in 1 shot set hive.msck.repair.batch.size=0; @@ -127,23 +127,23 @@ MSCK REPAIR TABLE default.repairtable_n1 DROP PARTITIONS; show partitions default.repairtable_n1; -- test add parition keyword: begin -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=3/p2=31/p3=311; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=3/p2=31/p3=311/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=3/p2=32/p3=321; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=3/p2=32/p3=321/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=3/p2=31/p3=311; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=3/p2=31/p3=311/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=3/p2=32/p3=321; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=3/p2=32/p3=321/datafile; MSCK TABLE default.repairtable_n1; MSCK REPAIR TABLE default.repairtable_n1; show partitions default.repairtable_n1; -- Create p1=4 in filesystem -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=4/p2=41/p3=411; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=4/p2=41/p3=411/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=4/p2=42/p3=421; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=4/p2=42/p3=421/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=4/p2=41/p3=411; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=4/p2=41/p3=411/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=4/p2=42/p3=421; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=4/p2=42/p3=421/datafile; -- Remove p1=3 from filesystem -dfs -rmr ${system:test.warehouse.dir}/repairtable/p1=3; +dfs -rmr ${system:test.warehouse.dir}/repairtable_n1/p1=3; -- Status: p1=3 dropped from filesystem, but exists in metastore -- p1=4 exists in filesystem but not in metastore @@ -154,10 +154,10 @@ show partitions default.repairtable_n1; -- test add partition keyword: end -- test drop partition keyword: begin -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=5/p2=51/p3=511; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=5/p2=51/p3=511/datafile; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=5/p2=52/p3=521; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=5/p2=52/p3=521/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=5/p2=51/p3=511; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=5/p2=51/p3=511/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n1/p1=5/p2=52/p3=521; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n1/p1=5/p2=52/p3=521/datafile; -- Status: p1=3 removed from filesystem, but exists in metastore (as part of add test) -- p1=5 exists in filesystem but not in metastore @@ -169,7 +169,7 @@ show partitions default.repairtable_n1; -- test sync partition keyword: begin -- Remove p1=4 from filesystem -dfs -rmr ${system:test.warehouse.dir}/repairtable/p1=4; +dfs -rmr ${system:test.warehouse.dir}/repairtable_n1/p1=4; -- Status: p1=4 dropped from filesystem, but exists in metastore -- p1=5 exists in filesystem but not in metastore (as part of drop test) diff --git a/ql/src/test/queries/clientpositive/repair.q b/ql/src/test/queries/clientpositive/repair.q index d48417f914..27ae8d1643 100644 --- a/ql/src/test/queries/clientpositive/repair.q +++ b/ql/src/test/queries/clientpositive/repair.q @@ -4,9 +4,9 @@ CREATE TABLE repairtable_n4(col STRING) PARTITIONED BY (p1 STRING, p2 STRING); MSCK TABLE repairtable_n4; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=a/p2=a; -dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=b/p2=a; -dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=b/p2=a/datafile; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n4/p1=a/p2=a; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n4/p1=b/p2=a; +dfs -touchz ${system:test.warehouse.dir}/repairtable_n4/p1=b/p2=a/datafile; MSCK TABLE default.repairtable_n4; diff --git a/ql/src/test/results/clientpositive/exim_hidden_files.q.out b/ql/src/test/results/clientpositive/exim_hidden_files.q.out index 659b36d6de..a1f1948d73 100644 --- a/ql/src/test/results/clientpositive/exim_hidden_files.q.out +++ b/ql/src/test/results/clientpositive/exim_hidden_files.q.out @@ -15,11 +15,11 @@ POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@exim_employee_n6 POSTHOOK: Output: default@exim_employee_n6@emp_country=in -PREHOOK: query: export table exim_employee_n6 to 'ql/test/data/exports/exim_employee' +PREHOOK: query: export table exim_employee_n6 to 'ql/test/data/exports/exim_employee_n6' PREHOOK: type: EXPORT PREHOOK: Input: default@exim_employee_n6@emp_country=in #### A masked pattern was here #### -POSTHOOK: query: export table exim_employee_n6 to 'ql/test/data/exports/exim_employee' +POSTHOOK: query: export table exim_employee_n6 to 'ql/test/data/exports/exim_employee_n6' POSTHOOK: type: EXPORT POSTHOOK: Input: default@exim_employee_n6@emp_country=in #### A masked pattern was here #### @@ -43,11 +43,11 @@ PREHOOK: Input: database:importer POSTHOOK: query: use importer POSTHOOK: type: SWITCHDATABASE POSTHOOK: Input: database:importer -PREHOOK: query: import from 'ql/test/data/exports/exim_employee' +PREHOOK: query: import from 'ql/test/data/exports/exim_employee_n6' PREHOOK: type: IMPORT #### A masked pattern was here #### PREHOOK: Output: database:importer -POSTHOOK: query: import from 'ql/test/data/exports/exim_employee' +POSTHOOK: query: import from 'ql/test/data/exports/exim_employee_n6' POSTHOOK: type: IMPORT #### A masked pattern was here #### POSTHOOK: Output: database:importer diff --git a/ql/src/test/results/clientpositive/msck_repair_0.q.out b/ql/src/test/results/clientpositive/msck_repair_0.q.out index fa6e4a9882..94da7c3aaf 100644 --- a/ql/src/test/results/clientpositive/msck_repair_0.q.out +++ b/ql/src/test/results/clientpositive/msck_repair_0.q.out @@ -28,6 +28,7 @@ PREHOOK: Output: default@repairtable_n5 POSTHOOK: query: MSCK TABLE default.repairtable_n5 POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n5 +Partitions not in metastore: repairtable_n5:p1=c/p2=a PREHOOK: query: show partitions default.repairtable_n5 PREHOOK: type: SHOWPARTITIONS PREHOOK: Input: default@repairtable_n5 @@ -40,12 +41,15 @@ PREHOOK: Output: default@repairtable_n5 POSTHOOK: query: MSCK REPAIR TABLE default.repairtable_n5 POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n5 +Partitions not in metastore: repairtable_n5:p1=c/p2=a +#### A masked pattern was here #### PREHOOK: query: show partitions default.repairtable_n5 PREHOOK: type: SHOWPARTITIONS PREHOOK: Input: default@repairtable_n5 POSTHOOK: query: show partitions default.repairtable_n5 POSTHOOK: type: SHOWPARTITIONS POSTHOOK: Input: default@repairtable_n5 +p1=c/p2=a PREHOOK: query: MSCK TABLE repairtable_n5 PREHOOK: type: MSCK PREHOOK: Output: default@repairtable_n5 @@ -58,18 +62,23 @@ PREHOOK: Input: default@repairtable_n5 POSTHOOK: query: show partitions repairtable_n5 POSTHOOK: type: SHOWPARTITIONS POSTHOOK: Input: default@repairtable_n5 +p1=c/p2=a PREHOOK: query: MSCK REPAIR TABLE default.repairtable_n5 PREHOOK: type: MSCK PREHOOK: Output: default@repairtable_n5 POSTHOOK: query: MSCK REPAIR TABLE default.repairtable_n5 POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n5 +Partitions not in metastore: repairtable_n5:p1=e/p2=f +#### A masked pattern was here #### PREHOOK: query: show partitions default.repairtable_n5 PREHOOK: type: SHOWPARTITIONS PREHOOK: Input: default@repairtable_n5 POSTHOOK: query: show partitions default.repairtable_n5 POSTHOOK: type: SHOWPARTITIONS POSTHOOK: Input: default@repairtable_n5 +p1=c/p2=a +p1=e/p2=f PREHOOK: query: DROP TABLE default.repairtable_n5 PREHOOK: type: DROPTABLE PREHOOK: Input: default@repairtable_n5 diff --git a/ql/src/test/results/clientpositive/msck_repair_2.q.out b/ql/src/test/results/clientpositive/msck_repair_2.q.out index 7fbd934e11..90f77b7cde 100644 --- a/ql/src/test/results/clientpositive/msck_repair_2.q.out +++ b/ql/src/test/results/clientpositive/msck_repair_2.q.out @@ -28,6 +28,7 @@ PREHOOK: Output: default@repairtable_n2 POSTHOOK: query: MSCK TABLE default.repairtable_n2 POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n2 +Partitions not in metastore: repairtable_n2:p1=c/p2=a PREHOOK: query: show partitions repairtable_n2 PREHOOK: type: SHOWPARTITIONS PREHOOK: Input: default@repairtable_n2 @@ -40,12 +41,15 @@ PREHOOK: Output: default@repairtable_n2 POSTHOOK: query: MSCK REPAIR TABLE default.repairtable_n2 POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n2 +Partitions not in metastore: repairtable_n2:p1=c/p2=a +#### A masked pattern was here #### PREHOOK: query: show partitions repairtable_n2 PREHOOK: type: SHOWPARTITIONS PREHOOK: Input: default@repairtable_n2 POSTHOOK: query: show partitions repairtable_n2 POSTHOOK: type: SHOWPARTITIONS POSTHOOK: Input: default@repairtable_n2 +p1=c/p2=a PREHOOK: query: MSCK TABLE repairtable_n2 PREHOOK: type: MSCK PREHOOK: Output: default@repairtable_n2 @@ -58,6 +62,7 @@ PREHOOK: Input: default@repairtable_n2 POSTHOOK: query: show partitions repairtable_n2 POSTHOOK: type: SHOWPARTITIONS POSTHOOK: Input: default@repairtable_n2 +p1=c/p2=a PREHOOK: query: DROP TABLE default.repairtable_n2 PREHOOK: type: DROPTABLE PREHOOK: Input: default@repairtable_n2 diff --git a/ql/src/test/results/clientpositive/msck_repair_3.q.out b/ql/src/test/results/clientpositive/msck_repair_3.q.out index 0e153fbe69..c18da6f437 100644 --- a/ql/src/test/results/clientpositive/msck_repair_3.q.out +++ b/ql/src/test/results/clientpositive/msck_repair_3.q.out @@ -28,6 +28,7 @@ PREHOOK: Output: default@repairtable_n3 POSTHOOK: query: MSCK TABLE default.repairtable_n3 POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n3 +Partitions not in metastore: repairtable_n3:p1=c/p2=a PREHOOK: query: show partitions repairtable_n3 PREHOOK: type: SHOWPARTITIONS PREHOOK: Input: default@repairtable_n3 @@ -40,12 +41,15 @@ PREHOOK: Output: default@repairtable_n3 POSTHOOK: query: MSCK REPAIR TABLE default.repairtable_n3 POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n3 +Partitions not in metastore: repairtable_n3:p1=c/p2=a +#### A masked pattern was here #### PREHOOK: query: show partitions repairtable_n3 PREHOOK: type: SHOWPARTITIONS PREHOOK: Input: default@repairtable_n3 POSTHOOK: query: show partitions repairtable_n3 POSTHOOK: type: SHOWPARTITIONS POSTHOOK: Input: default@repairtable_n3 +p1=c/p2=a PREHOOK: query: MSCK TABLE repairtable_n3 PREHOOK: type: MSCK PREHOOK: Output: default@repairtable_n3 @@ -58,6 +62,7 @@ PREHOOK: Input: default@repairtable_n3 POSTHOOK: query: show partitions repairtable_n3 POSTHOOK: type: SHOWPARTITIONS POSTHOOK: Input: default@repairtable_n3 +p1=c/p2=a PREHOOK: query: DROP TABLE default.repairtable_n3 PREHOOK: type: DROPTABLE PREHOOK: Input: default@repairtable_n3 diff --git a/ql/src/test/results/clientpositive/msck_repair_batchsize.q.out b/ql/src/test/results/clientpositive/msck_repair_batchsize.q.out index ab4b83137d..bedfac7d28 100644 --- a/ql/src/test/results/clientpositive/msck_repair_batchsize.q.out +++ b/ql/src/test/results/clientpositive/msck_repair_batchsize.q.out @@ -22,6 +22,7 @@ PREHOOK: Output: default@repairtable_n0 POSTHOOK: query: MSCK TABLE default.repairtable_n0 POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n0 +Partitions not in metastore: repairtable_n0:p1=a/p2=a repairtable_n0:p1=b/p2=a repairtable_n0:p1=c/p2=a PREHOOK: query: show partitions default.repairtable_n0 PREHOOK: type: SHOWPARTITIONS PREHOOK: Input: default@repairtable_n0 @@ -34,12 +35,17 @@ PREHOOK: Output: default@repairtable_n0 POSTHOOK: query: MSCK REPAIR TABLE default.repairtable_n0 POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n0 +Partitions not in metastore: repairtable_n0:p1=a/p2=a repairtable_n0:p1=b/p2=a repairtable_n0:p1=c/p2=a +#### A masked pattern was here #### PREHOOK: query: show partitions default.repairtable_n0 PREHOOK: type: SHOWPARTITIONS PREHOOK: Input: default@repairtable_n0 POSTHOOK: query: show partitions default.repairtable_n0 POSTHOOK: type: SHOWPARTITIONS POSTHOOK: Input: default@repairtable_n0 +p1=a/p2=a +p1=b/p2=a +p1=c/p2=a PREHOOK: query: MSCK TABLE repairtable_n0 PREHOOK: type: MSCK PREHOOK: Output: default@repairtable_n0 @@ -52,6 +58,9 @@ PREHOOK: Input: default@repairtable_n0 POSTHOOK: query: show partitions repairtable_n0 POSTHOOK: type: SHOWPARTITIONS POSTHOOK: Input: default@repairtable_n0 +p1=a/p2=a +p1=b/p2=a +p1=c/p2=a PREHOOK: query: DROP TABLE default.repairtable_n0 PREHOOK: type: DROPTABLE PREHOOK: Input: default@repairtable_n0 diff --git a/ql/src/test/results/clientpositive/msck_repair_drop.q.out b/ql/src/test/results/clientpositive/msck_repair_drop.q.out index 971c138127..683dc82da4 100644 --- a/ql/src/test/results/clientpositive/msck_repair_drop.q.out +++ b/ql/src/test/results/clientpositive/msck_repair_drop.q.out @@ -16,18 +16,33 @@ PREHOOK: Output: default@repairtable_n1 POSTHOOK: query: MSCK TABLE default.repairtable_n1 POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n1 +Partitions not in metastore: repairtable_n1:p1=1/p2=11 repairtable_n1:p1=1/p2=12 repairtable_n1:p1=2/p2=21 repairtable_n1:p1=2/p2=210 repairtable_n1:p1=2/p2=22 repairtable_n1:p1=2/p2=23 repairtable_n1:p1=2/p2=24 repairtable_n1:p1=2/p2=25 repairtable_n1:p1=2/p2=26 repairtable_n1:p1=2/p2=27 repairtable_n1:p1=2/p2=28 repairtable_n1:p1=2/p2=29 PREHOOK: query: MSCK REPAIR TABLE default.repairtable_n1 PREHOOK: type: MSCK PREHOOK: Output: default@repairtable_n1 POSTHOOK: query: MSCK REPAIR TABLE default.repairtable_n1 POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n1 +Partitions not in metastore: repairtable_n1:p1=1/p2=11 repairtable_n1:p1=1/p2=12 repairtable_n1:p1=2/p2=21 repairtable_n1:p1=2/p2=210 repairtable_n1:p1=2/p2=22 repairtable_n1:p1=2/p2=23 repairtable_n1:p1=2/p2=24 repairtable_n1:p1=2/p2=25 repairtable_n1:p1=2/p2=26 repairtable_n1:p1=2/p2=27 repairtable_n1:p1=2/p2=28 repairtable_n1:p1=2/p2=29 +#### A masked pattern was here #### PREHOOK: query: show partitions default.repairtable_n1 PREHOOK: type: SHOWPARTITIONS PREHOOK: Input: default@repairtable_n1 POSTHOOK: query: show partitions default.repairtable_n1 POSTHOOK: type: SHOWPARTITIONS POSTHOOK: Input: default@repairtable_n1 +p1=1/p2=11 +p1=1/p2=12 +p1=2/p2=21 +p1=2/p2=210 +p1=2/p2=22 +p1=2/p2=23 +p1=2/p2=24 +p1=2/p2=25 +p1=2/p2=26 +p1=2/p2=27 +p1=2/p2=28 +p1=2/p2=29 #### A masked pattern was here #### PREHOOK: query: MSCK TABLE default.repairtable_n1 DROP PARTITIONS PREHOOK: type: MSCK @@ -35,36 +50,65 @@ PREHOOK: Output: default@repairtable_n1 POSTHOOK: query: MSCK TABLE default.repairtable_n1 DROP PARTITIONS POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n1 +Partitions missing from filesystem: repairtable_n1:p1=2/p2=21 repairtable_n1:p1=2/p2=210 repairtable_n1:p1=2/p2=22 repairtable_n1:p1=2/p2=23 repairtable_n1:p1=2/p2=24 repairtable_n1:p1=2/p2=25 repairtable_n1:p1=2/p2=26 repairtable_n1:p1=2/p2=27 repairtable_n1:p1=2/p2=28 repairtable_n1:p1=2/p2=29 PREHOOK: query: MSCK REPAIR TABLE default.repairtable_n1 DROP PARTITIONS PREHOOK: type: MSCK PREHOOK: Output: default@repairtable_n1 POSTHOOK: query: MSCK REPAIR TABLE default.repairtable_n1 DROP PARTITIONS POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n1 +Partitions missing from filesystem: repairtable_n1:p1=2/p2=21 repairtable_n1:p1=2/p2=210 repairtable_n1:p1=2/p2=22 repairtable_n1:p1=2/p2=23 repairtable_n1:p1=2/p2=24 repairtable_n1:p1=2/p2=25 repairtable_n1:p1=2/p2=26 repairtable_n1:p1=2/p2=27 repairtable_n1:p1=2/p2=28 repairtable_n1:p1=2/p2=29 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=210 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=24 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=29 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=22 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=28 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=27 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=23 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=25 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=21 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=26 PREHOOK: query: show partitions default.repairtable_n1 PREHOOK: type: SHOWPARTITIONS PREHOOK: Input: default@repairtable_n1 POSTHOOK: query: show partitions default.repairtable_n1 POSTHOOK: type: SHOWPARTITIONS POSTHOOK: Input: default@repairtable_n1 +p1=1/p2=11 +p1=1/p2=12 PREHOOK: query: MSCK TABLE default.repairtable_n1 PREHOOK: type: MSCK PREHOOK: Output: default@repairtable_n1 POSTHOOK: query: MSCK TABLE default.repairtable_n1 POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n1 +Partitions not in metastore: repairtable_n1:p1=2/p2=21 repairtable_n1:p1=2/p2=210 repairtable_n1:p1=2/p2=22 repairtable_n1:p1=2/p2=23 repairtable_n1:p1=2/p2=24 repairtable_n1:p1=2/p2=25 repairtable_n1:p1=2/p2=26 repairtable_n1:p1=2/p2=27 repairtable_n1:p1=2/p2=28 repairtable_n1:p1=2/p2=29 PREHOOK: query: MSCK REPAIR TABLE default.repairtable_n1 PREHOOK: type: MSCK PREHOOK: Output: default@repairtable_n1 POSTHOOK: query: MSCK REPAIR TABLE default.repairtable_n1 POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n1 +Partitions not in metastore: repairtable_n1:p1=2/p2=21 repairtable_n1:p1=2/p2=210 repairtable_n1:p1=2/p2=22 repairtable_n1:p1=2/p2=23 repairtable_n1:p1=2/p2=24 repairtable_n1:p1=2/p2=25 repairtable_n1:p1=2/p2=26 repairtable_n1:p1=2/p2=27 repairtable_n1:p1=2/p2=28 repairtable_n1:p1=2/p2=29 +#### A masked pattern was here #### PREHOOK: query: show partitions default.repairtable_n1 PREHOOK: type: SHOWPARTITIONS PREHOOK: Input: default@repairtable_n1 POSTHOOK: query: show partitions default.repairtable_n1 POSTHOOK: type: SHOWPARTITIONS POSTHOOK: Input: default@repairtable_n1 +p1=1/p2=11 +p1=1/p2=12 +p1=2/p2=21 +p1=2/p2=210 +p1=2/p2=22 +p1=2/p2=23 +p1=2/p2=24 +p1=2/p2=25 +p1=2/p2=26 +p1=2/p2=27 +p1=2/p2=28 +p1=2/p2=29 #### A masked pattern was here #### PREHOOK: query: MSCK TABLE default.repairtable_n1 DROP PARTITIONS PREHOOK: type: MSCK @@ -72,36 +116,65 @@ PREHOOK: Output: default@repairtable_n1 POSTHOOK: query: MSCK TABLE default.repairtable_n1 DROP PARTITIONS POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n1 +Partitions missing from filesystem: repairtable_n1:p1=2/p2=21 repairtable_n1:p1=2/p2=210 repairtable_n1:p1=2/p2=22 repairtable_n1:p1=2/p2=23 repairtable_n1:p1=2/p2=24 repairtable_n1:p1=2/p2=25 repairtable_n1:p1=2/p2=26 repairtable_n1:p1=2/p2=27 repairtable_n1:p1=2/p2=28 repairtable_n1:p1=2/p2=29 PREHOOK: query: MSCK REPAIR TABLE default.repairtable_n1 DROP PARTITIONS PREHOOK: type: MSCK PREHOOK: Output: default@repairtable_n1 POSTHOOK: query: MSCK REPAIR TABLE default.repairtable_n1 DROP PARTITIONS POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n1 +Partitions missing from filesystem: repairtable_n1:p1=2/p2=21 repairtable_n1:p1=2/p2=210 repairtable_n1:p1=2/p2=22 repairtable_n1:p1=2/p2=23 repairtable_n1:p1=2/p2=24 repairtable_n1:p1=2/p2=25 repairtable_n1:p1=2/p2=26 repairtable_n1:p1=2/p2=27 repairtable_n1:p1=2/p2=28 repairtable_n1:p1=2/p2=29 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=27 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=29 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=22 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=210 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=24 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=26 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=21 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=25 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=23 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=28 PREHOOK: query: show partitions default.repairtable_n1 PREHOOK: type: SHOWPARTITIONS PREHOOK: Input: default@repairtable_n1 POSTHOOK: query: show partitions default.repairtable_n1 POSTHOOK: type: SHOWPARTITIONS POSTHOOK: Input: default@repairtable_n1 +p1=1/p2=11 +p1=1/p2=12 PREHOOK: query: MSCK TABLE default.repairtable_n1 PREHOOK: type: MSCK PREHOOK: Output: default@repairtable_n1 POSTHOOK: query: MSCK TABLE default.repairtable_n1 POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n1 +Partitions not in metastore: repairtable_n1:p1=2/p2=21 repairtable_n1:p1=2/p2=210 repairtable_n1:p1=2/p2=22 repairtable_n1:p1=2/p2=23 repairtable_n1:p1=2/p2=24 repairtable_n1:p1=2/p2=25 repairtable_n1:p1=2/p2=26 repairtable_n1:p1=2/p2=27 repairtable_n1:p1=2/p2=28 repairtable_n1:p1=2/p2=29 PREHOOK: query: MSCK REPAIR TABLE default.repairtable_n1 PREHOOK: type: MSCK PREHOOK: Output: default@repairtable_n1 POSTHOOK: query: MSCK REPAIR TABLE default.repairtable_n1 POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n1 +Partitions not in metastore: repairtable_n1:p1=2/p2=21 repairtable_n1:p1=2/p2=210 repairtable_n1:p1=2/p2=22 repairtable_n1:p1=2/p2=23 repairtable_n1:p1=2/p2=24 repairtable_n1:p1=2/p2=25 repairtable_n1:p1=2/p2=26 repairtable_n1:p1=2/p2=27 repairtable_n1:p1=2/p2=28 repairtable_n1:p1=2/p2=29 +#### A masked pattern was here #### PREHOOK: query: show partitions default.repairtable_n1 PREHOOK: type: SHOWPARTITIONS PREHOOK: Input: default@repairtable_n1 POSTHOOK: query: show partitions default.repairtable_n1 POSTHOOK: type: SHOWPARTITIONS POSTHOOK: Input: default@repairtable_n1 +p1=1/p2=11 +p1=1/p2=12 +p1=2/p2=21 +p1=2/p2=210 +p1=2/p2=22 +p1=2/p2=23 +p1=2/p2=24 +p1=2/p2=25 +p1=2/p2=26 +p1=2/p2=27 +p1=2/p2=28 +p1=2/p2=29 #### A masked pattern was here #### PREHOOK: query: MSCK TABLE default.repairtable_n1 DROP PARTITIONS PREHOOK: type: MSCK @@ -109,36 +182,57 @@ PREHOOK: Output: default@repairtable_n1 POSTHOOK: query: MSCK TABLE default.repairtable_n1 DROP PARTITIONS POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n1 +Partitions missing from filesystem: repairtable_n1:p1=2/p2=21 repairtable_n1:p1=2/p2=210 repairtable_n1:p1=2/p2=22 repairtable_n1:p1=2/p2=23 repairtable_n1:p1=2/p2=24 repairtable_n1:p1=2/p2=25 repairtable_n1:p1=2/p2=26 repairtable_n1:p1=2/p2=27 repairtable_n1:p1=2/p2=28 repairtable_n1:p1=2/p2=29 PREHOOK: query: MSCK REPAIR TABLE default.repairtable_n1 DROP PARTITIONS PREHOOK: type: MSCK PREHOOK: Output: default@repairtable_n1 POSTHOOK: query: MSCK REPAIR TABLE default.repairtable_n1 DROP PARTITIONS POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n1 +Partitions missing from filesystem: repairtable_n1:p1=2/p2=21 repairtable_n1:p1=2/p2=210 repairtable_n1:p1=2/p2=22 repairtable_n1:p1=2/p2=23 repairtable_n1:p1=2/p2=24 repairtable_n1:p1=2/p2=25 repairtable_n1:p1=2/p2=26 repairtable_n1:p1=2/p2=27 repairtable_n1:p1=2/p2=28 repairtable_n1:p1=2/p2=29 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=28 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=210 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=29 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=21 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=23 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=27 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=25 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=22 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=24 +Repair: Dropped partition from metastore default.repairtable_n1:p1=2/p2=26 PREHOOK: query: show partitions default.repairtable_n1 PREHOOK: type: SHOWPARTITIONS PREHOOK: Input: default@repairtable_n1 POSTHOOK: query: show partitions default.repairtable_n1 POSTHOOK: type: SHOWPARTITIONS POSTHOOK: Input: default@repairtable_n1 +p1=1/p2=11 +p1=1/p2=12 PREHOOK: query: MSCK TABLE default.repairtable_n1 PREHOOK: type: MSCK PREHOOK: Output: default@repairtable_n1 POSTHOOK: query: MSCK TABLE default.repairtable_n1 POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n1 +Partitions not in metastore: repairtable_n1:p1=3/p2=31 repairtable_n1:p1=3/p2=32 PREHOOK: query: MSCK REPAIR TABLE default.repairtable_n1 PREHOOK: type: MSCK PREHOOK: Output: default@repairtable_n1 POSTHOOK: query: MSCK REPAIR TABLE default.repairtable_n1 POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n1 +Partitions not in metastore: repairtable_n1:p1=3/p2=31 repairtable_n1:p1=3/p2=32 +#### A masked pattern was here #### PREHOOK: query: show partitions default.repairtable_n1 PREHOOK: type: SHOWPARTITIONS PREHOOK: Input: default@repairtable_n1 POSTHOOK: query: show partitions default.repairtable_n1 POSTHOOK: type: SHOWPARTITIONS POSTHOOK: Input: default@repairtable_n1 +p1=1/p2=11 +p1=1/p2=12 +p1=3/p2=31 +p1=3/p2=32 #### A masked pattern was here #### PREHOOK: query: MSCK TABLE default.repairtable_n1 ADD PARTITIONS PREHOOK: type: MSCK @@ -146,36 +240,57 @@ PREHOOK: Output: default@repairtable_n1 POSTHOOK: query: MSCK TABLE default.repairtable_n1 ADD PARTITIONS POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n1 +Partitions not in metastore: repairtable_n1:p1=4/p2=41 repairtable_n1:p1=4/p2=42 +Partitions missing from filesystem: repairtable_n1:p1=3/p2=31 repairtable_n1:p1=3/p2=32 PREHOOK: query: MSCK REPAIR TABLE default.repairtable_n1 ADD PARTITIONS PREHOOK: type: MSCK PREHOOK: Output: default@repairtable_n1 POSTHOOK: query: MSCK REPAIR TABLE default.repairtable_n1 ADD PARTITIONS POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n1 +Partitions not in metastore: repairtable_n1:p1=4/p2=41 repairtable_n1:p1=4/p2=42 +Partitions missing from filesystem: repairtable_n1:p1=3/p2=31 repairtable_n1:p1=3/p2=32 +#### A masked pattern was here #### PREHOOK: query: show partitions default.repairtable_n1 PREHOOK: type: SHOWPARTITIONS PREHOOK: Input: default@repairtable_n1 POSTHOOK: query: show partitions default.repairtable_n1 POSTHOOK: type: SHOWPARTITIONS POSTHOOK: Input: default@repairtable_n1 +p1=1/p2=11 +p1=1/p2=12 +p1=3/p2=31 +p1=3/p2=32 +p1=4/p2=41 +p1=4/p2=42 PREHOOK: query: MSCK TABLE default.repairtable_n1 DROP PARTITIONS PREHOOK: type: MSCK PREHOOK: Output: default@repairtable_n1 POSTHOOK: query: MSCK TABLE default.repairtable_n1 DROP PARTITIONS POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n1 +Partitions not in metastore: repairtable_n1:p1=5/p2=51 repairtable_n1:p1=5/p2=52 +Partitions missing from filesystem: repairtable_n1:p1=3/p2=31 repairtable_n1:p1=3/p2=32 PREHOOK: query: MSCK REPAIR TABLE default.repairtable_n1 DROP PARTITIONS PREHOOK: type: MSCK PREHOOK: Output: default@repairtable_n1 POSTHOOK: query: MSCK REPAIR TABLE default.repairtable_n1 DROP PARTITIONS POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n1 +Partitions not in metastore: repairtable_n1:p1=5/p2=51 repairtable_n1:p1=5/p2=52 +Partitions missing from filesystem: repairtable_n1:p1=3/p2=31 repairtable_n1:p1=3/p2=32 +Repair: Dropped partition from metastore default.repairtable_n1:p1=3/p2=32 +Repair: Dropped partition from metastore default.repairtable_n1:p1=3/p2=31 PREHOOK: query: show partitions default.repairtable_n1 PREHOOK: type: SHOWPARTITIONS PREHOOK: Input: default@repairtable_n1 POSTHOOK: query: show partitions default.repairtable_n1 POSTHOOK: type: SHOWPARTITIONS POSTHOOK: Input: default@repairtable_n1 +p1=1/p2=11 +p1=1/p2=12 +p1=4/p2=41 +p1=4/p2=42 #### A masked pattern was here #### PREHOOK: query: MSCK TABLE default.repairtable_n1 SYNC PARTITIONS PREHOOK: type: MSCK @@ -183,15 +298,26 @@ PREHOOK: Output: default@repairtable_n1 POSTHOOK: query: MSCK TABLE default.repairtable_n1 SYNC PARTITIONS POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n1 +Partitions not in metastore: repairtable_n1:p1=5/p2=51 repairtable_n1:p1=5/p2=52 +Partitions missing from filesystem: repairtable_n1:p1=4/p2=41 repairtable_n1:p1=4/p2=42 PREHOOK: query: MSCK REPAIR TABLE default.repairtable_n1 SYNC PARTITIONS PREHOOK: type: MSCK PREHOOK: Output: default@repairtable_n1 POSTHOOK: query: MSCK REPAIR TABLE default.repairtable_n1 SYNC PARTITIONS POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n1 +Partitions not in metastore: repairtable_n1:p1=5/p2=51 repairtable_n1:p1=5/p2=52 +Partitions missing from filesystem: repairtable_n1:p1=4/p2=41 repairtable_n1:p1=4/p2=42 +#### A masked pattern was here #### +Repair: Dropped partition from metastore default.repairtable_n1:p1=4/p2=42 +Repair: Dropped partition from metastore default.repairtable_n1:p1=4/p2=41 PREHOOK: query: show partitions default.repairtable_n1 PREHOOK: type: SHOWPARTITIONS PREHOOK: Input: default@repairtable_n1 POSTHOOK: query: show partitions default.repairtable_n1 POSTHOOK: type: SHOWPARTITIONS POSTHOOK: Input: default@repairtable_n1 +p1=1/p2=11 +p1=1/p2=12 +p1=5/p2=51 +p1=5/p2=52 diff --git a/ql/src/test/results/clientpositive/repair.q.out b/ql/src/test/results/clientpositive/repair.q.out index a8dbda92fe..90252938a5 100644 --- a/ql/src/test/results/clientpositive/repair.q.out +++ b/ql/src/test/results/clientpositive/repair.q.out @@ -22,12 +22,15 @@ PREHOOK: Output: default@repairtable_n4 POSTHOOK: query: MSCK TABLE default.repairtable_n4 POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n4 +Partitions not in metastore: repairtable_n4:p1=a/p2=a repairtable_n4:p1=b/p2=a PREHOOK: query: MSCK REPAIR TABLE default.repairtable_n4 PREHOOK: type: MSCK PREHOOK: Output: default@repairtable_n4 POSTHOOK: query: MSCK REPAIR TABLE default.repairtable_n4 POSTHOOK: type: MSCK POSTHOOK: Output: default@repairtable_n4 +Partitions not in metastore: repairtable_n4:p1=a/p2=a repairtable_n4:p1=b/p2=a +#### A masked pattern was here #### PREHOOK: query: MSCK TABLE repairtable_n4 PREHOOK: type: MSCK PREHOOK: Output: default@repairtable_n4