Index: src/test/e2e/hcatalog/tests/hcat.conf =================================================================== --- src/test/e2e/hcatalog/tests/hcat.conf (revision 1341191) +++ src/test/e2e/hcatalog/tests/hcat.conf (working copy) @@ -64,7 +64,7 @@ { 'num' => 3 ,'hcat' => q\ -drop table if exists hcat_create_table_3; +drop table if exists hcat_createtable_3; create table if not exists hcat_createtable_3(name string, age int, gpa double) stored as textfile; create table if not exists hcat_createtable_3(name string, age int, gpa double) stored as textfile; describe hcat_createtable_3; @@ -72,8 +72,54 @@ ,'rc' => 0 ,'expected_out_regex' => 'name string(\s)*age int(\s)*gpa double' ,'expected_err_regex' => 'OK(.*)OK(.*)OK(.*)OK' + }, + + { + 'num' => 4 + ,'hcat' => q\ +drop table if exists hcat_createtable_4; +create table hcat_createtable_4(name string, age int, gpa double) partitioned by (grade string, section string) stored as textfile; +alter table hcat_createtable_4 add partition (grade='7',section='a'); +alter table hcat_createtable_4 add partition (grade='8',section='b'); +show partitions hcat_createtable_4; +\, + ,'rc' => 0 + ,'expected_out_regex' => 'grade=7/section=a(\s)*grade=8/section=b' + ,'expected_err_regex' => 'OK(.*)OK(.*)OK' }, + { + 'num' => 5 + ,'hcat' => q\ +drop table if exists hcat_createtable_5; +create table hcat_createtable_5(name string, age int, gpa double) stored as +inputformat 'org.apache.hadoop.hive.ql.io.RCFileInputFormat' +outputformat 'org.apache.hadoop.hive.ql.io.RCFileOutputFormat'; +describe extended hcat_createtable_5; +\, + ,'rc' => 0 + ,'expected_out_regex' => 'name string(\s)*age int(\s)*gpa double(.*(\s))*inputFormat:org.apache.hadoop.hive.ql.io.RCFileInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.RCFileOutputFormat' + ,'expected_err_regex' => 'OK(.*)OK(.*)OK' + }, + { + 'num' => 6 + ,'hcat' => q# +drop table if exists hcat_createtable_6; +create table hcat_createtable_6(a int, b array, c map) +row format delimited + fields terminated by '^' + collection items terminated by ',' + map keys terminated by ':' + lines terminated by '\\n' +stored as textfile; +describe extended hcat_createtable_6; +#, + ,'rc' => 0 + ,'expected_out_regex' => 'a int(\s)*b array(\s)*c map(.*(\s))*.*parameters:.*colelction.delim=,, mapkey.delim=:, serialization.format=|, line.delim=\n, field.delim=^' + ,'expected_err_regex' => 'OK(.*)OK(.*)OK' + }, + + ], }, # end g { @@ -104,6 +150,41 @@ ,'expected_out_regex' => 'does not exist' }, + { + 'num' => 3 + ,'hcat' => q\ +drop table if exists hcat_drop_table_4; +dfs -rmr :OUTPATH:/../drop_table_ext; +dfs -cp :INPATH:/studentnull10k/ :OUTPATH:/../drop_table_ext; +\, + }, + { + 'num' => 4 + ,'depends_on' => 'HCat_DropTable_3' + ,'hcat' => q\ +create external table hcat_drop_table_4(name string, age int, gpa double) stored as textfile location 'hdfs://:OUTPATH:/../drop_table_ext'; +describe extended hcat_drop_table_4; +\, + ,'rc' => 0 + ,'expected_out_regex' => '(.*)location:hdfs://(.*)drop_table_ext(.*)tableType:EXTERNAL_TABLE' + }, + { + 'num' => 5 + ,'depends_on' => 'HCat_DropTable_4' + ,'hcat' => q\ +drop table hcat_drop_table_4; +\, + ,'rc' => 0 + }, + { + 'num' => 6 + ,'depends_on' => 'HCat_DropTable_5' + ,'hcat' => q\ +dfs -ls :OUTPATH:/../drop_table_ext +\, + ,'rc' => 0 + ,'expected_out_regex' => '(.*(\s))*.*drop_table_ext/studentnull10k' + }, ], }, # end g { @@ -164,6 +245,121 @@ \ ,'rc' => 0 }, + { + 'num' => 7 + ,'hcat' => q\ +drop table if exists hcat_altertable_7; +create table hcat_altertable_7(name string, age int, gpa double) partitioned by (grade string, section string) stored as textfile; +alter table hcat_altertable_7 add partition (grade='7',section='a'); +show partitions hcat_altertable_7;\, + ,'rc' => 0 + ,'expected_out_regex' => 'grade=7/section=a' + }, + { + 'num' => 8 + ,'depends_on' => 'HCat_AlterTable_7' + ,'hcat' => q\ +alter table hcat_altertable_7 add partition (grade='7',section='b'); +alter table hcat_altertable_7 add partition (grade='7',section='c'); +show partitions hcat_altertable_7;\, + ,'rc' => 0 + ,'expected_out_regex' => 'grade=7/section=a(\s)*grade=7/section=b(\s)*grade=7/section=c' + }, + { + 'num' => 9 + ,'depends_on' => 'HCat_AlterTable_8' + ,'hcat' => q\ +alter table hcat_altertable_7 add if not exists partition (grade='7',section='a');\ + ,'rc' => 0 + }, + + { + 'num' => 10 + ,'depends_on' => 'HCat_AlterTable_9' + ,'hcat' => q\ +alter table hcat_altertable_7 add partition (grade='8',section='a') partition (grade='8',section='b'); +show partitions hcat_altertable_7;\, + ,'rc' => 0 + ,'expected_out_regex' => 'grade=7/section=a(\s)*grade=7/section=b(\s)*grade=7/section=c(\s)*grade=8/section=a(\s)*grade=8/section=b' + }, + { + 'num' => 11 + ,'depends_on' => 'HCat_AlterTable_10' + ,'hcat' => q\ +alter table hcat_altertable_7 drop partition (grade='8',section='a'), partition (grade='8',section='b'); +show partitions hcat_altertable_7;\, + ,'rc' => 0 + ,'expected_out_regex' => 'grade=7/section=a(\s)*grade=7/section=b(\s)*grade=7/section=c' + ,'not_expected_out_regex' => 'grade=8/section=a(\s)*grade=8/section=b' + }, + { + 'num' => 12 + ,'depends_on' => 'HCat_AlterTable_11' + ,'hcat' => q\ +set hive.exec.drop.ignorenonexistent = false; +alter table hcat_altertable_7 drop if exists partition (grade='8',section='a'), partition(grade='7',section='d'), partition(grade='7',section='c'); +set hive.exec.drop.ignorenonexistent = true; +show partitions hcat_altertable_7;\, + ,'expected_out_regex' => 'grade=7/section=a(\s)*grade=7/section=b' + ,'rc' => 0 + }, + { + 'num' => 13 + ,'hcat' => q\ +drop table if exists hcat_altertable_13; +create table hcat_altertable_13(a int, b string) partitioned by (c string, d string) stored as textfile; +alter table hcat_altertable_13 add partition (c='1',d='1'); +alter table hcat_altertable_13 add columns (x string comment 'extra'); +describe hcat_altertable_13;\, + ,'rc' => 0 + ,'expected_out_regex' => 'a int(\s)*b string(\s)*x string extra(\s)*c string(\s)*d string' + }, + { + 'num' => 14 + ,'depends_on' => 'HCat_AlterTable_13' + ,'hcat' => q\ +alter table hcat_altertable_13 replace columns (k int, l string, m string); +describe hcat_altertable_13;\, + ,'rc' => 0 + ,'expected_out_regex' => 'k int(\s)*l string(\s)*m string(\s)*c string(\s)*d string' + }, + { + 'num' => 15 + ,'hcat' => q\ +drop table if exists hcat_altertable_15b; +drop table if exists hcat_altertable_15a; +create table hcat_altertable_15a(a int, b string) stored as textfile; +alter table hcat_altertable_15a rename to hcat_altertable_15b; +show tables like 'hcat_altertable_15.*';\, + ,'rc' => 0 + ,'expected_out_regex' => 'hcat_altertable_15b' + ,'not_expected_out_regex' => 'hcat_altertable_15a' + }, + + { + 'num' => 16 + ,'hcat' => q\ +drop table if exists hcat_altertable_16; +create table hcat_altertable_16(a int, b string) stored as textfile; +show table extended like hcat_altertable_16; +;\, + ,'rc' => 0 + ,'expected_out_regex' => '((.*)\s)*location(.*)hcat_altertable_16' + }, + + { + 'num' => 17 + ,'depends_on' => 'HCat_AlterTable_16' + ,'hcat' => q\ +alter table hcat_altertable_16 set location 'hdfs:///tmp/table_has_moved/'; +show table extended like hcat_altertable_16; +;\, + ,'rc' => 0 + ,'expected_out_regex' => 'location:hdfs:///tmp/table_has_moved' + ,'not_expected_out_regex' => 'location(.*)hcat_altertable_16' + }, + + ], }, # end g { @@ -193,7 +389,7 @@ drop table if exists hcat_view_1; drop view if exists hcat_view_1_1; drop view if exists hcat_view_1_2; -create external table hcat_view_1 (name string, age int, gpa double) row format delimited fields terminated by '\t' stored as TEXTFILE location '/user/hcat/tests/data/studenttab10k'; +create external table hcat_view_1 (name string, age int, gpa double) row format delimited fields terminated by '\t' stored as TEXTFILE location ':INPATH:/studenttab10k'; create view hcat_view_1_1 as select name, gpa, age from studenttab10k; create view hcat_view_1_2 partitioned on (age) as select name, gpa, age from studenttab10k; alter view hcat_view_1_1 set tblproperties('key'='value'); @@ -204,7 +400,17 @@ drop view hcat_view_1_3;" ,'rc' => 0 }, - ], + { + 'num' => 2 + ,'hcat' => q" +drop view if exists hcat_view_2_1; +create view hcat_view_2_1(name,age) as select name, age from studenttab10k; +describe extended hcat_view_2_1; +" + ,'expected_out_regex' => 'name string.*(\s)age int(.*(\s))*viewOriginalText:select name, age from studenttab10k.*tableType:VIRTUAL_VIEW' + ,'rc' => 0 + }, + ], }, # end g { 'name' => 'HCat_Authorize', @@ -240,10 +446,43 @@ create index hcat_index_1_1 on table hcat_index_1(a) as 'compact' with deferred rebuild comment 'hcat test'; alter index hcat_index_1_1 on hcat_index_1 set idxproperties ('prop1'='val1'); show indexes on hcat_index_1; +create index hcat_index_1_2 on table hcat_index_1(a) as 'compact' with deferred rebuild comment 'hcat test second'; +alter index hcat_index_1_2 on hcat_index_1 set idxproperties ('prop1'='val2'); +show indexes on hcat_index_1; drop index hcat_index_1_1 on hcat_index_1; +drop index hcat_index_1_2 on hcat_index_1; ;\ ,'rc' => 0 }, + + { + 'num' => 2 + ,'hcat' => q\ +drop index if exists hcat_index_2_1 on hcat_index_2; +drop table if exists hcat_index_2; +create table hcat_index_2 (a string) partitioned by (b string) stored as TEXTFILE; +create index hcat_index_2_1 on table hcat_index_2(a) as 'compact' with deferred rebuild comment 'hcat compact idx'; +alter index hcat_index_2_1 on hcat_index_2 set idxproperties ('prop1'='val1'); +show indexes on hcat_index_2; +;\ + ,'rc' => 0 + ,'expected_out_regex' => 'hcat_index_2_1(\s)*hcat_index_2(\s)*a(\s)*default__hcat_index_2_hcat_index_2_1__(\s)*compact(\s)*hcat compact idx' + }, + + { + 'num' => 3 + ,'hcat' => q\ +drop index if exists hcat_index_3_1 on hcat_index_3; +drop table if exists hcat_index_3; +create table hcat_index_3 (a string) partitioned by (b string) stored as TEXTFILE; +create index hcat_index_3_1 on table hcat_index_3(a) as 'bitmap' with deferred rebuild comment 'hcat bitmap idx'; +alter index hcat_index_3_1 on hcat_index_3 set idxproperties ('prop1'='val1'); +show indexes on hcat_index_3; +;\ + ,'rc' => 0 + ,'expected_out_regex' => 'hcat_index_3_1(\s)*hcat_index_3(\s)*a(\s)*default__hcat_index_3_hcat_index_3_1__(\s)*bitmap(\s)*hcat bitmap idx' + }, + ], }, # end g { @@ -264,6 +503,145 @@ ], }, # end g { + 'name' => 'HCat_ShowDes', + 'tests' => [ + { + 'num' => 1 + ,'hcat' => q\ +drop table if exists hcat_showdes.hcat_showdestable_1; +drop database if exists hcat_showdes; +drop database if exists t_showdes_abc; +drop table if exists hcat_showdestable_2; +create database hcat_showdes; +create database t_showdes_abc; +create table hcat_showdes.hcat_showdestable_1 (a int, b string); +create table hcat_showdestable_2 (a int comment 'first', b struct< foo : int, bar : struct< bar1:int, bar2:int > > comment 'second') partitioned by (c string); +alter table hcat_showdestable_2 add partition (c = '41'); +alter table hcat_showdestable_2 add partition (c = '42'); +;\ + ,'rc' => 0 + }, + { + 'num' => 2 + ,'depends_on' => 'HCat_ShowDes_1' + ,'hcat' => q\ +show databases;\ + ,'rc' => 0 + ,'expected_out_regex' => 'hcat_showdes' + }, + { + 'num' => 3 + ,'depends_on' => 'HCat_ShowDes_1' + ,'hcat' => q\ +show databases like '.*abc';\ + ,'rc' => 0 + ,'expected_out_regex' => 't_showdes_abc' + ,'not_expected_out_regex' => 'hcat_showdes' + }, + { + 'num' => 4 + ,'depends_on' => 'HCat_ShowDes_1' + ,'hcat' => q\ +show tables;\, + ,'rc' => 0 + ,'expected_out_regex' => 'hcat_showdestable_2' + }, + + { + 'num' => 5 + ,'depends_on' => 'HCat_ShowDes_1' + ,'hcat' => q\ +show tables in hcat_showdes;\, + ,'rc' => 0 + ,'expected_out_regex' => 'hcat_showdestable_1' + ,'not_expected_out_regex' => 'hcat_createtable_1' + }, + { + 'num' => 6 + ,'depends_on' => 'HCat_ShowDes_1' + ,'hcat' => q\ +show tables in hcat_showdes like '.*table_1';\, + ,'rc' => 0 + ,'expected_out_regex' => 'hcat_showdestable_1' + }, + { + 'num' => 7 + ,'depends_on' => 'HCat_ShowDes_1' + ,'hcat' => q\ +show table extended like hcat_showdestable_2;\, + ,'rc' => 0 + ,'expected_out_regex' => 'tableName(.*(\s))*owner(.*(\s))*location(.*(\s))*columns(.*(\s))*totalNumberFiles(.*(\s))*totalFileSize(.*(\s))*maxFileSize(.*(\s))*minFileSize(.*(\s))*lastAccessTime(.*(\s))*lastUpdateTime(.*)' + }, + { + 'num' => 8 + ,'depends_on' => 'HCat_ShowDes_1' + ,'hcat' => q\ +show table extended like hcat_showdestable_2 partition(c='42');\, + ,'rc' => 0 + ,'expected_out_regex' => 'tableName(.*(\s))*owner(.*(\s))*location(.*(\s))*columns(.*(\s))*totalNumberFiles(.*(\s))*totalFileSize(.*(\s))*maxFileSize(.*(\s))*minFileSize(.*(\s))*lastAccessTime(.*(\s))*lastUpdateTime(.*)' + }, + { + 'num' => 9 + ,'depends_on' => 'HCat_ShowDes_1' + ,'hcat' => q\ +describe extended hcat_showdestable_2 partition(c='42');\ + ,'rc' => 0 + ,'expected_out_regex' => 'a int first(\s)*b struct> second(\s)*c string(.*(\s))*Detailed Partition Information(\s)*Partition(.*)values:\[42\]' + }, + { + 'num' => 10 + ,'depends_on' => 'HCat_ShowDes_1' + ,'hcat' => q\ +describe extended hcat_showdestable_2.b;\ + ,'rc' => 0 + ,'expected_out_regex' => 'foo int from deserializer(\s)*bar struct from deserializer' + }, + { + 'num' => 11 + ,'depends_on' => 'HCat_ShowDes_1' + ,'hcat' => q\ +describe extended hcat_showdestable_2.b.foo;\ + ,'rc' => 0 + ,'expected_out_regex' => 'foo int from deserializer' + ,'not_expected_out_regex' => 'bar' + }, + { + 'num' => 12 + ,'depends_on' => 'HCat_ShowDes_1' + ,'hcat' => q\ +describe extended hcat_showdestable_2.b.bar;\ + ,'rc' => 0 + ,'expected_out_regex' => 'bar1 int from deserializer(\s)*bar2 int from deserializer' + }, + { + 'num' => 13 + ,'depends_on' => 'HCat_ShowDes_1' + ,'hcat' => q\ +describe extended hcat_showdestable_2.b.bar.bar1;\ + ,'rc' => 0 + ,'expected_out_regex' => 'bar1 int from deserializer' + ,'not_expected_out_regex' => 'bar2' + }, + { + 'num' => 14 + ,'depends_on' => 'HCat_ShowDes_1' + ,'hcat' => q\ +show partitions hcat_showdestable_2;\ + ,'rc' => 0 + ,'expected_out_regex' => 'c=41(\s)*c=42' + }, + { + 'num' => 15 + ,'depends_on' => 'HCat_ShowDes_1' + ,'hcat' => q\ +show partitions hcat_showdestable_2 partition(c='41');\ + ,'rc' => 0 + ,'expected_out_regex' => 'c=41' + ,'not_expected_out_regex' => 'c=42' + }, + ], + }, # end g + { 'name' => 'HCat_Misc', 'tests' => [ { Index: src/test/e2e/hcatalog/drivers/TestDriverHCat.pm =================================================================== --- src/test/e2e/hcatalog/drivers/TestDriverHCat.pm (revision 1341191) +++ src/test/e2e/hcatalog/drivers/TestDriverHCat.pm (working copy) @@ -135,7 +135,7 @@ open(FH, "> $hcatfiles[0]") or die "Unable to open file $hcatfiles[0] to write SQL script, $ERRNO\n"; - print FH $testCmd->{'hcat'} . "\n"; + print FH $hcatCmd . "\n"; close(FH); Util::runHCatCmdFromFile($testCmd, $log, $hcatfiles[0], $stdoutfile, $stderrfile, 1);