Index: src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/ReadRC.java =================================================================== --- src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/ReadRC.java (revision 1227799) +++ src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/ReadRC.java (working copy) @@ -68,6 +68,7 @@ name = (String)value.get(0); age = (Integer)value.get(1); gpa = (Double)value.get(2); + gpa = Math.floor(gpa) + 0.1; HCatRecord record = new DefaultHCatRecord(3); record.set(0, name); Index: src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteRC.java =================================================================== --- src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteRC.java (revision 1227799) +++ src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteRC.java (working copy) @@ -67,6 +67,8 @@ name = value.get(0)==null?null:(String)value.get(0); age = value.get(1)==null?null:(Integer)value.get(1); gpa = value.get(2)==null?null:(Double)value.get(2); + + if (gpa != null) gpa = Math.floor(gpa) + 0.1; HCatRecord record = new DefaultHCatRecord(5); record.set(0, name); Index: src/test/e2e/hcatalog/udfs/java/build.xml =================================================================== --- src/test/e2e/hcatalog/udfs/java/build.xml (revision 1227799) +++ src/test/e2e/hcatalog/udfs/java/build.xml (working copy) @@ -1,23 +1,23 @@ - - + - - + + @@ -27,17 +27,17 @@ - + - + *** Compiling UDFs *** - + Index: src/test/e2e/hcatalog/tests/hive.conf =================================================================== --- src/test/e2e/hcatalog/tests/hive.conf (revision 1227799) +++ src/test/e2e/hcatalog/tests/hive.conf (working copy) @@ -94,7 +94,8 @@ 'name' => 'Hive_Read', 'tests' => [ { 'num' => 1, - 'sql' => q\select * from all100krc;\, + 'sql' => q\select name, age, floor(gpa) from all100krc;\, + 'verify_sql' => q\select name, age, truncate(gpa, 0) from all100krc;\, 'floatpostprocess' => 1, 'delimiter' => ' ', } ] @@ -106,9 +107,9 @@ 'sql' => q\ drop table if exists hive_write_1; create table hive_write_1 (name string, age int, gpa double) stored as rcfile; -insert into TABLE hive_write_1 select * from all100krc;\, +insert into TABLE hive_write_1 select name, age, floor(gpa) + 0.1 from all100krc;\, 'result_table' => 'hive_write_1', - 'verify_sql' =>"select name, age, gpa from all100krc;", + 'verify_sql' =>"select name, age, floor(gpa) + 0.1 from all100krc;", 'floatpostprocess' => 1, 'delimiter' => ' ', } ] Index: src/test/e2e/hcatalog/tests/pig.conf =================================================================== --- src/test/e2e/hcatalog/tests/pig.conf (revision 1227799) +++ src/test/e2e/hcatalog/tests/pig.conf (working copy) @@ -86,6 +86,7 @@ 'tests' => [ { + 'ignore' => 1, # Need to checkin HCATALOG-168. 'num' => 1 ,'pig' => q\a = load 'all100k' using org.apache.hcatalog.pig.HCatLoader(); store a into ':OUTPATH:';\, @@ -105,8 +106,9 @@ { 'num' => 3 ,'pig' => q\a = load 'all100krc' using org.apache.hcatalog.pig.HCatLoader(); -store a into ':OUTPATH:';\, - ,'sql' => q\select * from all100krc;\ +b = foreach a generate name, age; +store b into ':OUTPATH:';\, + ,'sql' => q\select name, age from all100krc;\ ,'floatpostprocess' => 1 ,'delimiter' => ' ' } @@ -116,6 +118,7 @@ 'name' => 'Pig_Write', 'tests' => [ { + 'ignore' => 1, # Need to checkin HCATALOG-168. 'num' => 1 ,'hcat_prep'=>q\drop table if exists pig_write_1; create table pig_write_1(t tinyint,si smallint,i int,b bigint,bool boolean,f float,d double,s string) stored as rcfile;\ @@ -142,8 +145,9 @@ \ ,'pig' => q\a = load 'all100kjson' using org.apache.hcatalog.pig.HCatLoader(); b = foreach a generate s, i, d; -store b into ':OUTPATH:';\, +store b into 'pig_write_2' using org.apache.hcatalog.pig.HCatStorer();\, ,'sql' => q\select IFNULL(s, ""), IFNULL(i, ""), IFNULL(d, "") from all100kjson;\ + ,'result_table' => 'pig_write_2' ,'floatpostprocess' => 1 ,'delimiter' => ' ' }, @@ -161,8 +165,10 @@ ); \ ,'pig' => q\a = load 'all100krc' using org.apache.hcatalog.pig.HCatLoader(); -store a into ':OUTPATH:';\, - ,'sql' => q\select * from all100krc;\ +b = foreach a generate name, age; +store b into 'pig_write_3' using org.apache.hcatalog.pig.HCatStorer();\, + ,'sql' => q\select name, age from all100krc;\ + ,'result_table' => 'pig_write_3' ,'floatpostprocess' => 1 ,'delimiter' => ' ' } Index: src/test/e2e/hcatalog/tests/hadoop.conf =================================================================== --- src/test/e2e/hcatalog/tests/hadoop.conf (revision 1227799) +++ src/test/e2e/hcatalog/tests/hadoop.conf (working copy) @@ -79,7 +79,7 @@ ,'hadoop' => q\ jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.ReadRC -libjars :HCAT_JAR: :THRIFTSERVER: all100krc :OUTPATH: \, - ,'sql' => q\select * from all100krc;\ + ,'sql' => q\select name, age, floor(gpa) + 0.1 from all100krc;\ ,'floatpostprocess' => 1 ,'delimiter' => ' ' }, @@ -89,6 +89,7 @@ 'name' => 'Hadoop_Write', 'tests' => [ { + 'ignore' => 1, # Need to checkin HCATALOG-168. 'num' => 1 ,'hcat_prep'=>q\ drop table if exists hadoop_write_1; @@ -151,7 +152,7 @@ jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.WriteRC -libjars :HCAT_JAR: :THRIFTSERVER: all100krc hadoop_write_3 \, ,'result_table' => 'hadoop_write_3' - ,'sql' => q\select * from all100krc;\ + ,'sql' => q\select name, age, floor(gpa) + 0.1 from all100krc;\ ,'floatpostprocess' => 1 ,'delimiter' => ' ' }, Index: src/test/e2e/hcatalog/conf/default.conf =================================================================== --- src/test/e2e/hcatalog/conf/default.conf (revision 1227799) +++ src/test/e2e/hcatalog/conf/default.conf (working copy) @@ -1,19 +1,19 @@ -############################################################################ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - +############################################################################ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + my $me = `whoami`; chomp $me; @@ -64,7 +64,8 @@ , 'hadoopbin' => "$ENV{PH_CLUSTER_BIN}" , 'funcjarPath' => "$ENV{PH_ROOT}/lib/java" , 'paramPath' => "$ENV{PH_ROOT}/paramfiles" - , 'pigpath' => "$ENV{PIG_ROOT}" + , 'pigpath' => "$ENV{PIG_HOME}" + , 'pigjar' => "$ENV{PIG_JAR}" # Pig jar that doesn't have Antlr , 'oldpigpath' => "$ENV{PH_OLDPIG}" , 'additionaljars' => "$ENV{HCAT_ROOT}/build/hcatalog/hcatalog-0.3.0-dev.jar:$ENV{HCAT_ROOT}/hive/external/build/metastore/hive-metastore-0.9.0-SNAPSHOT.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/libthrift.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-exec-0.9.0-SNAPSHOT.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/libfb303.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/jdo2-api-2.3-ec.jar:$ENV{'HCAT_INSTALL_DIR'}/etc/hcatalog" Index: src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm =================================================================== --- src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm (revision 1227799) +++ src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm (working copy) @@ -339,6 +339,7 @@ my $hadoop_classpath = $self->replaceParameters( $testCmd->{'hadoop_classpath'}, $outfile, $testCmd, $log ); my $cp = $testCmd->{'hcatalog.jar'}; $cp =~ s/,/:/g; + $cp .= ":" . Util::findPigWithoutHadoopJar($testCmd, $log); $ENV{'HADOOP_CLASSPATH'} = $cp; } Index: src/test/e2e/hcatalog/drivers/Util.pm =================================================================== --- src/test/e2e/hcatalog/drivers/Util.pm (revision 1227799) +++ src/test/e2e/hcatalog/drivers/Util.pm (working copy) @@ -1,22 +1,22 @@ #!/usr/bin/env perl -############################################################################ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - +############################################################################ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + ########################################################################### # Class: Util # @@ -27,7 +27,6 @@ package Util; use IPC::Run qw(run); -use Log::Log4perl qw(:easy); sub prepareHCat { @@ -237,6 +236,7 @@ # unset HADOOP_CLASSPATH $ENV{'HADOOP_CLASSPATH'} = ""; + $ENV{'HADOOP_CLASSPATH'} = $cfg->{'pigjar'}; my @cmd; if (defined($sql)) { @@ -309,138 +309,7 @@ die "Failed running " . join(" ", @cmd) . "\n"; } -############################################################################## -# Sub: localTime -# -# Returns: -# A string with the local time -sub localTime() { - - my $retval = time(); - - my $local_time = gmtime( $retval); - - return $local_time; - -} - -############################################################################## -# Sub: formatedTime -# Returns the time with following format "$mday/$mon/$year $hour:$min:$sec $weekday[$wday]" -# -# Returns: -# formated time - -sub formatedTime() { - - my @weekday = ("Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"); - - my $retval = time(); - - my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = gmtime(time); - $year = $year + 1900; - $mon += 1; - return "$mday/$mon/$year $hour:$min:$sec $weekday[$wday]\n"; - - -} - -############################################################################## -# Sub: execCmd -# Records the command in the log and prints it to stdout before executing. -# -# Paramaters: -# $log - The log object -# $subName - The name of the subroutine the message originated at -# $lineNo - The line number of the subroutine the message originated at -# $cmd - The command string to execute -# $msg - A string, the message to print -# $level - (optional)The logging level for the message: DEBUG, INFO, WARN, ERROR, FATAL -# defaults to DEBUG. -# -# Returns: -# An array containing the output from the executed command. -# - -sub execCmd() { - - my ( $log, $subName, $lineNo, $cmd, $level ) = @_ ; - - my $count = @_; - my $thisSubName = (caller(0))[3]; - - #Check for errors in arguments - if ( $count < 4 ){ - - if ( $log ) { - $log->msg( $level, $thisSubName, __LINE__ , "Invalid number of arguments, got $count=( @_ )" ); - - } else { - print "ERROR: $0 $thisSubName at ".__LINE__."Invalid number of arguments\n"; - - } - return 1; - } - - #Log command, execute commdand, return results - $level = "DEBUG" if ( !$level ); - $log->msg( $level, $subName, $lineNo , "$cmd"); - - my @result = `$cmd`; - - $log->msg( $level, $subName, $lineNo , "@cmd") if ( @cmd ); - - return @result; -} - -sub getHiveCmd -{ - my ( $properties ) = @_; - - my $subName = (caller(0))[3]; - my @baseCmd; - - die "$0.$subName: null properties" if (! $properties ); - - my $cmd; - - $cmd = $properties->{'hive_bin_location'} . "/hive"; - if ( ! -x "$cmd" ) { - die "\n$0::$subName FATAL: Hive command does not exist: $cmd\n"; - } - push (@baseCmd, $cmd); - -# push (@baseCmd, '--config', $properties->{'testconfigpath'}) if defined($properties->{'testconfigpath'}); - - return @baseCmd; -} - -sub getHCatCmd -{ - my ( $properties ) = @_; - - my $subName = (caller(0))[3]; - my @baseCmd; - - die "$0.$subName: null properties" if (! $properties ); - - my $cmd; - - $cmd = $properties->{'hcat_bin_location'}; - if ( ! -x "$cmd" ) { - print STDERR "\n$0::$subName WARNING: Can't find hcat command: $cmd\n"; - $cmd = `which hcat`; - chomp $cmd; - print STDERR "$0::$subName WARNING: Instead using command: $cmd\n"; - } - die "\n$0::$subName FATAL: hcat command does not exist: $cmd\n" if ( ! -x $cmd ); - $ENV{"hive.metastore.local"} = "false"; - $ENV{"hive.metastore.uris"} = "thrift://".$properties->{'thriftserver'}; - push (@baseCmd, $cmd); - return @baseCmd; -} - sub show_call_stack { my ( $path, $line, $subr ); my $max_depth = 30; @@ -651,43 +520,6 @@ } -sub getBasePigSqlCmd -{ - - my $subName = (caller(0))[3]; - - Util::getPigCmd( 'testsql', @_ ); - -} - -sub getBasePigCmd -{ - - my $subName = (caller(0))[3]; - - Util::getPigCmd( 'testjar', @_ ); - -} - -sub getLatestBasePigCmd -{ - - my $subName = (caller(0))[3]; - - Util::getPigCmd( 'latesttestjar', @_ ); - -} - - -sub getBenchmarkBasePigCmd -{ - my $subName = (caller(0))[3]; - my ( $properties ) = @_; - - Util::getPigCmd( 'benchmarkjar', @_ ); - -} - sub setLocale { my $locale= shift; @@ -725,4 +557,13 @@ ."export LC_MEASUREMENT=\"$locale\";" ."export LC_IDENTIFICATION=\"$locale\""; } + +sub findPigWithoutHadoopJar($$) +{ + my ($cfg, $log) = @_; + + my $jar = `ls $cfg->{'pigpath'}/pig-*-withouthadoop.jar`; + return $jar; +} + 1; Index: src/test/e2e/hcatalog/build.xml =================================================================== --- src/test/e2e/hcatalog/build.xml (revision 1227799) +++ src/test/e2e/hcatalog/build.xml (working copy) @@ -197,7 +197,7 @@ - + @@ -232,6 +232,8 @@ + + Index: build.xml =================================================================== --- build.xml (revision 1227799) +++ build.xml (working copy) @@ -72,6 +72,9 @@ + + + @@ -363,6 +366,7 @@ + + + + + + + + + + + + + + + + +