Index: src/test/e2e/hcatalog/tools/generate/generate_data.pl =================================================================== --- src/test/e2e/hcatalog/tools/generate/generate_data.pl (revision 1310604) +++ src/test/e2e/hcatalog/tools/generate/generate_data.pl (working copy) @@ -361,8 +361,8 @@ die 'Please set $HIVE_HOME\n'; } - my $execJar = `ls ../../../../../hive/external/build/ql/hive-exec-*.jar`; - my $cliJar = `ls ../../../../../hive/external/build/cli/hive-cli-*.jar`; + my $execJar = `ls $ENV{HIVE_HOME}/lib/hive-exec-*.jar`; + my $cliJar = `ls $ENV{HIVE_HOME}/lib/hive-cli-*.jar`; chomp $execJar; chomp $cliJar; return ($execJar, $cliJar); Index: src/test/e2e/hcatalog/conf/existing_deployer.conf =================================================================== --- src/test/e2e/hcatalog/conf/existing_deployer.conf (revision 1310604) +++ src/test/e2e/hcatalog/conf/existing_deployer.conf (working copy) @@ -26,7 +26,7 @@ # hadoop values 'hadoopdir' => $ENV{'PH_CLUSTER'}, - 'hcat_data_dir' => ("$ENV{'PH_HDFS_BASE'}" || '/user/hcat').'/test/data', + 'hcat_data_dir' => ("$ENV{'PH_HDFS_BASE'}" || '/user/hcat').'/tests/data', # db values # 'dbuser' => 'pigtester', Index: src/test/e2e/hcatalog/conf/rpm.conf =================================================================== --- src/test/e2e/hcatalog/conf/rpm.conf (revision 0) +++ src/test/e2e/hcatalog/conf/rpm.conf (revision 0) @@ -0,0 +1,84 @@ +############################################################################ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +my $me = `whoami`; +chomp $me; + +# The contents of this file can be rewritten to fit your installation. +# Also, you can define the following environment variables and set things up as in the test setup +# PH_CLUSTER Root directory for cluster being used +# HCAT_ROOT Root directory for hcat version being used + +my $hdfsBase = $ENV{'PH_HDFS_BASE'} || "/user/hcat"; + +$ENV{'HCAT_HOME'} = "/usr/lib/hcatalog"; +$ENV{'HADOOP_HOME'} = "/usr/lib/hadoop"; +$ENV{'HIVE_HOME'} = "/usr/lib/hive"; +$ENV{'HBASE_HOME'} = "/usr/lib/hbase"; +$ENV{'PIG_HOME'} = "/usr/lib/pig"; + +$cfg = { + #HDFS + 'inpathbase' => "$hdfsBase/tests/data" + , 'outpathbase' => "$hdfsBase/out" + + #LOCAL + , 'localinpathbase' => "$ENV{HARNESS_ROOT}/in" + , 'localoutpathbase' => "$ENV{HARNESS_ROOT}/out/log" + , 'localxmlpathbase' => "$ENV{HARNESS_ROOT}/out/xml" + , 'localpathbase' => "$ENV{HARNESS_ROOT}/out/pigtest/$me" + + #TEST + , 'benchmarkPath' => "$ENV{HARNESS_ROOT}/benchmarks" + , 'scriptPath' => "$ENV{HARNESS_ROOT}/libexec" + , 'tmpPath' => "/tmp/pigtest" + , 'jythonjar' => "$ENV{PH_JYTHON_JAR}" + , 'propertiesFile' => "./conf/testpropertiesfile.conf" + , 'funcjarPath' => "$ENV{HARNESS_ROOT}/lib/java" + + #TESTDB + , 'dbuser' => "$ENV{'PH_DBUSER'}" || 'hcattest' + , 'dbhost' => "$ENV{'PH_DBHOST'}" || 'localhost' + , 'dbpasswd' => "$ENV{'PH_DBPASSWD'}" || 'hcattest' + , 'dbdb' => "$ENV{'PH_DBDB'}" || 'hcattestdb' + + #HCAT + , 'hcathome' => $ENV{'HCAT_HOME'} + , 'hcatshare' => "$ENV{'HCAT_HOME'}/share/hcatalog" + , 'hcatlib' => "$ENV{'HCAT_HOME'}/lib" + , 'hcatconf' => "$ENV{'HCAT_HOME'}/conf" + , 'hcatbin' => "$ENV{'HCAT_HOME'}/bin/hcat" + + #PIG + , 'pighome' => $ENV{'PIG_HOME'} + , 'pigbin' => "/usr/bin/pig" + + #HADOOP + , 'hadoopconfdir' => "$ENV{'HADOOP_HOME'}/conf" + , 'hadoopbin' => "/usr/bin/hadoop" + + #HIVE + , 'hivehome' => $ENV{'HIVE_HOME'} + , 'hivelib' => "$ENV{'HIVE_HOME'}/lib" + , 'hivebin' => "/usr/bin/hive" + , 'hiveconf' => "$ENV{'HIVE_HOME'}/conf" + + #HBASE + , 'hbaseconf' => "$ENV{'HBASE_HOME'}/conf" + , 'hbaselibdir' => "$ENV{'HBASE_HOME'}/" + , 'zklibdir' => "$ENV{'HBASE_HOME'}/lib" + +}; Index: src/test/e2e/hcatalog/conf/default.conf =================================================================== --- src/test/e2e/hcatalog/conf/default.conf (revision 1310604) +++ src/test/e2e/hcatalog/conf/default.conf (working copy) @@ -19,9 +19,6 @@ # The contents of this file can be rewritten to fit your installation. # Also, you can define the following environment variables and set things up as in the test setup -# PH_ROOT Root directory where test harness is installed -# PH_LOCAL Root directory for input and output for local mode tests -# PH_OUT Root directory where output data will be stored (on local disk, not HDFS) # PH_CLUSTER Root directory for cluster being used # HCAT_ROOT Root directory for hcat version being used @@ -33,16 +30,18 @@ , 'outpathbase' => "$hdfsBase/out" #LOCAL - , 'localinpathbase' => "$ENV{PH_LOCAL}/in" - , 'localoutpathbase' => "$ENV{PH_LOCAL}/out/log" - , 'localxmlpathbase' => "$ENV{PH_LOCAL}/out/xml" - , 'localpathbase' => "$ENV{PH_LOCAL}/out/pigtest/$me" + , 'localinpathbase' => "$ENV{HARNESS_ROOT}/in" + , 'localoutpathbase' => "$ENV{HARNESS_ROOT}/out/log" + , 'localxmlpathbase' => "$ENV{HARNESS_ROOT}/out/xml" + , 'localpathbase' => "$ENV{HARNESS_ROOT}/out/pigtest/$me" #TEST - , 'benchmarkPath' => "$ENV{PH_OUT}/benchmarks" - , 'scriptPath' => "$ENV{PH_ROOT}/libexec" + , 'benchmarkPath' => "$ENV{HARNESS_ROOT}/benchmarks" + , 'scriptPath' => "$ENV{HARNESS_ROOT}/libexec" , 'tmpPath' => "/tmp/pigtest" , 'jythonjar' => "$ENV{PH_JYTHON_JAR}" + , 'propertiesFile' => "./conf/testpropertiesfile.conf" + , 'funcjarPath' => "$ENV{HARNESS_ROOT}/lib/java" #TESTDB , 'dbuser' => "$ENV{'PH_DBUSER'}" || 'hcattest' @@ -50,38 +49,30 @@ , 'dbpasswd' => "$ENV{'PH_DBPASSWD'}" || 'hcattest' , 'dbdb' => "$ENV{'PH_DBDB'}" || 'hcattestdb' - #COMMON - , 'metastore.principal' => "$ENV{METASTORE_PRINCIPAL}" - , 'metastore_thrift' => $ENV{'PH_METASTORE_THRIFT'} - , 'thriftserver' => "$ENV{HCAT_URL}" - #HCAT - , 'hcat_data_dir' => '/user/hcat/tests/data' - , 'hivehome' => $ENV{'PH_HIVE_HOME'} - , 'hcathome' => $ENV{'HCAT_INSTALL_DIR'} - , 'hcatalog.jar' => $ENV{'HADOOP_LIBJARS'} + , 'hcathome' => $ENV{'HCAT_HOME'} + , 'hcatshare' => "$ENV{'HCAT_HOME'}/share/hcatalog" + , 'hcatlib' => "$ENV{'HCAT_HOME'}/share/hcatalog/lib" + , 'hcatconf' => "$ENV{'HCAT_HOME'}/etc/hcatalog" + , 'hcatbin' => "$ENV{'HCAT_HOME'}/bin/hcat" + #PIG - , 'testconfigpath' => "$ENV{PH_CLUSTER}" - , 'hadoopbin' => "$ENV{PH_CLUSTER_BIN}" - , 'funcjarPath' => "$ENV{PH_ROOT}/lib/java" - , 'paramPath' => "$ENV{PH_ROOT}/paramfiles" - , 'pigpath' => "$ENV{PIG_HOME}" - , 'oldpigpath' => "$ENV{PH_OLDPIG}" - , 'additionaljars' => "$ENV{PIG_ADDITIONAL_JARS}" + , 'pighome' => $ENV{'PIG_HOME'} + , 'pigbin' => "$ENV{'PIG_HOME'}/bin/pig" #HADOOP - , 'UNUSEDhadoopHome' => "$ENV{HCAT_ROOT}/lib" - , 'userhomePath' => "$ENV{HOME}" - , 'local.bin' => '/usr/bin' - , 'logDir' => "$ENV{PH_OUT}/log" - , 'propertiesFile' => "./conf/testpropertiesfile.conf" - , 'harness.console.level' => 'ERROR' + , 'hadoopconfdir' => "$ENV{'HADOOP_HOME'}/conf" + , 'hadoopbin' => "$ENV{'HADOOP_HOME'}/bin/hadoop" #HIVE - , 'hive_bin_location' => "$ENV{HIVE_ROOT}/build/dist/bin" - , 'hbaseconfigpath' => "$ENV{HBASE_CONF_DIR}" - , 'hivehome' => "$ENV{HIVE_HOME}" - , 'hive.additionaljars' => "$ENV{HCAT_JARS}" - , 'hive.conf.dir' => "$ENV{HIVE_CONF_DIR}" || "$ENV{'HCAT_INSTALL_DIR'}/etc/hcatalog" + , 'hivehome' => $ENV{'HIVE_HOME'} + , 'hivelib' => "$ENV{'HIVE_HOME'}/lib" + , 'hivebin' => "$ENV{'HIVE_HOME'}/bin/hive" + , 'hiveconf' => "$ENV{'HIVE_HOME'}/conf" + #HBASE + , 'hbaseconf' => "$ENV{'HBASE_HOME'}/conf" + , 'hbaselibdir' => "$ENV{'HBASE_HOME'}/" + , 'zklibdir' => "$ENV{'HBASE_HOME'}/lib" + }; Index: src/test/e2e/hcatalog/conf/envbased.conf =================================================================== --- src/test/e2e/hcatalog/conf/envbased.conf (revision 0) +++ src/test/e2e/hcatalog/conf/envbased.conf (revision 0) @@ -0,0 +1,78 @@ +############################################################################ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +my $me = `whoami`; +chomp $me; + +# The contents of this file can be rewritten to fit your installation. +# Also, you can define the following environment variables and set things up as in the test setup +# PH_CLUSTER Root directory for cluster being used +# HCAT_ROOT Root directory for hcat version being used + +my $hdfsBase = $ENV{'PH_HDFS_BASE'} || "/user/hcat"; + +$cfg = { + #HDFS + 'inpathbase' => "$hdfsBase/tests/data" + , 'outpathbase' => "$hdfsBase/out" + + #LOCAL + , 'localinpathbase' => "$ENV{HARNESS_ROOT}/in" + , 'localoutpathbase' => "$ENV{HARNESS_ROOT}/out/log" + , 'localxmlpathbase' => "$ENV{HARNESS_ROOT}/out/xml" + , 'localpathbase' => "$ENV{HARNESS_ROOT}/out/pigtest/$me" + + #TEST + , 'benchmarkPath' => "$ENV{HARNESS_ROOT}/benchmarks" + , 'scriptPath' => "$ENV{HARNESS_ROOT}/libexec" + , 'tmpPath' => "/tmp/pigtest" + , 'jythonjar' => "$ENV{PH_JYTHON_JAR}" + , 'propertiesFile' => "./conf/testpropertiesfile.conf" + , 'funcjarPath' => "$ENV{HARNESS_ROOT}/lib/java" + + #TESTDB + , 'dbuser' => "$ENV{'PH_DBUSER'}" || 'hcattest' + , 'dbhost' => "$ENV{'PH_DBHOST'}" || 'localhost' + , 'dbpasswd' => "$ENV{'PH_DBPASSWD'}" || 'hcattest' + , 'dbdb' => "$ENV{'PH_DBDB'}" || 'hcattestdb' + + #HCAT + , 'hcathome' => $ENV{'HCAT_HOME'} + , 'hcatshare' => $ENV{'HCAT_SHARE_DIR'} + , 'hcatlib' => $ENV{'HCAT_LIB_DIR'} + , 'hcatconf' => $ENV{'HCAT_CONF_DIR'} + , 'hcatbin' => $ENV{'HCAT_CMD'} + + #PIG + , 'pighome' => $ENV{'PIG_HOME'} + , 'pigbin' => "$ENV{'PIG_HOME'}/bin/pig" + + #HADOOP + , 'hadoopconfdir' => $ENV{'HADOOP_CONF_DIR'} + , 'hadoopbin' => "$ENV{'HADOOP_HOME'}/bin/hadoop" + + #HIVE + , 'hivehome' => $ENV{'HIVE_HOME'} + , 'hivelib' => "$ENV{'HIVE_HOME'}/lib" + , 'hivebin' => $ENV{'HIVE_CMD'} + , 'hiveconf' => "$ENV{'HIVE_HOME'}/conf" + + #HBASE + , 'hbaseconf' => "$ENV{'HBASE_HOME'}/conf" + , 'hbaselibdir' => "$ENV{'HBASE_HOME'}/" + , 'zklibdir' => "$ENV{'HBASE_HOME'}/lib" + +}; Index: src/test/e2e/hcatalog/deployers/HCatExistingClusterDeployer.pm =================================================================== --- src/test/e2e/hcatalog/deployers/HCatExistingClusterDeployer.pm (revision 1310604) +++ src/test/e2e/hcatalog/deployers/HCatExistingClusterDeployer.pm (working copy) @@ -70,6 +70,14 @@ print $log "You must set the environment variable HADOOP_HOME"; die "HADOOP_HOME not defined"; } + if (! defined $ENV{'HCAT_HOME'} || $ENV{'HCAT_HOME'} eq "") { + print $log "You must set the environment variable HCAT_HOME"; + die "HCAT_HOME not defined"; + } + if (! defined $ENV{'HIVE_HOME'} || $ENV{'HIVE_HOME'} eq "") { + print $log "You must set the environment variable HIVEOP_HOME"; + die "HIVE_HOME not defined"; + } # Run a quick and easy Hadoop command to make sure we can Util::runHadoopCmd($cfg, $log, "fs -ls /"); @@ -171,9 +179,9 @@ ); - if (defined($cfg->{'load_hive_only'}) && $cfg->{'load_hive_only'} == 1) { - return $self->hiveMetaOnly($cfg, $log, \@tables); - } +# if (defined($cfg->{'load_hive_only'}) && $cfg->{'load_hive_only'} == 1) { +# return $self->hiveMetaOnly($cfg, $log, \@tables); +# } # Create the HDFS directories Util::runHadoopCmd($cfg, $log, "fs -mkdir $cfg->{'hcat_data_dir'}"); Index: src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm =================================================================== --- src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm (revision 1310604) +++ src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm (working copy) @@ -52,52 +52,6 @@ return $self; } -sub replaceParameters -{ -##!!! Move this to Util.pm - - my ($self, $cmd, $outfile, $testCmd, $log) = @_; - - # $self - $cmd =~ s/:LATESTOUTPUTPATH:/$self->{'latestoutputpath'}/g; - - # $outfile - $cmd =~ s/:OUTPATH:/$outfile/g; - - # $ENV - $cmd =~ s/:PIGHARNESS:/$ENV{HARNESS_ROOT}/g; - - # $testCmd - $cmd =~ s/:INPATH:/$testCmd->{'inpathbase'}/g; - $cmd =~ s/:OUTPATH:/$outfile/g; - $cmd =~ s/:FUNCPATH:/$testCmd->{'funcjarPath'}/g; - $cmd =~ s/:PIGPATH:/$testCmd->{'pigpath'}/g; - $cmd =~ s/:RUNID:/$testCmd->{'UID'}/g; - $cmd =~ s/:USRHOMEPATH:/$testCmd->{'userhomePath'}/g; - $cmd =~ s/:MAPREDJARS:/$testCmd->{'mapredjars'}/g; - $cmd =~ s/:SCRIPTHOMEPATH:/$testCmd->{'scriptPath'}/g; - $cmd =~ s/:DBUSER:/$testCmd->{'dbuser'}/g; - $cmd =~ s/:DBNAME:/$testCmd->{'dbdb'}/g; -# $cmd =~ s/:LOCALINPATH:/$testCmd->{'localinpathbase'}/g; -# $cmd =~ s/:LOCALOUTPATH:/$testCmd->{'localoutpathbase'}/g; -# $cmd =~ s/:LOCALTESTPATH:/$testCmd->{'localpathbase'}/g; - $cmd =~ s/:BMPATH:/$testCmd->{'benchmarkPath'}/g; - $cmd =~ s/:TMP:/$testCmd->{'tmpPath'}/g; - $cmd =~ s/:HDFSTMP:/tmp\/$testCmd->{'runid'}/g; - - if ( $testCmd->{'hadoopSecurity'} eq "secure" ) { - $cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteSecureCluster'}/g; - } else { - $cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteNotSecureCluster'}/g; - } - - $cmd =~ s/:THRIFTSERVER:/$testCmd->{'thriftserver'}/g; - $cmd =~ s/:HADOOP_CLASSPATH:/$testCmd->{'hadoop_classpath'}/g; - $cmd =~ s/:HCAT_JAR:/$testCmd->{'hcatalog.jar'}/g; - - return $cmd; -} - sub globalSetup { my ($self, $globalHash, $log) = @_; @@ -127,11 +81,8 @@ $ENV{'PATH'} = $globalHash->{'scriptPath'}; } - my @cmd = ($self->getPigCmd($globalHash, $log), '-e', 'mkdir', $globalHash->{'outpath'}); + Util::runHadoopCmd($globalHash, $log, "fs -mkdir $globalHash->{'outpath'}"); - print $log "Going to run " . join(" ", @cmd) . "\n"; - IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create HDFS directory " . $globalHash->{'outpath'} . ": $? - $!\n"; - IPC::Run::run(['mkdir', '-p', $globalHash->{'localpath'}], \undef, $log, $log) or die "Cannot create localpath directory " . $globalHash->{'localpath'} . " " . "$ERRNO\n"; @@ -145,10 +96,7 @@ die "Cannot create temporary directory " . $globalHash->{'tmpPath'} . " " . "$ERRNO\n"; - # Create the HDFS temporary directory - @cmd = ($self->getPigCmd($globalHash, $log), '-e', 'mkdir', "tmp/$globalHash->{'runid'}"); - print $log "Going to run " . join(" ", @cmd) . "\n"; - IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create HDFS directory " . $globalHash->{'outpath'} . ": $? - $!\n"; + Util::runHadoopCmd($globalHash, $log, "fs -mkdir tmp/$globalHash->{'runid'}"); } sub globalCleanup @@ -199,8 +147,8 @@ my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out"; # Copy result file out of hadoop - my @baseCmd = $self->getPigCmd($testCmd, $log); - my $testOut = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log); + my @baseCmd = Util::getPigCmd($testCmd, $log); + my $testOut = $self->postProcessSingleOutputFile($outfile, $localdir, $testCmd, $log); $outputs[$i] = $testOut; $id++; } @@ -236,18 +184,16 @@ # Build the command - my @baseCmd = $self->getPigCmd($testCmd, $log); + my @baseCmd = Util::getPigCmd($testCmd, $log); my @cmd = @baseCmd; push(@cmd, $pigfile); # Run the command - print $log "Setting PIG_CLASSPATH to $ENV{'PIG_CLASSPATH'}\n"; print $log "$0::$className::$subName INFO: Going to run pig command: @cmd\n"; - IPC::Run::run(\@cmd, \undef, $log, $log) or - die "Failed running $pigfile\n"; + IPC::Run::run(\@cmd, \undef, $log, $log) or die "Failed running $pigfile\n"; $result{'rc'} = $? >> 8; @@ -255,23 +201,17 @@ my $localoutfile; my $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . $id . ".dump.out"; - $outfile = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log); + $outfile = $self->postProcessSingleOutputFile($outfile, $localdir, $testCmd, $log); return $outfile; } sub postProcessSingleOutputFile { - my ($self, $outfile, $localdir, $baseCmd, $testCmd, $log) = @_; + my ($self, $outfile, $localdir, $testCmd, $log) = @_; my $subName = (caller(0))[3]; - my @baseCmd = @{$baseCmd}; - my @copyCmd = @baseCmd; - push(@copyCmd, ('-e', 'copyToLocal', $outfile, $localdir)); - print $log "$0::$className::$subName INFO: Going to run pig command: @copyCmd\n"; + Util::runHadoopCmd($globalHash, $log, "fs -copyToLocal $outfile $localdir"); - IPC::Run::run(\@copyCmd, \undef, $log, $log) or die "Cannot copy results from HDFS $outfile to $localdir\n"; - - # Sort the result if necessary. Keep the original output in one large file. # Use system not IPC run so that the '*' gets interpolated by the shell. @@ -301,8 +241,6 @@ } sub runHadoop -# Being modified from runPig -# !!! Works, but need to add other arguments, like queue...??? { my ($self, $testCmd, $log) = @_; my $subName = (caller(0))[3]; @@ -313,7 +251,13 @@ my $hadoopfile = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".hadoop"; my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out"; - my $hadoopcmd = $self->replaceParameters( $testCmd->{'hadoop'}, $outfile, $testCmd, $log ); + # Get all of the additional jars we'll need. + my $additionalJars = Util::getHiveLibs($testCmd, $log); + $additionalJars .= Util::getHCatLibs($testCmd, $log); + $additionalJars .= Util::getHBaseLibs($testCmd, $log); + $testCmd->{'libjars'} = $additionalJars; + $testCmd->{'libjars'} =~ s/:/,/g; + my $hadoopcmd = Util::replaceParameters( $testCmd->{'hadoop'}, $outfile, $testCmd, $log ); # adjust for the leading and trailing new line often seen in the conf file's command directives $hadoopcmd =~ s/^\s*(.*?)\s*$/\1/s; @@ -338,18 +282,14 @@ my $cp = $testCmd->{'hcatalog.jar'}; $cp =~ s/,/:/g; # Add in the hcat config file - $cp .= ":" . $testCmd->{'hive.conf.dir'}; + $cp .= ":" . $testCmd->{'hcatconf'}; + $cp .= ":" . $additionalJars; $ENV{'HADOOP_CLASSPATH'} = $cp; - if (defined($testCmd->{'hbaseconfigpath'})) { - $ENV{'HADOOP_CLASSPATH'} = "$ENV{'HADOOP_CLASSPATH'}:$testCmd->{'hbaseconfigpath'}"; + if (defined($testCmd->{'hbaseconf'})) { + $ENV{'HADOOP_CLASSPATH'} = "$ENV{'HADOOP_CLASSPATH'}:$testCmd->{'hbaseconf'}"; } - if (defined($testCmd->{'metastore.principal'}) && ($testCmd->{'metastore.principal'} =~ m/\S+/)) { - $ENV{'HADOOP_OPTS'} = "$ENV{'HADOOP_OPTS'} -Dhive.metastore.kerberos.principal=" . $testCmd->{'metastore.principal'}; - $ENV{'HADOOP_CLIENT_OPTS'} = "-Dhive.metastore.kerberos.principal=" . $testCmd->{'metastore.principal'}; - } - # Add su user if provided if (defined($testCmd->{'run_as'})) { my $cmd = '"' . join (" ", @cmd) . '"'; @@ -373,10 +313,10 @@ die "Failed running $script\n"; my $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . $id . ".dump.out"; - my @baseCmd = $self->getPigCmd($testCmd, $log); + my @baseCmd = Util::getPigCmd($testCmd, $log); if ($self->countStores($testCmd)==1) { @outputs = (); - $outputs[0] = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log); + $outputs[0] = $self->postProcessSingleOutputFile($outfile, $localdir, $testCmd, $log); $result{'outputs'} = \@outputs; } @@ -535,7 +475,7 @@ my $pigfile = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".pig"; my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out"; - my $pigcmd = $self->replaceParameters( $testCmd->{'pig'}, $outfile, $testCmd, $log ); + my $pigcmd = Util::replaceParameters( $testCmd->{'pig'}, $outfile, $testCmd, $log ); open(FH, "> $pigfile") or die "Unable to open file $pigfile to write pig script, $ERRNO\n"; print FH $pigcmd . "\n"; @@ -543,7 +483,8 @@ # Build the command - my @baseCmd = $self->getPigCmd($testCmd, $log); + #my @baseCmd = $self->getPigCmd($testCmd, $log); + my @baseCmd = Util::getPigCmd($testCmd, $log); my @cmd = @baseCmd; # Add option -l giving location for secondary logs @@ -580,7 +521,7 @@ # single query if ($stores == 1) { if ($copyResults) { - $result{'output'} = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log); + $result{'output'} = $self->postProcessSingleOutputFile($outfile, $localdir, $testCmd, $log); $result{'originalOutput'} = "$localdir/out_original"; # populated by postProcessSingleOutputFile } else { $result{'output'} = "NO_COPY"; @@ -596,7 +537,7 @@ # Copy result file out of hadoop my $testOut; if ($copyResults) { - $testOut = $self->postProcessSingleOutputFile($localoutfile, $localdir, \@baseCmd, $testCmd, $log); + $testOut = $self->postProcessSingleOutputFile($localoutfile, $localdir, $testCmd, $log); } else { $testOut = "NO_COPY"; } @@ -615,47 +556,6 @@ return \%result; } -sub getPigCmd($$$) -{ - my ($self, $testCmd, $log) = @_; - - my @pigCmd; - - # set the PIG_CLASSPATH environment variable - my $pcp .= $testCmd->{'jythonjar'} if (defined($testCmd->{'jythonjar'})); - $pcp .= ":" . $testCmd->{'classpath'} if (defined($testCmd->{'classpath'})); - $pcp .= ":" . $testCmd->{'additionaljars'} if (defined($testCmd->{'additionaljars'})); - # Only add testconfigpath to PIG_CLASSPATH if HADOOP_HOME isn't defined - $pcp .= ":" . $testCmd->{'testconfigpath'} if ($testCmd->{'exectype'} ne "local"); #&& (! defined $ENV{'HADOOP_HOME'}); - $pcp .= ":" . $testCmd->{'hbaseconfigpath'} if ($testCmd->{'exectype'} ne "local" && defined($testCmd->{'hbaseconfigpath'} && $testCmd->{'hbaseconfigpath'} ne "")); - - # Set it in our current environment. It will get inherited by the IPC::Run - # command. - $ENV{'PIG_CLASSPATH'} = $pcp; - - @pigCmd = ("$testCmd->{'pigpath'}/bin/pig"); - - if (defined($testCmd->{'additionaljars'})) { - push(@pigCmd, '-Dpig.additional.jars='.$testCmd->{'additionaljars'}); - } - - if ($testCmd->{'exectype'} eq "local") { - push(@{$testCmd->{'java_params'}}, "-Xmx1024m"); - push(@pigCmd, ("-x", "local")); - } - - my $opts .= "-Dhive.metastore.uris=$testCmd->{'thriftserver'}"; - if (defined($testCmd->{'java_params'})) { - $opts = $opts . " " . join(" ", @{$testCmd->{'java_params'}}); - } - - $ENV{'PIG_OPTS'} = $opts; - - print $log "Returning Pig command " . join(" ", @pigCmd) . "\n"; - print $log "With PIG_CLASSPATH set to " . $ENV{'PIG_CLASSPATH'} . " and PIG_OPTS set to " . $ENV{'PIG_OPTS'} . "\n"; - return @pigCmd; -} - sub compareSingleOutput { my ($self, $testResult, $testOutput, $benchmarkOutput, $log) = @_; Index: src/test/e2e/hcatalog/drivers/TestDriverHive.pm =================================================================== --- src/test/e2e/hcatalog/drivers/TestDriverHive.pm (revision 1310604) +++ src/test/e2e/hcatalog/drivers/TestDriverHive.pm (working copy) @@ -53,35 +53,11 @@ return $self; } -sub replaceParameters -{ -##!!! Move this to Util.pm - - my ($self, $cmd, $outfile, $testCmd, $log) = @_; - - # $self - $cmd =~ s/:LATESTOUTPUTPATH:/$self->{'latestoutputpath'}/g; - - # $outfile - $cmd =~ s/:OUTPATH:/$outfile/g; - - # $ENV - $cmd =~ s/:HARNESS:/$ENV{HARNESS_ROOT}/g; - - # $testCmd - $cmd =~ s/:INPATH:/$testCmd->{'inpathbase'}/g; - - return $cmd; -} - sub globalSetup { my ($self, $globalHash, $log) = @_; my $subName = (caller(0))[3]; - # Set up values for the metastore - Util::setupHiveProperties($globalHash, $log); - # Setup the output path my $me = `whoami`; chomp $me; Index: src/test/e2e/hcatalog/drivers/TestDriverHCat.pm =================================================================== --- src/test/e2e/hcatalog/drivers/TestDriverHCat.pm (revision 1310604) +++ src/test/e2e/hcatalog/drivers/TestDriverHCat.pm (working copy) @@ -53,48 +53,8 @@ return $self; } -sub replaceParameters -{ -##!!! Move this to Util.pm - my ($self, $cmd, $outfile, $testCmd, $log) = @_; - # $self - $cmd =~ s/:LATESTOUTPUTPATH:/$self->{'latestoutputpath'}/g; - - # $outfile - $cmd =~ s/:OUTPATH:/$outfile/g; - - # $ENV - $cmd =~ s/:PIGHARNESS:/$ENV{HARNESS_ROOT}/g; - - # $testCmd - $cmd =~ s/:INPATH:/$testCmd->{'inpathbase'}/g; - $cmd =~ s/:OUTPATH:/$outfile/g; - $cmd =~ s/:FUNCPATH:/$testCmd->{'funcjarPath'}/g; - $cmd =~ s/:PIGPATH:/$testCmd->{'pigpath'}/g; - $cmd =~ s/:RUNID:/$testCmd->{'UID'}/g; - $cmd =~ s/:USRHOMEPATH:/$testCmd->{'userhomePath'}/g; - $cmd =~ s/:MAPREDJARS:/$testCmd->{'mapredjars'}/g; - $cmd =~ s/:SCRIPTHOMEPATH:/$testCmd->{'scriptPath'}/g; - $cmd =~ s/:DBUSER:/$testCmd->{'dbuser'}/g; - $cmd =~ s/:DBNAME:/$testCmd->{'dbdb'}/g; -# $cmd =~ s/:LOCALINPATH:/$testCmd->{'localinpathbase'}/g; -# $cmd =~ s/:LOCALOUTPATH:/$testCmd->{'localoutpathbase'}/g; -# $cmd =~ s/:LOCALTESTPATH:/$testCmd->{'localpathbase'}/g; - $cmd =~ s/:BMPATH:/$testCmd->{'benchmarkPath'}/g; - $cmd =~ s/:TMP:/$testCmd->{'tmpPath'}/g; - $cmd =~ s/:HDFSTMP:/tmp\/$testCmd->{'runid'}/g; - - if ( $testCmd->{'hadoopSecurity'} eq "secure" ) { - $cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteSecureCluster'}/g; - } else { - $cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteNotSecureCluster'}/g; - } - - return $cmd; -} - sub globalSetup { my ($self, $globalHash, $log) = @_; @@ -152,7 +112,7 @@ my $subName = (caller(0))[3]; my %result; my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out"; - my $hcatCmd = $self->replaceParameters( $testCmd->{'hcat'}, $outfile, $testCmd, $log); + my $hcatCmd = Util::replaceParameters( $testCmd->{'hcat'}, $outfile, $testCmd, $log); my $outdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out"; my ($stdoutfile, $stderrfile); Index: src/test/e2e/hcatalog/drivers/TestDriverPig.pm =================================================================== --- src/test/e2e/hcatalog/drivers/TestDriverPig.pm (revision 1310604) +++ src/test/e2e/hcatalog/drivers/TestDriverPig.pm (working copy) @@ -52,48 +52,6 @@ return $self; } -sub replaceParameters -{ -##!!! Move this to Util.pm - - my ($self, $cmd, $outfile, $testCmd, $log) = @_; - - # $self - $cmd =~ s/:LATESTOUTPUTPATH:/$self->{'latestoutputpath'}/g; - - # $outfile - $cmd =~ s/:OUTPATH:/$outfile/g; - - # $ENV - $cmd =~ s/:PIGHARNESS:/$ENV{HARNESS_ROOT}/g; - - # $testCmd - $cmd =~ s/:INPATH:/$testCmd->{'inpathbase'}/g; - $cmd =~ s/:OUTPATH:/$outfile/g; - $cmd =~ s/:FUNCPATH:/$testCmd->{'funcjarPath'}/g; - $cmd =~ s/:PIGPATH:/$testCmd->{'pigpath'}/g; - $cmd =~ s/:RUNID:/$testCmd->{'UID'}/g; - $cmd =~ s/:USRHOMEPATH:/$testCmd->{'userhomePath'}/g; - $cmd =~ s/:MAPREDJARS:/$testCmd->{'mapredjars'}/g; - $cmd =~ s/:SCRIPTHOMEPATH:/$testCmd->{'scriptPath'}/g; - $cmd =~ s/:DBUSER:/$testCmd->{'dbuser'}/g; - $cmd =~ s/:DBNAME:/$testCmd->{'dbdb'}/g; -# $cmd =~ s/:LOCALINPATH:/$testCmd->{'localinpathbase'}/g; -# $cmd =~ s/:LOCALOUTPATH:/$testCmd->{'localoutpathbase'}/g; -# $cmd =~ s/:LOCALTESTPATH:/$testCmd->{'localpathbase'}/g; - $cmd =~ s/:BMPATH:/$testCmd->{'benchmarkPath'}/g; - $cmd =~ s/:TMP:/$testCmd->{'tmpPath'}/g; - $cmd =~ s/:HDFSTMP:/tmp\/$testCmd->{'runid'}/g; - - if ( $testCmd->{'hadoopSecurity'} eq "secure" ) { - $cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteSecureCluster'}/g; - } else { - $cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteNotSecureCluster'}/g; - } - - return $cmd; -} - sub globalSetup { my ($self, $globalHash, $log) = @_; @@ -123,7 +81,7 @@ $ENV{'PATH'} = $globalHash->{'scriptPath'}; } - my @cmd = ($self->getPigCmd($globalHash, $log), '-e', 'mkdir', $globalHash->{'outpath'}); + my @cmd = (Util::getPigCmd($globalHash, $log), '-e', 'mkdir', $globalHash->{'outpath'}); print $log "Going to run " . join(" ", @cmd) . "\n"; @@ -143,7 +101,7 @@ " " . "$ERRNO\n"; # Create the HDFS temporary directory - @cmd = ($self->getPigCmd($globalHash, $log), '-e', 'mkdir', "tmp/$globalHash->{'runid'}"); + @cmd = (Util::getPigCmd($globalHash, $log), '-e', 'mkdir', "tmp/$globalHash->{'runid'}"); print $log "Going to run " . join(" ", @cmd) . "\n"; IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create HDFS directory " . $globalHash->{'outpath'} . ": $? - $!\n"; } @@ -157,7 +115,7 @@ " " . "$ERRNO\n"; # Cleanup the HDFS temporary directory - my @cmd = ($self->getPigCmd($globalHash, $log), '-e', 'fs', '-rmr', "tmp/$globalHash->{'runid'}"); + my @cmd = (Util::getPigCmd($globalHash, $log), '-e', 'fs', '-rmr', "tmp/$globalHash->{'runid'}"); print $log "Going to run " . join(" ", @cmd) . "\n"; IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create HDFS directory " . $globalHash->{'outpath'} . ": $? - $!\n"; } @@ -221,7 +179,7 @@ my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out"; # Copy result file out of hadoop - my @baseCmd = $self->getPigCmd($testCmd, $log); + my @baseCmd = Util::getPigCmd($testCmd, $log); my $testOut = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log); $outputs[$i] = $testOut; $id++; @@ -264,22 +222,16 @@ } # Write the pig script to a file. - my $pigcmd = $self->replaceParameters( $testCmd->{'pig'}, $outfile, $testCmd, $log ); + my $pigcmd = Util::replaceParameters( $testCmd->{'pig'}, $outfile, $testCmd, $log ); open(FH, "> $pigfile") or die "Unable to open file $pigfile to write pig script, $ERRNO\n"; print FH $pigcmd . "\n"; close(FH); # Build the command - my @baseCmd = $self->getPigCmd($testCmd, $log); + my @baseCmd = Util::getPigCmd($testCmd, $log); my @cmd = @baseCmd; - # Add option -l giving location for secondary logs - ##!!! Should that even be here? - my $locallog = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".log"; - push(@cmd, "-logfile"); - push(@cmd, $locallog); - # Add pig parameters if they're provided if (defined($testCmd->{'pig_params'})) { # Processing :PARAMPATH: in parameters @@ -333,7 +285,7 @@ } # Write the script to a file - my $cmd = $self->replaceParameters( $testCmd->{'script'}, $outfile, $testCmd, $log ); + my $cmd = Util::replaceParameters( $testCmd->{'script'}, $outfile, $testCmd, $log ); open(FH, ">$script") or die "Unable to open file $script to write script, $ERRNO\n"; print FH $cmd . "\n"; @@ -363,88 +315,6 @@ } -sub getPigCmd($$$) -{ - my ($self, $testCmd, $log) = @_; - - my @pigCmd; - - # set the PIG_CLASSPATH environment variable - my $pcp .= $testCmd->{'jythonjar'} if (defined($testCmd->{'jythonjar'})); - $pcp .= ":" . $testCmd->{'classpath'} if (defined($testCmd->{'classpath'})); - $pcp .= ":" . $testCmd->{'additionaljars'} if (defined($testCmd->{'additionaljars'})); - # Only add testconfigpath to PIG_CLASSPATH if HADOOP_HOME isn't defined - $pcp .= ":" . $testCmd->{'testconfigpath'} if ($testCmd->{'exectype'} ne "local"); #&& (! defined $ENV{'HADOOP_HOME'}); - $pcp .= ":" . $testCmd->{'hbaseconfigpath'} if ($testCmd->{'exectype'} ne "local" && defined($testCmd->{'hbaseconfigpath'} && $testCmd->{'hbaseconfigpath'} ne "")); - - # Set it in our current environment. It will get inherited by the IPC::Run - # command. - $ENV{'PIG_CLASSPATH'} = $pcp; - - @pigCmd = ("$testCmd->{'pigpath'}/bin/pig"); - - if (defined($testCmd->{'additionaljars'})) { - push(@pigCmd, '-Dpig.additional.jars='.$testCmd->{'additionaljars'}); - } - - if ($testCmd->{'exectype'} eq "local") { - push(@{$testCmd->{'java_params'}}, "-Xmx1024m"); - push(@pigCmd, ("-x", "local")); - } - - my $opts .= "-Dhive.metastore.uris=$testCmd->{'thriftserver'}"; - if (defined($testCmd->{'java_params'})) { - $opts = $opts . " " . join(" ", @{$testCmd->{'java_params'}}); - } - - $ENV{'PIG_OPTS'} = $ENV{'PIG_OPTS'} . " " . $opts; - - print $log "Returning Pig command " . join(" ", @pigCmd) . "\n"; - print $log "With PIG_CLASSPATH set to " . $ENV{'PIG_CLASSPATH'} . " and PIG_OPTS set to " . $ENV{'PIG_OPTS'} . "\n"; - return @pigCmd; -} - -sub dumpPigTable -{ - my ($self, $testCmd, $table, $log, $id) = @_; - my $subName = (caller(0))[3]; - - my %result; - - # Write the pig script to a file. - my $pigfile = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . $id . ".dump.pig"; - my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . $id . "dump.out"; - - open(FH, "> $pigfile") or die "Unable to open file $pigfile to write pig script, $ERRNO\n"; - print FH "a = load '$table' using org.apache.hcatalog.pig.HCatLoader(); store a into '$outfile';\n"; - close(FH); - - - # Build the command - my @baseCmd = $self->getPigCmd($testCmd, $log); - my @cmd = @baseCmd; - - push(@cmd, $pigfile); - - - # Run the command - print $log "Setting PIG_CLASSPATH to $ENV{'PIG_CLASSPATH'}\n"; - print $log "$0::$className::$subName INFO: Going to run pig command: @cmd\n"; - - IPC::Run::run(\@cmd, \undef, $log, $log) or - die "Failed running $pigfile\n"; - $result{'rc'} = $? >> 8; - - - # Get results from the command locally - my $localoutfile; - my $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . $id . ".dump.out"; - my $stores = $self->countStores($testCmd); - - $outfile = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log); - return $outfile; -} - sub runPig { my ($self, $testCmd, $log, $copyResults, $noFailOnFail) = @_; @@ -456,7 +326,7 @@ my $pigfile = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".pig"; my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out"; - my $pigcmd = $self->replaceParameters( $testCmd->{'pig'}, $outfile, $testCmd, $log ); + my $pigcmd = Util::replaceParameters( $testCmd->{'pig'}, $outfile, $testCmd, $log ); open(FH, "> $pigfile") or die "Unable to open file $pigfile to write pig script, $ERRNO\n"; print FH $pigcmd . "\n"; @@ -464,7 +334,7 @@ # Build the command - my @baseCmd = $self->getPigCmd($testCmd, $log); + my @baseCmd = Util::getPigCmd($testCmd, $log); my @cmd = @baseCmd; # Add option -l giving location for secondary logs @@ -485,8 +355,9 @@ # Run the command - print $log "Setting PIG_CLASSPATH to $ENV{'PIG_CLASSPATH'}\n"; print $log "$0::$className::$subName INFO: Going to run pig command: @cmd\n"; + print $log "With PIG_CLASSPATH set to $ENV{'PIG_CLASSPATH'}\n"; + print $log "and HADOOP_HOME set to $ENV{'HADOOP_HOME'}\n"; my $runrc = IPC::Run::run(\@cmd, \undef, $log, $log); Index: src/test/e2e/hcatalog/drivers/Util.pm =================================================================== --- src/test/e2e/hcatalog/drivers/Util.pm (revision 1310604) +++ src/test/e2e/hcatalog/drivers/Util.pm (working copy) @@ -27,12 +27,14 @@ package Util; use IPC::Run qw(run); +use strict; +use English; sub prepareHCat { my ($self, $testCmd, $log) = @_; my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out"; - my $hcatCmd = $self->replaceParameters( $testCmd->{'hcat_prep'}, $outfile, $testCmd, $log); + my $hcatCmd = replaceParameters( $testCmd->{'hcat_prep'}, $outfile, $testCmd, $log); my @hivefiles = (); my @outfiles = (); @@ -50,46 +52,6 @@ Util::runHCatCmdFromFile($testCmd, $log, $hivefiles[0]); } -############################################################################## -# Sub: setupHiveProperties -# -# Assure that necessary values are set in config in order to set Hive -# Java properties. -# -# Returns: -# Nothing -sub setupHiveProperties($$) -{ - my ($cfg, $log) = @_; - - # Set up values for the metastore - if (defined($cfg->{'metastore_thrift'}) && $cfg->{'metastore_thrift'} == 1) { - if (! defined $cfg->{'metastore_host'} || $cfg->{'metastore_host'} eq "") { - print $log "When using thrift, you must set the key " . - " 'metastore_host' to the machine your metastore is on\n"; - die "metastore_host is not set in existing.conf\n"; - } - - $cfg->{'metastore_connection'} = - "jdbc:$cfg->{'metastore_db'}://$cfg->{'metastore_host'}/hivemetastoredb?createDatabaseIfNotExist=true"; - - if (! defined $cfg->{'metastore_passwd'} || $cfg->{'metastore_passwd'} eq "") { - $cfg->{'metastore_passwd'} = 'hive'; - } - - if (! defined $cfg->{'metastore_port'} || $cfg->{'metastore_port'} eq "") { - $cfg->{'metastore_port'} = '9933'; - } - - $cfg->{'metastore_uri'} = - "thrift://$cfg->{'metastore_host'}:$cfg->{'metastore_port'}"; - } else { - $cfg->{'metastore_connection'} = - "jdbc:derby:;databaseName=metastore_db;create=true"; - $cfg->{'metastore_driver'} = "org.apache.derby.jdbc.EmbeddedDriver"; - } -} - sub getHadoopCmd { my ( $properties ) = @_; @@ -113,7 +75,8 @@ } push (@baseCmd, $cmd); - push (@baseCmd, '--config', $properties->{'testconfigpath'}) if defined($properties->{'testconfigpath'}); + push (@baseCmd, '--config', $properties->{'hadoopconfdir'}) + if defined($properties->{'hadoopconfdir'}); return @baseCmd; } @@ -148,31 +111,11 @@ $outfile = $log if (!defined($outfile)); $errfile = $log if (!defined($errfile)); - my @cmd = ("$cfg->{'hivehome'}/bin/hive"); - - # Add all of the modified properties we want to set -# push(@cmd, "--hiveconf", "hive.metastore.uris=$cfg->{'thriftserver'}"); -# push(@cmd, "--hiveconf", "hive.metastore.local=false"); - -# if( defined($cfg->{'metastore.principal'}) && ($cfg->{'metastore.principal'} =~ m/\S+/) -# && ($cfg->{'metastore.principal'} ne '${metastore.principal}')){ -# push(@cmd, "--hiveconf", "hive.metastore.sasl.enabled=true", "--hiveconf", "hive.metastore.kerberos.principal=$cfg->{'metastore.principal'}"); -# } else { -# push(@cmd, "--hiveconf", "hive.metastore.sasl.enabled=false"); -# } - - $ENV{'HIVE_CONF_DIR'} = "$cfg->{'hive.conf.dir'}"; - - if (defined($cfg->{'hive.additionaljars'})) { - $ENV{'HIVE_AUX_JARS_PATH'} = $cfg->{'hive.additionaljars'}; - } - -# if (defined($cfg->{'hiveconf'})) { -# foreach my $hc (@{$cfg->{'hiveconf'}}) { -# push(@cmd, "--hiveconf", $hc); -# } -# } - + my @cmd = ($cfg->{'hivebin'}); + + $ENV{'HIVE_CONF_DIR'} = $cfg->{'hcatconf'}; + $ENV{'HIVE_AUX_JARS_PATH'} = $cfg->{'hcatshare'}; + if (defined($cfg->{'hivecmdargs'})) { push(@cmd, @{$cfg->{'hivecmdargs'}}); } @@ -230,26 +173,28 @@ my ($cfg, $log, $sql, $outfile, $errfile, $noFailOnFail) = @_; if (!defined($ENV{'HADOOP_HOME'})) { - die "Cannot run hive when HADOOP_HOME environment variable is not set."; + die "Cannot run hcat when HADOOP_HOME environment variable is not set."; } $outfile = $log if (!defined($outfile)); $errfile = $log if (!defined($errfile)); # unset HADOOP_CLASSPATH - $ENV{'HADOOP_CLASSPATH'} = ""; - $ENV{'HADOOP_CLASSPATH'} = $cfg->{'pigjar'}; +# $ENV{'HADOOP_CLASSPATH'} = ""; + $ENV{'HADOOP_CLASSPATH'} = $cfg->{'hbaseconf'}; + $ENV{'HCAT_CLASSPATH'} = Util::getHBaseLibs($cfg, $log); my @cmd; if (defined($sql)) { - @cmd = ("$cfg->{'hcathome'}/bin/hcat", "-f", $sql); + @cmd = ("$cfg->{'hcatbin'}", "-f", $sql); } else { - @cmd = ("$cfg->{'hcathome'}/bin/hcat"); + @cmd = ("$cfg->{'hcatbin'}"); } my $envStr; for my $k (keys(%ENV)) { - $envStr .= $k . "=" . $ENV{$k} . " " if ($k =~ /HADOOP/ || $k =~ /HIVE/); + $envStr .= $k . "=" . $ENV{$k} . " " if ($k =~ /HADOOP/ || $k =~ /HIVE/ || + $k =~ /HCAT/); } $envStr .= " "; print $log "Going to run hcat command [" . join(" ", @cmd) . @@ -327,198 +272,33 @@ sub getPigCmd { - my $subName = (caller(0))[3]; - my $jarkey = shift; - my ( $properties ) = @_; - my $isPigSqlEnabled= 0; - my @baseCmd; - die "$0.$subName: null properties" if (! $properties ); + my ( $cfg, $log ) = @_; -show_call_stack(); - #UGLY HACK for pig sql support - if ( $jarkey =~ /testsql/ ) { + my @cmd = ("$cfg->{'pigbin'}"); - $isPigSqlEnabled= 1; - $jarkey = "testjar"; - + + # sets the queue, for exampel "grideng" + if(defined($cfg->{'queue'})) { + push( @cmd,'-Dmapred.job.queue.name='.$cfg->{'queue'}); } - - my $cmd; - if ( $properties->{'use-pig.pl'} ) { - # The directive gives that - # 1) the 'pig' command will be called, as opposed to java - # 2) the conf file has full control over what options and parameters are - # passed to pig. - # I.e. no parameters should be passed automatically by the script here. - # - # This allows for testing of the pig script as installed, and for testin of - # the pig script's options, including error testing. - -print 'use-pig.pl?????'; - - $cmd = $properties->{'gridstack.root'} . "/pig/" . $properties->{'pigTestBuildName'} . "/bin/pig"; - if ( ! -x "$cmd" ) { - print STDERR "\n$0::$subName WARNING: Can't find pig command: $cmd\n"; - $cmd = `which pig`; - chomp $cmd; - print STDERR "$0::$subName WARNING: Instead using command: $cmd\n"; - } - die "\n$0::$subName FATAL: Pig command does not exist: $cmd\n" if ( ! -x $cmd ); - push (@baseCmd, $cmd ); - - if(defined($properties->{'additionaljars'})) { - push( @baseCmd,'-Dpig.additional.jars='.$properties->{'additionaljars'}); - } - $ENV{'PIG_CLASSPATH'}=$properties->{'additionaljars'}; - - if ( $properties->{'use-pig.pl'} eq 'raw' ) { # add _no_ arguments automatically - # !!! - return @baseCmd; - } - - } else { - $cmd="java"; - -print 'not use-pig.pl?????'; - # Set JAVA options - - # User can provide only one of - # (-c ) OR (-testjar -testconfigpath ) - # "-c " is allowed only in non local mode - if(defined($properties->{'cluster.name'})) { - # use provided cluster - @baseCmd = ($cmd, '-c', $properties->{'cluster.name'}); - } else { - die "\n$0::$subName FATAL: The jar file name must be passed in at the command line or defined in the configuration file\n" if ( !defined( $properties->{$jarkey} ) ); - die "\n$0::$subName FATAL: The jar file does not exist.\n" . $properties->{$jarkey}."\n" if ( ! -e $properties->{$jarkey} ); + my $cp = Util::getHCatLibs($cfg, $log) . Util::getHiveLibsForPig($cfg, $log) . + Util::getHBaseLibs($cfg, $log); + push(@cmd, ('-Dpig.additional.jars='. $cp)); + $cp .= ':' . $cfg->{'hcatconf'}; + $cp .= ':' . $cfg->{'hbaseconf'}; + $ENV{'PIG_CLASSPATH'} = $cp; - # use user provided jar - my $classpath; + # sets the permissions on the jobtracker for the logs + push( @cmd,'-Dmapreduce.job.acl-view-job=*'); - if (defined $properties->{'jythonjar'}) { - $classpath = "$classpath:" . $properties->{'jythonjar'}; - } - if( $properties->{'exectype'} eq "local") { - # in local mode, we should not use - # any hadoop-site.xml - $classpath= "$classpath:" . $properties->{$jarkey}; - $classpath= "$classpath:$properties->{'classpath'}" if ( defined( $properties->{'classpath'} ) ); - @baseCmd = ($cmd, '-cp', $classpath, '-Xmx1024m'); - - } else { - - # non local mode, we also need to specify - # location of hadoop-site.xml - die "\n$0::$subName FATAL: The hadoop configuration file name must be passed in at the command line or defined in the configuration file\n" - if ( !defined( $properties->{'testconfigpath'} ) ); - die "\n$0::$subName FATAL $! " . $properties->{'testconfigpath'}."\n\n" - if (! -e $properties->{'testconfigpath'} ); - $classpath= "$classpath:" . $properties->{$jarkey}.":".$properties->{'testconfigpath'}; - $classpath= "$classpath:$properties->{'classpath'}" if ( defined( $properties->{'classpath'} ) ); - $classpath= "$classpath:$properties->{'howl.jar'}" if ( defined( $properties->{'howl.jar'} ) ); - @baseCmd = ($cmd, '-cp', $classpath ); - } - } - - # sets the queue, for exampel "grideng" - if(defined($properties->{'queue'})) { - push( @baseCmd,'-Dmapred.job.queue.name='.$properties->{'queue'}); - } - - if(defined($properties->{'additionaljars'})) { - push( @baseCmd,'-Dpig.additional.jars='.$properties->{'additionaljars'}); - } - - if( ( $isPigSqlEnabled == 1 ) ){ - - if(defined($properties->{'metadata.uri'})) { - push( @baseCmd, '-Dmetadata.uri='.$properties->{'metadata.uri'}); - } - - if(defined($properties->{'metadata.impl'})) { - push( @baseCmd, '-Dmetadata.impl='.$properties->{'metadata.impl'}); - }else{ - push( @baseCmd, '-Dmetadata.impl=org.apache.hadoop.owl.pig.metainterface.OwlPigMetaTables'); - } - } - - # Add howl support - if(defined($properties->{'howl.metastore.uri'})) { - push( @baseCmd, '-Dhowl.metastore.uri='.$properties->{'howl.metastore.uri'}); - } - - # Set local mode property - # if ( defined($properties->{'exectype'}) && $properties->{'exectype'}=~ "local" ) { - # Removed above 'if...' for Pig 8. - my $java=`which java`; - my $version=`file $java`; - if ( $version =~ '32-bit' ){ - push(@baseCmd,'-Djava.library.path='.$ENV{HADOOP_HOME}.'/lib/native/Linux-i386-32'); - } else { - push(@baseCmd,'-Djava.library.path='.$ENV{HADOOP_HOME}.'/lib/native/Linux-amd64-64'); - } - # } - - - # Add user provided java options if they exist - if (defined($properties->{'java_params'})) { - push(@baseCmd, @{$properties->{'java_params'}}); - } - - if(defined($properties->{'hod'})) { - push( @baseCmd, '-Dhod.server='); - } - - # sets the permissions on the jobtracker for the logs - push( @baseCmd,'-Dmapreduce.job.acl-view-job=*'); - - - # Add Main - push(@baseCmd, 'org.apache.pig.Main'); - # Set local mode PIG option - if ( defined($properties->{'exectype'}) && $properties->{'exectype'}=~ "local" ) { - push(@baseCmd, '-x'); - push(@baseCmd, 'local'); - } - - # Set Pig SQL options - if( ( $isPigSqlEnabled == 1 ) && defined($properties->{'metadata.uri'})) { - - if ( defined($properties->{'testoutpath'}) ) { - push( @baseCmd, '-u' ); - push( @baseCmd, $properties->{'testoutpath'} ); - } - - push( @baseCmd, '-s' ); - push( @baseCmd, '-f' ); - } - - } # end else of if use-pig.pl - - - # Add -latest or -useversion - if ( $cmd =~ 'pig$' ) { - # Add -latest, or -useversion if 'current' is not target build - if ( defined($properties->{'pigTestBuildName'})) { - if ($properties->{'pigTestBuildName'} eq 'latest') { - push(@baseCmd, '-latest'); - } elsif ($properties->{'pigTestBuildName'} ne 'current') { - push(@baseCmd, '-useversion', "$properties->{'pigTestBuildName'}"); - } - } - } elsif ( $cmd =~ 'java' ) { - - # is this ever used: ??? - # Add latest if it's there - if (defined($properties->{'latest'})) { - push(@baseCmd, '-latest'); - } + if ( defined($cfg->{'exectype'}) && $cfg->{'exectype'}=~ "local" ) { + push(@cmd, ('-x', 'local')); } - return @baseCmd; + return @cmd; } @@ -527,18 +307,18 @@ my $locale= shift; # $locale = "en_US.UTF-8" if ( !$locale ); $locale = "ja_JP.utf8" if ( !$locale ); - $ENV[LC_CTYPE]="$locale"; - $ENV[LC_NUMERIC]="$locale"; - $ENV[LC_TIME]="$locale"; - $ENV[LC_COLLATE]="$locale"; - $ENV[LC_MONETARY]="$locale"; - $ENV[LC_MESSAGES]="$locale"; - $ENV[LC_PAPER]="$locale"; - $ENV[LC_NAME]="$locale"; - $ENV[LC_ADDRESS]="$locale"; - $ENV[LC_TELEPHONE]="$locale"; - $ENV[LC_MEASUREMENT]="$locale"; - $ENV[LC_IDENTIFICATION]="$locale"; + $ENV['LC_CTYPE']="$locale"; + $ENV['LC_NUMERIC']="$locale"; + $ENV['LC_TIME']="$locale"; + $ENV['LC_COLLATE']="$locale"; + $ENV['LC_MONETARY']="$locale"; + $ENV['LC_MESSAGES']="$locale"; + $ENV['LC_PAPER']="$locale"; + $ENV['LC_NAME']="$locale"; + $ENV['LC_ADDRESS']="$locale"; + $ENV['LC_TELEPHONE']="$locale"; + $ENV['LC_MEASUREMENT']="$locale"; + $ENV['LC_IDENTIFICATION']="$locale"; } sub getLocaleCmd @@ -560,4 +340,154 @@ ."export LC_IDENTIFICATION=\"$locale\""; } +sub replaceParameters +{ + + my ($cmd, $outfile, $testCmd, $log) = @_; + + # $self +# $cmd =~ s/:LATESTOUTPUTPATH:/$self->{'latestoutputpath'}/g; + + # $outfile + $cmd =~ s/:OUTPATH:/$outfile/g; + + # $ENV + $cmd =~ s/:PIGHARNESS:/$ENV{HARNESS_ROOT}/g; + + # $testCmd + $cmd =~ s/:INPATH:/$testCmd->{'inpathbase'}/g; + $cmd =~ s/:OUTPATH:/$outfile/g; + $cmd =~ s/:FUNCPATH:/$testCmd->{'funcjarPath'}/g; + $cmd =~ s/:PIGPATH:/$testCmd->{'pighome'}/g; + $cmd =~ s/:RUNID:/$testCmd->{'UID'}/g; + $cmd =~ s/:USRHOMEPATH:/$testCmd->{'userhomePath'}/g; + $cmd =~ s/:MAPREDJARS:/$testCmd->{'mapredjars'}/g; + $cmd =~ s/:SCRIPTHOMEPATH:/$testCmd->{'scriptPath'}/g; + $cmd =~ s/:DBUSER:/$testCmd->{'dbuser'}/g; + $cmd =~ s/:DBNAME:/$testCmd->{'dbdb'}/g; +# $cmd =~ s/:LOCALINPATH:/$testCmd->{'localinpathbase'}/g; +# $cmd =~ s/:LOCALOUTPATH:/$testCmd->{'localoutpathbase'}/g; +# $cmd =~ s/:LOCALTESTPATH:/$testCmd->{'localpathbase'}/g; + $cmd =~ s/:BMPATH:/$testCmd->{'benchmarkPath'}/g; + $cmd =~ s/:TMP:/$testCmd->{'tmpPath'}/g; + $cmd =~ s/:HDFSTMP:/tmp\/$testCmd->{'runid'}/g; + $cmd =~ s/:HCAT_JAR:/$testCmd->{'libjars'}/g; + + if ( $testCmd->{'hadoopSecurity'} eq "secure" ) { + $cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteSecureCluster'}/g; + } else { + $cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteNotSecureCluster'}/g; + } + + return $cmd; +} + +sub getHiveLibs($$) +{ + my ($cfg, $log) = @_; + + my $cp; + opendir(LIB, $cfg->{'hivelib'}) or die "Cannot open $cfg->{'hivelib'}, $!\n"; + my @jars = readdir(LIB); + foreach (@jars) { + /\.jar$/ && do { + $cp .= $cfg->{'hivelib'} . '/' . $_ . ':'; + }; + } + closedir(LIB); + return $cp; +} + +# Pig needs a limited set of the Hive libs, since they include some of the same jars +# and we get version mismatches if it picks up all the libraries. +sub getHiveLibsForPig($$) +{ + my ($cfg, $log) = @_; + + my $cp; + opendir(LIB, $cfg->{'hivelib'}) or die "Cannot open $cfg->{'hivelib'}, $!\n"; + my @jars = readdir(LIB); + foreach (@jars) { + /hive-.*\.jar$/ && do { + $cp .= $cfg->{'hivelib'} . '/' . $_ . ':'; + }; + /libfb303.jar/ && do { + $cp .= $cfg->{'hivelib'} . '/' . $_ . ':'; + }; + /libthrift.jar/ && do { + $cp .= $cfg->{'hivelib'} . '/' . $_ . ':'; + }; + /datanucleus-.*\.jar$/ && do { + $cp .= $cfg->{'hivelib'} . '/' . $_ . ':'; + }; + /jdo2-api-.*\.jar$/ && do { + $cp .= $cfg->{'hivelib'} . '/' . $_ . ':'; + }; + /commons-dbcp-.*\.jar$/ && do { + $cp .= $cfg->{'hivelib'} . '/' . $_ . ':'; + }; + /commons-pool-.*\.jar$/ && do { + $cp .= $cfg->{'hivelib'} . '/' . $_ . ':'; + }; +# /hbase-.*\.jar$/ && do { +# $cp .= $cfg->{'hivelib'} . '/' . $_ . ':'; +# }; +# /zookeeper-.*\.jar$/ && do { +# $cp .= $cfg->{'hivelib'} . '/' . $_ . ':'; +# }; + } + closedir(LIB); + return $cp; +} + +sub getHBaseLibs($$) +{ + my ($cfg, $log) = @_; + + my $cp; + opendir(LIB, $cfg->{'hbaselibdir'}) or die "Cannot open $cfg->{'hbaselibdir'}, $!\n"; + my @jars = readdir(LIB); + foreach (@jars) { + /hbase-.*\.jar$/ && do { + $cp .= $cfg->{'hbaselibdir'} . '/' . $_ . ':'; + }; + } + closedir(LIB); + opendir(LIB, $cfg->{'zklibdir'}) or die "Cannot open $cfg->{'zklibdir'}, $!\n"; + my @jars = readdir(LIB); + foreach (@jars) { + /zookeeper.*\.jar$/ && do { + $cp .= $cfg->{'zklibdir'} . '/' . $_ . ':'; + }; + } + closedir(LIB); + return $cp; +} + + +sub getHCatLibs($$) +{ + my ($cfg, $log) = @_; + + my $cp; + opendir(LIB, $cfg->{'hcatshare'}) or die "Cannot open $cfg->{'hcatshare'}, $!\n"; + my @jars = readdir(LIB); + foreach (@jars) { + /hcatalog-[0-9].*\.jar$/ && do { + $cp .= $cfg->{'hcatshare'} . '/' . $_ . ':'; + }; + } + closedir(LIB); + opendir(LIB, $cfg->{'hcatlib'}) or die "Cannot open $cfg->{'hcatlib'}, $!\n"; + my @jars = readdir(LIB); + foreach (@jars) { + /hbase-storage-handler.*\.jar$/ && do { + $cp .= $cfg->{'hcatlib'} . '/' . $_ . ':'; + }; + } + closedir(LIB); + return $cp; +} + + 1; Index: src/test/e2e/hcatalog/build.xml =================================================================== --- src/test/e2e/hcatalog/build.xml (revision 1310604) +++ src/test/e2e/hcatalog/build.xml (working copy) @@ -17,9 +17,8 @@ - - - + @@ -41,12 +40,12 @@ - - - - + + + + @@ -60,13 +59,15 @@ - + + - + + @@ -142,83 +143,71 @@ - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + @@ -241,28 +230,15 @@ - + - - - - - - - - - - - - - - - - - - - + + + + + + @@ -275,29 +251,20 @@ - - + - - - - - - - - - - - - - + + + + + @@ -319,35 +286,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - -