Index: src/test/e2e/hcatalog/conf/existing_deployer.conf =================================================================== --- src/test/e2e/hcatalog/conf/existing_deployer.conf (revision 1304625) +++ src/test/e2e/hcatalog/conf/existing_deployer.conf (working copy) @@ -26,7 +26,7 @@ # hadoop values 'hadoopdir' => $ENV{'PH_CLUSTER'}, - 'hcat_data_dir' => '/user/hcat/test/data', + 'hcat_data_dir' => ("$ENV{'PH_HDFS_BASE'}" || '/user/hcat').'/test/data', # db values # 'dbuser' => 'pigtester', Index: src/test/e2e/hcatalog/conf/default.conf =================================================================== --- src/test/e2e/hcatalog/conf/default.conf (revision 1304625) +++ src/test/e2e/hcatalog/conf/default.conf (working copy) @@ -17,11 +17,6 @@ my $me = `whoami`; chomp $me; -#get the hive version from the hive build.properties -my $hive_build_properties = "$ENV{HCAT_ROOT}/hive/external/build.properties"; -my $hive_version = `cat $hive_build_properties | grep '^version=' | cut -d= -f2`; -chomp $hive_version; - # The contents of this file can be rewritten to fit your installation. # Also, you can define the following environment variables and set things up as in the test setup # PH_ROOT Root directory where test harness is installed @@ -30,10 +25,12 @@ # PH_CLUSTER Root directory for cluster being used # HCAT_ROOT Root directory for hcat version being used +my $hdfsBase = $ENV{'PH_HDFS_BASE'} || "/user/hcat"; + $cfg = { #HDFS - 'inpathbase' => '/user/hcat/tests/data' - , 'outpathbase' => '/user/hcat/out' + 'inpathbase' => "$hdfsBase/tests/data" + , 'outpathbase' => "$hdfsBase/out" #LOCAL , 'localinpathbase' => "$ENV{PH_LOCAL}/in" @@ -48,10 +45,10 @@ , 'jythonjar' => "$ENV{PH_JYTHON_JAR}" #TESTDB - , 'dbuser' => 'hcattest' - , 'dbhost' => 'localhost' - , 'dbpasswd' => 'hcattest' - , 'dbdb' => 'hcattestdb' + , 'dbuser' => "$ENV{'PH_DBUSER'}" || 'hcattest' + , 'dbhost' => "$ENV{'PH_DBHOST'}" || 'localhost' + , 'dbpasswd' => "$ENV{'PH_DBPASSWD'}" || 'hcattest' + , 'dbdb' => "$ENV{'PH_DBDB'}" || 'hcattestdb' #COMMON , 'metastore.principal' => "$ENV{METASTORE_PRINCIPAL}" @@ -62,21 +59,18 @@ , 'hcat_data_dir' => '/user/hcat/tests/data' , 'hivehome' => $ENV{'PH_HIVE_HOME'} , 'hcathome' => $ENV{'HCAT_INSTALL_DIR'} - , 'hcatalog.jar' => "$ENV{HCAT_JAR},$ENV{HIVE_ROOT}/build/dist/lib/hive-serde-$hive_version.jar,$ENV{HIVE_ROOT}/build/dist/lib/hive-exec-$hive_version.jar,$ENV{PIG_HOME}/pig-0.9.2-withouthadoop.jar,$ENV{HIVE_ROOT}/build/dist/lib/hive-metastore-$hive_version.jar,$ENV{HIVE_ROOT}/build/dist/lib/libfb303-0.7.0.jar,$ENV{HIVE_ROOT}/build/dist/lib/jdo2-api-2.3-ec.jar,$ENV{HCAT_ROOT}/storage-handlers/hbase/build/ivy/lib/hbase-storage-handler/hbase-0.92.0.jar,$ENV{HCAT_ROOT}/storage-handlers/hbase/build/ivy/lib/hbase-storage-handler/zookeeper-3.4.3.jar,$ENV{HCAT_ROOT}/storage-handlers/hbase/build/hbase-storage-handler/hbase-storage-handler-0.1.0.jar,$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-hbase-handler-$hive_version.jar" - + , 'hcatalog.jar' => $ENV{'HADOOP_LIBJARS'} #PIG , 'testconfigpath' => "$ENV{PH_CLUSTER}" , 'hadoopbin' => "$ENV{PH_CLUSTER_BIN}" , 'funcjarPath' => "$ENV{PH_ROOT}/lib/java" , 'paramPath' => "$ENV{PH_ROOT}/paramfiles" , 'pigpath' => "$ENV{PIG_HOME}" - , 'pigjar' => "$ENV{PIG_JAR}" # Pig jar that doesn't have Antlr , 'oldpigpath' => "$ENV{PH_OLDPIG}" - , 'additionaljars' => "$ENV{HCAT_ROOT}/build/hcatalog/hcatalog-0.5.0.jar:$ENV{HCAT_ROOT}/hive/external/build/metastore/hive-metastore-$hive_version.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/libthrift.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-exec-$hive_version.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/libfb303.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/jdo2-api-2.3-ec.jar::$ENV{HCAT_ROOT}/storage-handlers/hbase/build/ivy/lib/hbase-storage-handler/hbase-0.92.0.jar:$ENV{HCAT_ROOT}/storage-handlers/hbase/build/ivy/lib/hbase-storage-handler/zookeeper-3.4.3.jar:$ENV{HCAT_ROOT}/storage-handlers/hbase/build/hbase-storage-handler/hbase-storage-handler-0.1.0.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-hbase-handler-$hive_version.jar:$ENV{'HCAT_INSTALL_DIR'}/etc/hcatalog" + , 'additionaljars' => "$ENV{PIG_ADDITIONAL_JARS}" #HADOOP - , 'hadoopHome' => "$ENV{HCAT_ROOT}/lib" - , 'hadoop_classpath' => "$ENV{HCAT_EXTRA_JARS}" + , 'UNUSEDhadoopHome' => "$ENV{HCAT_ROOT}/lib" , 'userhomePath' => "$ENV{HOME}" , 'local.bin' => '/usr/bin' , 'logDir' => "$ENV{PH_OUT}/log" @@ -87,6 +81,7 @@ , 'hive_bin_location' => "$ENV{HIVE_ROOT}/build/dist/bin" , 'hbaseconfigpath' => "$ENV{HBASE_CONF_DIR}" , 'hivehome' => "$ENV{HIVE_HOME}" - , 'hive.additionaljars' => "$ENV{HCAT_ROOT}/build/hcatalog/hcatalog-0.5.0.jar" + , 'hive.additionaljars' => "$ENV{HCAT_JARS}" + , 'hive.conf.dir' => "$ENV{HIVE_CONF_DIR}" || "$ENV{'HCAT_INSTALL_DIR'}/etc/hcatalog" }; Index: src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm =================================================================== --- src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm (revision 1304625) +++ src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm (working copy) @@ -335,25 +335,18 @@ push(@cmd, split(/ +/,$hadoopcmd)); # Set HADOOP_CLASSPATH environment variable if provided - if (defined($testCmd->{'hadoop_classpath'})) { - #my $hadoop_classpath = $self->replaceParameters( $testCmd->{'hadoop_classpath'}, $outfile, $testCmd, $log ); - # TODO This is a total mess. Half the jars we need are specified in hcatalog.jar, which is set in the default.conf file, and half are set in hadoop_classpath, which - # has to be passed on the command line. And most of the jars are Hive jars, thus they don't fit in the list of either hcatalog.jar or hadoop_classpath. We need to - # make sense of this. - my $cp = $testCmd->{'hcatalog.jar'} . ":" . $testCmd->{'hadoop_classpath'}; - $cp =~ s/,/:/g; - $cp .= ":" . Util::findPigWithoutHadoopJar($testCmd, $log); - # Add in the hcat config file - $cp .= ":" . $testCmd->{'hcathome'} . "/etc/hcatalog"; - $ENV{'HADOOP_CLASSPATH'} = $cp; - } + my $cp = $testCmd->{'hcatalog.jar'}; + $cp =~ s/,/:/g; + # Add in the hcat config file + $cp .= ":" . $testCmd->{'hive.conf.dir'}; + $ENV{'HADOOP_CLASSPATH'} = $cp; if (defined($testCmd->{'hbaseconfigpath'})) { $ENV{'HADOOP_CLASSPATH'} = "$ENV{'HADOOP_CLASSPATH'}:$testCmd->{'hbaseconfigpath'}"; } if (defined($testCmd->{'metastore.principal'}) && ($testCmd->{'metastore.principal'} =~ m/\S+/)) { - $ENV{'HADOOP_OPTS'} = "-Dhive.metastore.kerberos.principal=" . $testCmd->{'metastore.principal'}; + $ENV{'HADOOP_OPTS'} = "$ENV{'HADOOP_OPTS'} -Dhive.metastore.kerberos.principal=" . $testCmd->{'metastore.principal'}; $ENV{'HADOOP_CLIENT_OPTS'} = "-Dhive.metastore.kerberos.principal=" . $testCmd->{'metastore.principal'}; } @@ -367,6 +360,7 @@ open(FH, ">$script") or die "Unable to open file $script to write script, $ERRNO\n"; print FH join (" ", @cmd) . "\n"; close(FH); + my @result=`chmod +x $script`; # Run the command Index: src/test/e2e/hcatalog/drivers/Util.pm =================================================================== --- src/test/e2e/hcatalog/drivers/Util.pm (revision 1304625) +++ src/test/e2e/hcatalog/drivers/Util.pm (working copy) @@ -161,7 +161,7 @@ # push(@cmd, "--hiveconf", "hive.metastore.sasl.enabled=false"); # } - $ENV{'HIVE_CONF_DIR'} = "$cfg->{'hcathome'}/etc/hcatalog/"; + $ENV{'HIVE_CONF_DIR'} = "$cfg->{'hive.conf.dir'}"; if (defined($cfg->{'hive.additionaljars'})) { $ENV{'HIVE_AUX_JARS_PATH'} = $cfg->{'hive.additionaljars'}; @@ -560,13 +560,4 @@ ."export LC_IDENTIFICATION=\"$locale\""; } -sub findPigWithoutHadoopJar($$) -{ - my ($cfg, $log) = @_; - - my $jar = `ls $cfg->{'pigpath'}/pig-*withouthadoop.jar`; - chomp $jar; - return $jar; -} - 1; Index: src/test/e2e/hcatalog/build.xml =================================================================== --- src/test/e2e/hcatalog/build.xml (revision 1304625) +++ src/test/e2e/hcatalog/build.xml (working copy) @@ -20,19 +20,7 @@ - - - - - - - - - - - - @@ -53,6 +41,8 @@ + + @@ -158,6 +148,77 @@ unless="harness.hadoop.home"/> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -176,20 +237,21 @@ + - - + + - - - + + + @@ -198,8 +260,9 @@ - + + @@ -221,14 +284,13 @@ - - + + -