Index: src/test/e2e/hcatalog/conf/existing_deployer.conf
===================================================================
--- src/test/e2e/hcatalog/conf/existing_deployer.conf (revision 1304625)
+++ src/test/e2e/hcatalog/conf/existing_deployer.conf (working copy)
@@ -26,7 +26,7 @@
# hadoop values
'hadoopdir' => $ENV{'PH_CLUSTER'},
- 'hcat_data_dir' => '/user/hcat/test/data',
+ 'hcat_data_dir' => ("$ENV{'PH_HDFS_BASE'}" || '/user/hcat').'/test/data',
# db values
# 'dbuser' => 'pigtester',
Index: src/test/e2e/hcatalog/conf/default.conf
===================================================================
--- src/test/e2e/hcatalog/conf/default.conf (revision 1304625)
+++ src/test/e2e/hcatalog/conf/default.conf (working copy)
@@ -17,11 +17,6 @@
my $me = `whoami`;
chomp $me;
-#get the hive version from the hive build.properties
-my $hive_build_properties = "$ENV{HCAT_ROOT}/hive/external/build.properties";
-my $hive_version = `cat $hive_build_properties | grep '^version=' | cut -d= -f2`;
-chomp $hive_version;
-
# The contents of this file can be rewritten to fit your installation.
# Also, you can define the following environment variables and set things up as in the test setup
# PH_ROOT Root directory where test harness is installed
@@ -30,10 +25,12 @@
# PH_CLUSTER Root directory for cluster being used
# HCAT_ROOT Root directory for hcat version being used
+my $hdfsBase = $ENV{'PH_HDFS_BASE'} || "/user/hcat";
+
$cfg = {
#HDFS
- 'inpathbase' => '/user/hcat/tests/data'
- , 'outpathbase' => '/user/hcat/out'
+ 'inpathbase' => "$hdfsBase/tests/data"
+ , 'outpathbase' => "$hdfsBase/out"
#LOCAL
, 'localinpathbase' => "$ENV{PH_LOCAL}/in"
@@ -48,10 +45,10 @@
, 'jythonjar' => "$ENV{PH_JYTHON_JAR}"
#TESTDB
- , 'dbuser' => 'hcattest'
- , 'dbhost' => 'localhost'
- , 'dbpasswd' => 'hcattest'
- , 'dbdb' => 'hcattestdb'
+ , 'dbuser' => "$ENV{'PH_DBUSER'}" || 'hcattest'
+ , 'dbhost' => "$ENV{'PH_DBHOST'}" || 'localhost'
+ , 'dbpasswd' => "$ENV{'PH_DBPASSWD'}" || 'hcattest'
+ , 'dbdb' => "$ENV{'PH_DBDB'}" || 'hcattestdb'
#COMMON
, 'metastore.principal' => "$ENV{METASTORE_PRINCIPAL}"
@@ -62,21 +59,18 @@
, 'hcat_data_dir' => '/user/hcat/tests/data'
, 'hivehome' => $ENV{'PH_HIVE_HOME'}
, 'hcathome' => $ENV{'HCAT_INSTALL_DIR'}
- , 'hcatalog.jar' => "$ENV{HCAT_JAR},$ENV{HIVE_ROOT}/build/dist/lib/hive-serde-$hive_version.jar,$ENV{HIVE_ROOT}/build/dist/lib/hive-exec-$hive_version.jar,$ENV{PIG_HOME}/pig-0.9.2-withouthadoop.jar,$ENV{HIVE_ROOT}/build/dist/lib/hive-metastore-$hive_version.jar,$ENV{HIVE_ROOT}/build/dist/lib/libfb303-0.7.0.jar,$ENV{HIVE_ROOT}/build/dist/lib/jdo2-api-2.3-ec.jar,$ENV{HCAT_ROOT}/storage-handlers/hbase/build/ivy/lib/hbase-storage-handler/hbase-0.92.0.jar,$ENV{HCAT_ROOT}/storage-handlers/hbase/build/ivy/lib/hbase-storage-handler/zookeeper-3.4.3.jar,$ENV{HCAT_ROOT}/storage-handlers/hbase/build/hbase-storage-handler/hbase-storage-handler-0.1.0.jar,$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-hbase-handler-$hive_version.jar"
-
+ , 'hcatalog.jar' => $ENV{'HADOOP_LIBJARS'}
#PIG
, 'testconfigpath' => "$ENV{PH_CLUSTER}"
, 'hadoopbin' => "$ENV{PH_CLUSTER_BIN}"
, 'funcjarPath' => "$ENV{PH_ROOT}/lib/java"
, 'paramPath' => "$ENV{PH_ROOT}/paramfiles"
, 'pigpath' => "$ENV{PIG_HOME}"
- , 'pigjar' => "$ENV{PIG_JAR}" # Pig jar that doesn't have Antlr
, 'oldpigpath' => "$ENV{PH_OLDPIG}"
- , 'additionaljars' => "$ENV{HCAT_ROOT}/build/hcatalog/hcatalog-0.5.0.jar:$ENV{HCAT_ROOT}/hive/external/build/metastore/hive-metastore-$hive_version.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/libthrift.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-exec-$hive_version.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/libfb303.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/jdo2-api-2.3-ec.jar::$ENV{HCAT_ROOT}/storage-handlers/hbase/build/ivy/lib/hbase-storage-handler/hbase-0.92.0.jar:$ENV{HCAT_ROOT}/storage-handlers/hbase/build/ivy/lib/hbase-storage-handler/zookeeper-3.4.3.jar:$ENV{HCAT_ROOT}/storage-handlers/hbase/build/hbase-storage-handler/hbase-storage-handler-0.1.0.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-hbase-handler-$hive_version.jar:$ENV{'HCAT_INSTALL_DIR'}/etc/hcatalog"
+ , 'additionaljars' => "$ENV{PIG_ADDITIONAL_JARS}"
#HADOOP
- , 'hadoopHome' => "$ENV{HCAT_ROOT}/lib"
- , 'hadoop_classpath' => "$ENV{HCAT_EXTRA_JARS}"
+ , 'UNUSEDhadoopHome' => "$ENV{HCAT_ROOT}/lib"
, 'userhomePath' => "$ENV{HOME}"
, 'local.bin' => '/usr/bin'
, 'logDir' => "$ENV{PH_OUT}/log"
@@ -87,6 +81,7 @@
, 'hive_bin_location' => "$ENV{HIVE_ROOT}/build/dist/bin"
, 'hbaseconfigpath' => "$ENV{HBASE_CONF_DIR}"
, 'hivehome' => "$ENV{HIVE_HOME}"
- , 'hive.additionaljars' => "$ENV{HCAT_ROOT}/build/hcatalog/hcatalog-0.5.0.jar"
+ , 'hive.additionaljars' => "$ENV{HCAT_JARS}"
+ , 'hive.conf.dir' => "$ENV{HIVE_CONF_DIR}" || "$ENV{'HCAT_INSTALL_DIR'}/etc/hcatalog"
};
Index: src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
===================================================================
--- src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm (revision 1304625)
+++ src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm (working copy)
@@ -335,25 +335,18 @@
push(@cmd, split(/ +/,$hadoopcmd));
# Set HADOOP_CLASSPATH environment variable if provided
- if (defined($testCmd->{'hadoop_classpath'})) {
- #my $hadoop_classpath = $self->replaceParameters( $testCmd->{'hadoop_classpath'}, $outfile, $testCmd, $log );
- # TODO This is a total mess. Half the jars we need are specified in hcatalog.jar, which is set in the default.conf file, and half are set in hadoop_classpath, which
- # has to be passed on the command line. And most of the jars are Hive jars, thus they don't fit in the list of either hcatalog.jar or hadoop_classpath. We need to
- # make sense of this.
- my $cp = $testCmd->{'hcatalog.jar'} . ":" . $testCmd->{'hadoop_classpath'};
- $cp =~ s/,/:/g;
- $cp .= ":" . Util::findPigWithoutHadoopJar($testCmd, $log);
- # Add in the hcat config file
- $cp .= ":" . $testCmd->{'hcathome'} . "/etc/hcatalog";
- $ENV{'HADOOP_CLASSPATH'} = $cp;
- }
+ my $cp = $testCmd->{'hcatalog.jar'};
+ $cp =~ s/,/:/g;
+ # Add in the hcat config file
+ $cp .= ":" . $testCmd->{'hive.conf.dir'};
+ $ENV{'HADOOP_CLASSPATH'} = $cp;
if (defined($testCmd->{'hbaseconfigpath'})) {
$ENV{'HADOOP_CLASSPATH'} = "$ENV{'HADOOP_CLASSPATH'}:$testCmd->{'hbaseconfigpath'}";
}
if (defined($testCmd->{'metastore.principal'}) && ($testCmd->{'metastore.principal'} =~ m/\S+/)) {
- $ENV{'HADOOP_OPTS'} = "-Dhive.metastore.kerberos.principal=" . $testCmd->{'metastore.principal'};
+ $ENV{'HADOOP_OPTS'} = "$ENV{'HADOOP_OPTS'} -Dhive.metastore.kerberos.principal=" . $testCmd->{'metastore.principal'};
$ENV{'HADOOP_CLIENT_OPTS'} = "-Dhive.metastore.kerberos.principal=" . $testCmd->{'metastore.principal'};
}
@@ -367,6 +360,7 @@
open(FH, ">$script") or die "Unable to open file $script to write script, $ERRNO\n";
print FH join (" ", @cmd) . "\n";
close(FH);
+
my @result=`chmod +x $script`;
# Run the command
Index: src/test/e2e/hcatalog/drivers/Util.pm
===================================================================
--- src/test/e2e/hcatalog/drivers/Util.pm (revision 1304625)
+++ src/test/e2e/hcatalog/drivers/Util.pm (working copy)
@@ -161,7 +161,7 @@
# push(@cmd, "--hiveconf", "hive.metastore.sasl.enabled=false");
# }
- $ENV{'HIVE_CONF_DIR'} = "$cfg->{'hcathome'}/etc/hcatalog/";
+ $ENV{'HIVE_CONF_DIR'} = "$cfg->{'hive.conf.dir'}";
if (defined($cfg->{'hive.additionaljars'})) {
$ENV{'HIVE_AUX_JARS_PATH'} = $cfg->{'hive.additionaljars'};
@@ -560,13 +560,4 @@
."export LC_IDENTIFICATION=\"$locale\"";
}
-sub findPigWithoutHadoopJar($$)
-{
- my ($cfg, $log) = @_;
-
- my $jar = `ls $cfg->{'pigpath'}/pig-*withouthadoop.jar`;
- chomp $jar;
- return $jar;
-}
-
1;
Index: src/test/e2e/hcatalog/build.xml
===================================================================
--- src/test/e2e/hcatalog/build.xml (revision 1304625)
+++ src/test/e2e/hcatalog/build.xml (working copy)
@@ -20,19 +20,7 @@
-
-
-
-
-
-
-
-
-
-
-
-
@@ -53,6 +41,8 @@
+
+
@@ -158,6 +148,77 @@
unless="harness.hadoop.home"/>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -176,20 +237,21 @@
+
-
-
+
+
-
-
-
+
+
+
@@ -198,8 +260,9 @@
-
+
+
@@ -221,14 +284,13 @@
-
-
+
+
-