diff --git a/hcatalog/src/test/e2e/templeton/README.txt b/hcatalog/src/test/e2e/templeton/README.txt
index c461a47..f4829de 100644
--- a/hcatalog/src/test/e2e/templeton/README.txt
+++ b/hcatalog/src/test/e2e/templeton/README.txt
@@ -124,9 +124,12 @@ You may also need to adjust the following in Pig's build.xml as needed:
true
-8.Sqoop test require JDBC jar to be placed on HDFS for whichever DB the test is configured with,
+9. Sqoop test require JDBC jar to be placed on HDFS for whichever DB the test is configured with,
for example mysql-connector-java-5.1.30-bin.jar.
+10. One of the UTF-8 test requires a jar with utf8 characters in its name to be present in HDFS. This is automated in deploy_e2e_test
+${HADOOP_HOME}/bin/hdfs dfs -put ${HADOOP_HOME}/share/hadoop/mapreduce/hadoop-mapreduce-examples-${HADOOP_VERSION}.jar webhcate2e/hadoop_examples_䶴ㄩ鼾丄狜〇_2_2_0.jar
+
****
**** See deployers/ for scripts that automate a lot of the set up.
****
diff --git a/hcatalog/src/test/e2e/templeton/build.xml b/hcatalog/src/test/e2e/templeton/build.xml
index eefbd7a..4c6ca5f 100644
--- a/hcatalog/src/test/e2e/templeton/build.xml
+++ b/hcatalog/src/test/e2e/templeton/build.xml
@@ -120,6 +120,7 @@
+
diff --git a/hcatalog/src/test/e2e/templeton/deployers/deploy_e2e_artifacts.sh b/hcatalog/src/test/e2e/templeton/deployers/deploy_e2e_artifacts.sh
index f446424..2b11a82 100755
--- a/hcatalog/src/test/e2e/templeton/deployers/deploy_e2e_artifacts.sh
+++ b/hcatalog/src/test/e2e/templeton/deployers/deploy_e2e_artifacts.sh
@@ -33,6 +33,9 @@ ${HADOOP_HOME}/bin/hdfs dfs -put ${PROJ_HOME}/hcatalog/src/test/e2e/templeton/in
#$HADOOP_HOME/bin/hadoop fs -put hadoop-examples-1.2.1.jar webhcate2e/hexamples.jar
#$HADOOP_HOME/bin/hadoop fs -put hadoop-examples-1.2.1.jar webhcate2e/hclient.jar
+#For utf8 test(mapreduce) we need a jar with utf-8 characters in the name
+${HADOOP_HOME}/bin/hdfs dfs -put ${HADOOP_HOME}/share/hadoop/mapreduce/hadoop-mapreduce-examples-${HADOOP_VERSION}.jar webhcate2e/hadoop_examples_䶴ㄩ鼾丄狜〇_2_2_0.jar
+
#For hadoop2 there are 2 separate jars
${HADOOP_HOME}/bin/hdfs dfs -put ${HADOOP_HOME}/share/hadoop/mapreduce/hadoop-mapreduce-examples-${HADOOP_VERSION}.jar webhcate2e/hexamples.jar
${HADOOP_HOME}/bin/hdfs dfs -put ${HADOOP_HOME}/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-${HADOOP_VERSION}.jar webhcate2e/hclient.jar
@@ -65,3 +68,12 @@ ${HADOOP_HOME}/bin/hdfs dfs -put /Users/ekoifman/dev/hive/build.sh /tmp/tezin
#check what got deployed
${HADOOP_HOME}/bin/hdfs dfs -ls -R /apps webhcate2e /user/templeton /user/hive/warehouse
+
+
+#rename input files for utf8 tests and move them to HDFS
+current_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+inpdir_path=`dirname $current_dir`
+mkdir -p test_utf8_inpdir
+cp $inpdir_path/inpdir/artofwar.txt test_utf8_inpdir/artof䶴ㄩ鼾丄狜〇war.txt
+cp $inpdir_path/inpdir/PigJoinwork.pig test_utf8_inpdir/PigJoin䶴ㄩ鼾丄狜〇work.pig
+${HADOOP_HOME}/bin/hdfs dfs -put test_utf8_inpdir/* webhcate2e
\ No newline at end of file
diff --git a/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm b/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm
index 2eac653..8477421 100644
--- a/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm
+++ b/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm
@@ -37,6 +37,7 @@ package TestDriverCurl;
use Storable qw(dclone);
use File::Glob ':glob';
use JSON::Path;
+use utf8;
my $passedStr = 'passed';
my $failedStr = 'failed';
@@ -925,12 +926,38 @@ sub compare
$result = 0;
next;
}
-
+ my $exp_userargsvalue;
+ my $r_userargsvalue;
+ if(ref($exp_userargs{$key}) eq "ARRAY"){
+ my @values = $exp_userargs{$key};
+ my $num_values = @values;
+
+ for(my $i=0;$i<=$num_values;$i++){
+ if (utf8::is_utf8($exp_userargs{$key}[$i])){
+ $exp_userargs{$key}[$i] = utf8::decode($exp_userargs{$key}[$i]);
+ $r_userargs{$key}[$i] = utf8::decode($r_userargs{$key}[$i]);
+ }
+ }
+ $exp_userargsvalue = $exp_userargs{$key};
+ $r_userargsvalue = $r_userargs{$key};
+ }
+ else {
+ if (utf8::is_utf8($exp_userargs{$key}))
+ {
+ $exp_userargsvalue = utf8::decode($exp_userargs{$key});
+ $r_userargsvalue = utf8::decode($r_userargs{$key});
+ }
+ else
+ {
+ $exp_userargsvalue = $exp_userargs{$key};
+ $r_userargsvalue = $r_userargs{$key};
+ }
+ }
print $log "$0::$subName DEBUG comparing expected "
. " $key ->" . dump($exp_userargs{$key})
. " With result $key ->" . dump($r_userargs{$key}) . "\n";
- if (!Compare($exp_userargs{$key}, $r_userargs{$key})) {
+ if (!Compare($exp_userargsvalue, $r_userargsvalue)) {
print $log "$0::$subName WARN check failed:"
. " json compare failed. For field "
. "$key, regex <" . dump($r_userargs{$key})
diff --git a/hcatalog/src/test/e2e/templeton/inpdir/PigJoinwork.pig b/hcatalog/src/test/e2e/templeton/inpdir/PigJoinwork.pig
index e69de29..04e127b 100755
--- a/hcatalog/src/test/e2e/templeton/inpdir/PigJoinwork.pig
+++ b/hcatalog/src/test/e2e/templeton/inpdir/PigJoinwork.pig
@@ -0,0 +1,4 @@
+A = load '$INPDIR/table3.txt' using PigStorage('\t') AS (row:int, content:chararray);
+B = load '$INPDIR/table3ToJoin.txt' using PigStorage('\t') AS (row:int, content:chararray);
+C = JOIN A BY content, B BY content;
+store C into '$OUTDIR/PigJoin' USING PigStorage();
\ No newline at end of file
diff --git a/hcatalog/src/test/e2e/templeton/inpdir/artofwar.txt b/hcatalog/src/test/e2e/templeton/inpdir/artofwar.txt
index e69de29..25f59ec 100755
--- a/hcatalog/src/test/e2e/templeton/inpdir/artofwar.txt
+++ b/hcatalog/src/test/e2e/templeton/inpdir/artofwar.txt
@@ -0,0 +1,21 @@
+孫 子 曰 : 兵 者 , 國 之 大 事 , 死 生 之 地 , 存 亡 之 道 , 不 可 不 察 也 。
+
+ 故 經 之 以 五 , 校 之 以 計 , 而 索 其 情 : 一 曰 道 , 二 曰 天 , 三 曰 地 , 四 曰
+ 將 , 五 曰 法 。 道 者 , 令 民 于 上 同 意 者 也 , 可 與 之 死 , 可 與 之 生 , 民 不
+ 詭 也 。 天 者 , 陰 陽 、 寒 暑 、 時 制 也 。 地 者 , 高 下 、 遠 近 、 險 易 、 廣 狹
+ 、 死 生 也 。 將 者 , 智 、 信 、 仁 、 勇 、 嚴 也 。 法 者 , 曲 制 、 官 道 、 主 用
+ 也 。 凡 此 五 者 , 將 莫 不 聞 , 知 之 者 勝 , 不 知 之 者 不 勝 。 故 校 之 以 計 ,
+ 而 索 其 情 。 曰 : 主 孰 有 道 ? 將 孰 有 能 ? 天 地 孰 得 ? 法 令 孰 行 ? 兵 眾 孰
+ 強 ? 士 卒 孰 練 ? 賞 罰 孰 明 ? 吾 以 此 知 勝 負 矣 。
+
+ 將 聽 吾 計 , 用 之 必 勝 , 留 之 ﹔ 將 不 聽 吾 計 , 用 之 必 敗 , 去 之 。
+
+ 計 利 以 聽 , 乃 為 之 勢 , 以 佐 其 外 。 勢 者 , 因 利 而 制 權 也 。
+
+ 兵 者 , 詭 道 也 。 故 能 而 示 之 不 能 , 用 而 示 之 不 用 , 近 而 示 之 遠 , 遠 而
+ 示 之 近 。 利 而 誘 之 , 亂 而 取 之 , 實 而 備 之 , 強 而 避 之 , 怒 而 撓 之 , 卑
+ 而 驕 之 , 佚 而 勞 之 , 親 而 離 之 , 攻 其 不 備 , 出 其 不 意 。 此 兵 家 之 勝 ,
+ 不 可 先 傳 也 。
+
+ 夫 未 戰 而 廟 算 勝 者 , 得 算 多 也 ﹔ 未 戰 而 廟 算 不 勝 者 , 得 算 少 也 。 多 算
+ 勝 , 少 算 不 勝 , 而 況 無 算 乎 ! 吾 以 此 觀 之 , 勝 負 見 矣 。
\ No newline at end of file
diff --git a/hcatalog/src/test/e2e/templeton/inpdir/table1.txt b/hcatalog/src/test/e2e/templeton/inpdir/table1.txt
index e69de29..f40bd60 100755
--- a/hcatalog/src/test/e2e/templeton/inpdir/table1.txt
+++ b/hcatalog/src/test/e2e/templeton/inpdir/table1.txt
@@ -0,0 +1,27 @@
+868 狚鼾䶵鼾c鼾㐀ee䶴﨨b
+423 be狝h狛狝狝㐁﨨狛a
+440 hc鼾㐀啊狝g狜c狜䶵㐁狛㐀丄g
+701 䶴䶵ecea狝䶵e狝䶴
+550 﨨狝狝䶴狛g﨩䶵狝a䶴﨩﨩g狛a狝
+123 阿狛鼾㐀鼾﨨dg䶴㐁d
+316 狜﨨﨩a䶵d㐀g狝c狝䶵de
+807 e狛h齄狝e䶴啊h䶴㐁狚阿狜g䶴aa
+870 狝a狛g丄fe䶵﨨狜狛䶵b䶵狛a狚齄
+669 dc丄a丄狜﨨f齄﨨b齄hd鼾
+622 a狚齄㐁f阿阿狜c﨩狛b狝e
+216 fh﨨阿dcf㐀b㐀h㐀狛c﨩g狚
+700 狜﨩㐀阿﨨齄鼾狝阿b﨨﨩狚f䶴阿
+528 啊狚﨩狛鼾丄啊丄丄㐁a䶴狝d狜fd丄
+173 h狚ed㐀d狛bc阿㐀啊﨩
+3 䶴狝﨩狚㐁啊䶵齄齄﨩狜ee㐀㐁h狛﨨㐀
+567 a啊g狜丄狚c䶵h狝﨨h
+435 b齄阿阿丄g㐁cf丄阿ed
+943 㐁g䶵狛cc㐀䶵h狝﨩䶵啊
+365 狛啊d狝䶵cb﨩䶵bd﨨狝阿b
+298 䶵䶵﨩h鼾㐁a狛鼾狝狚䶵f狝
+377 bd䶵﨩齄䶵㐀h㐁㐁狛鼾狝
+951 g狚ee﨨ach狝f﨩㐀㐀䶵ba
+399 啊﨨鼾d鼾ff齄h丄﨨狚a阿ga啊齄c
+199 阿狛㐀eg齄狝狝g齄ga啊d狜㐁齄
+816 齄狛狛啊狜af狚狛㐁狜
+758 狛gc㐀狜fa﨨㐀㐁齄ab㐀h
\ No newline at end of file
diff --git a/hcatalog/src/test/e2e/templeton/inpdir/table3.txt b/hcatalog/src/test/e2e/templeton/inpdir/table3.txt
index e69de29..bc23ac5 100755
--- a/hcatalog/src/test/e2e/templeton/inpdir/table3.txt
+++ b/hcatalog/src/test/e2e/templeton/inpdir/table3.txt
@@ -0,0 +1,14 @@
+1 䶵
+2 䶵
+3 狚
+4 䶵
+5 鼾
+6 䶵
+112 﨩﨨狝狝
+12 﨩﨨狝狝
+7 鼾
+8 䶵
+9 狚䶵
+10 狚䶵齄
+11 﨩齄䶵
+12 﨩﨨狝狝
\ No newline at end of file
diff --git a/hcatalog/src/test/e2e/templeton/inpdir/table3ToJoin.txt b/hcatalog/src/test/e2e/templeton/inpdir/table3ToJoin.txt
index e69de29..bc5bf95 100755
--- a/hcatalog/src/test/e2e/templeton/inpdir/table3ToJoin.txt
+++ b/hcatalog/src/test/e2e/templeton/inpdir/table3ToJoin.txt
@@ -0,0 +1,5 @@
+12 﨩﨨狝狝
+2 䶵shouldnotshowup
+3 狚
+4 䶵
+11 﨩齄䶵
diff --git a/hcatalog/src/test/e2e/templeton/tests/utf8.conf b/hcatalog/src/test/e2e/templeton/tests/utf8.conf
index e69de29..eab049c 100644
--- a/hcatalog/src/test/e2e/templeton/tests/utf8.conf
+++ b/hcatalog/src/test/e2e/templeton/tests/utf8.conf
@@ -0,0 +1,134 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+###############################################################################
+# curl command tests for templeton
+#
+#
+use utf8;
+
+$cfg =
+{
+ 'driver' => 'Curl',
+
+ 'groups' =>
+ [
+##=============================================================================================================
+ {
+ 'name' => 'Hive_UTF8',
+ 'tests' =>
+ [
+ {
+ # Submitting Hive job with Unicode content on Unicode data
+ 'num' => 1,
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/hive',
+ 'post_options' => ['user.name=:UNAME:','execute=drop table if exists utf8_table;create table utf8_table(row int, content string) ROW FORMAT DELIMITED FIELDS TERMINATED BY \'\t\';LOAD DATA INPATH \':INPDIR_HDFS:/table1.txt\' OVERWRITE INTO TABLE utf8_table;','statusdir=:OUTDIR:/status/Hive_UTF8_:TNUM:'],
+ 'json_field_substr_match' => { 'id' => '\d+'},
+ #results
+ 'status_code' => 200,
+ 'check_job_created' => 1,
+ 'check_job_complete' => 'SUCCESS',
+ 'check_job_exit_value' => 0,
+ 'check_call_back' => 1,
+ },
+ ]
+ },
+##=============================================================================================================
+ {
+ 'name' => 'Pig_UTF8',
+ 'tests' =>
+ [
+ {
+ # Submitting Pig job with Unicode content on Unicode data
+ 'num' => 1,
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/pig',
+ 'post_options' => ['user.name=:UNAME:','file=:INPDIR_HDFS:/PigJoin䶴ㄩ鼾丄狜〇work.pig','arg=-p', 'arg=INPDIR=:INPDIR_HDFS:','arg=-p','arg=OUTDIR=:OUTDIR:','statusdir=:OUTDIR:/status/Pig_UTF8_:TNUM:'],
+ 'json_field_substr_match' => { 'id' => '\d+'},
+ #results
+ 'status_code' => 200,
+ 'check_job_created' => 1,
+ 'check_job_complete' => 'SUCCESS',
+ 'check_call_back' => 1,
+ },
+ ]
+ },
+##=============================================================================================================
+ {
+ 'name' => 'MapReduce_UTF8',
+ 'tests' =>
+ [
+ {
+
+ 'num' => 1,
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/jar',
+ 'post_options' => ['user.name=:UNAME:','arg=:INPDIR_HDFS:/artof䶴ㄩ鼾丄狜〇war.txt', 'arg= :OUTDIR:/utf8_wc.txt',
+ 'jar=:INPDIR_HDFS:/hadoop_examples_䶴ㄩ鼾丄狜〇_2_2_0.jar', 'class=wordcount','statusdir=:OUTDIR:/status/MapRed_UTF8_:TNUM:'],
+ 'json_field_substr_match' => { 'id' => '\d+'},
+ #results
+ 'status_code' => 200,
+ 'check_job_created' => 1,
+ 'check_job_complete' => 'SUCCESS',
+ 'check_job_percent_complete' => 'map 100% reduce 100%',
+ 'check_job_exit_value' => 0,
+ 'check_call_back' => 1,
+ },
+ ]
+ },
+##=============================================================================================================
+ {
+ 'name' => 'MapRedStreaming_UTF8',
+ 'tests' =>
+ [
+ {
+ 'num' => 1,
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/streaming',
+ 'post_options' => ['user.name=:UNAME:','input=:INPDIR_HDFS:/artof䶴ㄩ鼾丄狜〇war.txt', 'input=:INPDIR_HDFS:/artof䶴ㄩ鼾丄狜〇war.txt', 'output=:OUTDIR:/MapRedStreaming_UTF8_count', 'mapper=cat', 'reducer=wc','statusdir=:OUTDIR:/status/MapRedStreaming_UTF8_:TNUM:'],
+ 'json_field_substr_match' => { 'id' => '\d+'},
+ #results
+ 'status_code' => 200,
+ 'check_job_created' => 1,
+ 'check_job_complete' => 'SUCCESS',
+ 'check_job_exit_value' => 0,
+ 'check_call_back' => 1,
+ },
+ ]
+ },
+##=============================================================================================================
+
+ {
+ 'name' => 'JobFiltering_UTF8',
+ 'tests' =>
+ [
+ {
+ 'num' => 1,
+ 'depends_on' => 'Pig_UTF8',
+ 'method' => 'GET',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/jobs?user.name=:UNAME:&showall=true&fields=*',
+ 'user_name' => ':UNAME:',
+ 'format_header' => 'Content-Type: application/json',
+ 'filter_job_names' => ['TempletonControllerJob', 'PigLatin:PigJoin䶴ㄩ鼾丄狜〇work.pig'],
+ 'status_code' => 200,
+ },
+ ]
+ },
+ ]
+},
+ ;
\ No newline at end of file