From 4375c36e6a65af65359e8b38a2f31a3eb773ab28 Mon Sep 17 00:00:00 2001 From: asreekumar Date: Tue, 2 Sep 2014 18:33:20 -0700 Subject: [PATCH] Add E2E test for templeton.mapper.memory.mb --- hcatalog/src/test/e2e/templeton/README.txt | 14 +++++ hcatalog/src/test/e2e/templeton/build.xml | 29 ++++++++++ .../e2e/templeton/tests/modifyConfiguration.conf | 64 ++++++++++++++++++++++ 3 files changed, 107 insertions(+) create mode 100644 hcatalog/src/test/e2e/templeton/tests/modifyConfiguration.conf diff --git a/hcatalog/src/test/e2e/templeton/README.txt b/hcatalog/src/test/e2e/templeton/README.txt index bf2843c..ca16a92 100644 --- a/hcatalog/src/test/e2e/templeton/README.txt +++ b/hcatalog/src/test/e2e/templeton/README.txt @@ -202,6 +202,20 @@ Here is the schema of the table writen in MySQL: To prevent primary key violation and sqoop import directory conflict, make sure the "PERSON" table is empty and the folder hdfs://hostname:8020/sqoopoutputdir doesn't exist before running the test. +Running updateConfig tests +--------------------------- +Testsuite includes the test to verify fix for https://issues.apache.org/jira/browse/HIVE-7155 +The test in this test suite is trying to verify the use of property templeton.mapper.memory.mb in webhcat-site.xml. +For this, an attempt is made to load the data, of size greater than 100MB, from one hive table to another hive table with +the templeton.mapper.memory.mb set to 0.01. This test is expecting the failure of map job due to insufficient memory. + +Following steps need to be performed prior to test: + +1. templeton.mapper.memory.mb property in webhcat-site.xml should be set to 0.01 +2. Any data file that is over 100MB can be used as input. One sample input will be the ratings.dat from + http://files.grouplens.org/datasets/movielens/ml-1m.zip. Rename the file to ratings.txt. +3. Move input text file to the ratings directory inside hdfs inpdir + Notes ----- It's best to set HADOOP_HOME_WARN_SUPPRESS=true everywhere you can. diff --git a/hcatalog/src/test/e2e/templeton/build.xml b/hcatalog/src/test/e2e/templeton/build.xml index 8bc363d..b817d21 100644 --- a/hcatalog/src/test/e2e/templeton/build.xml +++ b/hcatalog/src/test/e2e/templeton/build.xml @@ -192,6 +192,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/hcatalog/src/test/e2e/templeton/tests/modifyConfiguration.conf b/hcatalog/src/test/e2e/templeton/tests/modifyConfiguration.conf new file mode 100644 index 0000000..bab047f --- /dev/null +++ b/hcatalog/src/test/e2e/templeton/tests/modifyConfiguration.conf @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +############################################################################### +# curl command tests for templeton +# +# + +#use Yahoo::Miners::Test::PigSetup; + +#PigSetup::setup(); + +#my $me = `whoami`; +#chomp $me; +$cfg = +{ + 'driver' => 'Curl', + + 'groups' => + [ +##============================================================================================================= + { + 'name' => 'ModifyWebhcatConfig', + 'tests' => + [ + { + #test to verify templeton.mapper.memory.mb update + 'num' => 1, + 'setup' => [ + { + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl', + 'status_code' => 200, + 'post_options' => ['user.name=:UNAME:','exec=drop table if exists ratings;CREATE EXTERNAL TABLE ratings(userid INT, movieid INT, ratings FLOAT, unixtime STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY \':\' location \':INPDIR_HDFS:/ratings/\';drop table if exists testtable;create table testtable(userid INT, movieid INT, ratings FLOAT, unixtime STRING) location \':INPDIR_HDFS:/testtable/\';'],'json_field_substr_match' => {'stderr' => 'OK'} + } + ], + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/hive', + 'post_options' => ['user.name=:UNAME:','execute=INSERT OVERWRITE TABLE testtable SELECT * from ratings','statusdir=:OUTDIR:/status'], + 'json_field_substr_match' => { 'id' => '\d+'}, + #results + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'FAILURE', + } + ] + } + ] +}, + ; \ No newline at end of file -- 1.8.5.2 (Apple Git-48)