diff --git src/test/e2e/hcatalog/tests/hadoop.conf src/test/e2e/hcatalog/tests/hadoop.conf index 89921c5..bc73ab4 100644 --- src/test/e2e/hcatalog/tests/hadoop.conf +++ src/test/e2e/hcatalog/tests/hadoop.conf @@ -114,6 +114,42 @@ jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.ReadRC -libjars :HCAT_JAR: ,'floatpostprocess' => 1 ,'delimiter' => ' ' }, + { + 'num' => 6 + ,'hadoop' => q\ + jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.SimpleRead -libjars :HCAT_JAR: :THRIFTSERVER: studentparttab30k :OUTPATH: 'ds="20110924" or ds="20110925"' +\, + ,'sql' => q\select name, age from studentparttab30k where ds='20110924' or ds='20110925';\ + ,'floatpostprocess' => 1 + ,'delimiter' => ' ' + }, + { + 'num' => 7 + ,'hadoop' => q\ + jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.SimpleRead -libjars :HCAT_JAR: :THRIFTSERVER: studentparttab30k :OUTPATH: 'ds > "20110924"' +\, + ,'sql' => q\select name, age from studentparttab30k where ds > '20110924';\ + ,'floatpostprocess' => 1 + ,'delimiter' => ' ' + }, + { + 'num' => 8 + ,'hadoop' => q\ + jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.SimpleRead -libjars :HCAT_JAR: :THRIFTSERVER: studentparttab30k :OUTPATH: 'ds < "20110925"' +\, + ,'sql' => q\select name, age from studentparttab30k where ds < '20110925';\ + ,'floatpostprocess' => 1 + ,'delimiter' => ' ' + }, + { + 'num' => 9 + ,'hadoop' => q\ + jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.SimpleRead -libjars :HCAT_JAR: :THRIFTSERVER: studentparttab30k :OUTPATH: 'ds <= "20110925" and ds >= "20110924"' +\, + ,'sql' => q\select name, age from studentparttab30k where ds <= '20110925' and ds >= '20110924';\ + ,'floatpostprocess' => 1 + ,'delimiter' => ' ' + } ], }, # end g { diff --git src/test/e2e/hcatalog/tests/pig.conf src/test/e2e/hcatalog/tests/pig.conf index 05c8863..36e9a7e 100644 --- src/test/e2e/hcatalog/tests/pig.conf +++ src/test/e2e/hcatalog/tests/pig.conf @@ -165,7 +165,43 @@ alter table pig_read_5 add partition (b='1') location '/user/hcat/tests/data/stu b = foreach a generate name, age, b; store b into ':OUTPATH:';\, ,'sql' => q\select name, age, 1 from studenttab10k;\ - } + }, + { + # Read from two partitions + 'num' => 6 + ,'pig' => q\a = load 'studentparttab30k' using org.apache.hcatalog.pig.HCatLoader(); +b = filter a by ds == '20110924' or ds == '20110925'; +c = foreach b generate name, age, ds; +store c into ':OUTPATH:';\, + ,'sql' => "select name, age, ds from studentparttab30k where ds = '20110924' or ds = '20110925';", + }, + { + # Read from open range of partitions + 'num' => 7 + ,'pig' => q\a = load 'studentparttab30k' using org.apache.hcatalog.pig.HCatLoader(); +b = filter a by ds > '20110924'; +c = foreach b generate name, age, ds; +store c into ':OUTPATH:';\, + ,'sql' => "select name, age, ds from studentparttab30k where ds > '20110924';", + }, + { + # Read from open range of partitions + 'num' => 8 + ,'pig' => q\a = load 'studentparttab30k' using org.apache.hcatalog.pig.HCatLoader(); +b = filter a by ds < '20110925'; +c = foreach b generate name, age, ds; +store c into ':OUTPATH:';\, + ,'sql' => "select name, age, ds from studentparttab30k where ds < '20110925';", + }, + { + # Read from closed range of partitions + 'num' => 9 + ,'pig' => q\a = load 'studentparttab30k' using org.apache.hcatalog.pig.HCatLoader(); +b = filter a by ds <= '20110925' and ds >= '20110924'; +c = foreach b generate name, age, ds; +store c into ':OUTPATH:';\, + ,'sql' => "select name, age, ds from studentparttab30k where ds <= '20110925' and ds >= '20110924';", + } ], }, # end g { diff --git src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/SimpleRead.java src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/SimpleRead.java index 2f1f7cd..e6373e4 100644 --- src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/SimpleRead.java +++ src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/SimpleRead.java @@ -66,7 +66,6 @@ public class SimpleRead extends Configured implements Tool { Text,IntWritable>.Context context) throws IOException ,InterruptedException { name = (String) value.get(0); -System.out.println(name); age = (Integer) value.get(1); gpa = (Double) value.get(2); context.write(new Text(name), new IntWritable(age)); @@ -75,12 +74,14 @@ System.out.println(name); } public int run(String[] args) throws Exception { + String filter = null; Configuration conf = getConf(); args = new GenericOptionsParser(conf, args).getRemainingArgs(); String serverUri = args[0]; String tableName = args[1]; String outputDir = args[2]; + if (args.length > 3) filter = args[3]; String dbName = null; String principalID = System.getProperty(HCatConstants.HCAT_METASTORE_PRINCIPAL); @@ -88,7 +89,7 @@ System.out.println(name); conf.set(HCatConstants.HCAT_METASTORE_PRINCIPAL, principalID); Job job = new Job(conf, "SimpleRead"); HCatInputFormat.setInput(job, InputJobInfo.create( - dbName, tableName, null, serverUri, principalID)); + dbName, tableName, filter, serverUri, principalID)); // initialize HCatOutputFormat job.setInputFormatClass(HCatInputFormat.class);