Index: contrib/src/test/org/apache/hadoop/hive/contrib/serde2/TestRegexSerDe.java =================================================================== --- contrib/src/test/org/apache/hadoop/hive/contrib/serde2/TestRegexSerDe.java (revision 901519) +++ contrib/src/test/org/apache/hadoop/hive/contrib/serde2/TestRegexSerDe.java (working copy) @@ -17,9 +17,10 @@ */ package org.apache.hadoop.hive.contrib.serde2; -import java.util.Arrays; import java.util.Properties; +import junit.framework.TestCase; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.hive.serde2.SerDe; @@ -28,12 +29,9 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; import org.apache.hadoop.io.Text; -import junit.framework.TestCase; - public class TestRegexSerDe extends TestCase { - - private SerDe createSerDe(String fieldNames, String fieldTypes, + private SerDe createSerDe(String fieldNames, String fieldTypes, String inputRegex, String outputFormatString) throws Throwable { Properties schema = new Properties(); schema.setProperty(Constants.LIST_COLUMNS, fieldNames); @@ -45,8 +43,7 @@ serde.initialize(new Configuration(), schema); return serde; } - - + /** * Test the LazySimpleSerDe class. */ @@ -57,39 +54,39 @@ "host,identity,user,time,request,status,size,referer,agent", "string,string,string,string,string,string,string,string,string", "([^ ]*) ([^ ]*) ([^ ]*) (-|\\[[^\\]]*\\]) ([^ \"]*|\"[^\"]*\") ([0-9]*) ([0-9]*) ([^ \"]*|\"[^\"]*\") ([^ \"]*|\"[^\"]*\")", - "%1$s %2$s %3$s %4$s %5$s %6$s %7$s %8$s %9$s" - ); - + "%1$s %2$s %3$s %4$s %5$s %6$s %7$s %8$s %9$s"); + // Data - Text t = new Text("127.0.0.1 - - [26/May/2009:00:00:00 +0000] " - + "\"GET /someurl/?track=Blabla(Main) HTTP/1.1\" 200 5864 - " - + "\"Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.65 Safari/525.19\""); - + Text t = new Text( + "127.0.0.1 - - [26/May/2009:00:00:00 +0000] " + + "\"GET /someurl/?track=Blabla(Main) HTTP/1.1\" 200 5864 - " + + "\"Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.65 Safari/525.19\""); + // Deserialize Object row = serDe.deserialize(t); ObjectInspector rowOI = serDe.getObjectInspector(); System.out.println("Deserialized row: " + row); - + // Serialize - Text serialized = (Text)serDe.serialize(row, rowOI); - assertEquals(t, serialized); - + Text serialized = (Text) serDe.serialize(row, rowOI); + assertEquals(t, serialized); + // Do some changes (optional) - ObjectInspector standardWritableRowOI = ObjectInspectorUtils.getStandardObjectInspector( - rowOI, ObjectInspectorCopyOption.WRITABLE); - Object standardWritableRow = ObjectInspectorUtils.copyToStandardObject(row, rowOI, - ObjectInspectorCopyOption.WRITABLE); - + ObjectInspector standardWritableRowOI = ObjectInspectorUtils + .getStandardObjectInspector(rowOI, ObjectInspectorCopyOption.WRITABLE); + Object standardWritableRow = ObjectInspectorUtils.copyToStandardObject( + row, rowOI, ObjectInspectorCopyOption.WRITABLE); + // Serialize - serialized = (Text)serDe.serialize(standardWritableRow, standardWritableRowOI); - assertEquals(t, serialized); - + serialized = (Text) serDe.serialize(standardWritableRow, + standardWritableRowOI); + assertEquals(t, serialized); + } catch (Throwable e) { e.printStackTrace(); throw e; } } - } Index: contrib/src/test/org/apache/hadoop/hive/contrib/mr/TestGenericMR.java =================================================================== --- contrib/src/test/org/apache/hadoop/hive/contrib/mr/TestGenericMR.java (revision 901519) +++ contrib/src/test/org/apache/hadoop/hive/contrib/mr/TestGenericMR.java (working copy) @@ -24,47 +24,48 @@ import junit.framework.TestCase; - public final class TestGenericMR extends TestCase { public void testReduceTooFar() throws Exception { - try { - new GenericMR().reduce(new StringReader("a\tb\tc"), new StringWriter(), new Reducer() { - public void reduce(String key, Iterator records, Output output) throws Exception { - while (true) { - records.next(); - } - } - }); + try { + new GenericMR().reduce(new StringReader("a\tb\tc"), new StringWriter(), + new Reducer() { + public void reduce(String key, Iterator records, + Output output) throws Exception { + while (true) { + records.next(); + } + } + }); } catch (final NoSuchElementException nsee) { - // expected - return; + // expected + return; } - + fail("Expected NoSuchElementException"); } public void testEmptyMap() throws Exception { final StringWriter out = new StringWriter(); - + new GenericMR().map(new StringReader(""), out, identityMapper()); - + assertEquals(0, out.toString().length()); } - + public void testIdentityMap() throws Exception { final String in = "a\tb\nc\td"; final StringWriter out = new StringWriter(); - + new GenericMR().map(new StringReader(in), out, identityMapper()); - + assertEquals(in + "\n", out.toString()); } - + public void testKVSplitMap() throws Exception { final String in = "k1=v1,k2=v2\nk1=v2,k2=v3"; final String expected = "k1\tv1\nk2\tv2\nk1\tv2\nk2\tv3\n"; final StringWriter out = new StringWriter(); - + new GenericMR().map(new StringReader(in), out, new Mapper() { public void map(String[] record, Output output) throws Exception { for (final String kvs : record[0].split(",")) { @@ -73,41 +74,42 @@ } } }); - - assertEquals(expected, out.toString()); + + assertEquals(expected, out.toString()); } - + public void testIdentityReduce() throws Exception { final String in = "a\tb\nc\td"; final StringWriter out = new StringWriter(); - + new GenericMR().reduce(new StringReader(in), out, identityReducer()); - + assertEquals(in + "\n", out.toString()); } - + public void testWordCountReduce() throws Exception { final String in = "hello\t1\nhello\t2\nokay\t4\nokay\t6\nokay\t2"; final StringWriter out = new StringWriter(); - + new GenericMR().reduce(new StringReader(in), out, new Reducer() { @Override - public void reduce(String key, Iterator records, Output output) throws Exception { + public void reduce(String key, Iterator records, Output output) + throws Exception { int count = 0; - + while (records.hasNext()) { count += Integer.parseInt(records.next()[1]); } - + output.collect(new String[] { key, String.valueOf(count) }); } }); - + final String expected = "hello\t3\nokay\t12\n"; - + assertEquals(expected, out.toString()); } - + private Mapper identityMapper() { return new Mapper() { @Override @@ -116,15 +118,16 @@ } }; } - + private Reducer identityReducer() { - return new Reducer() { - @Override - public void reduce(String key, Iterator records, Output output) throws Exception { - while (records.hasNext()) { - output.collect(records.next()); + return new Reducer() { + @Override + public void reduce(String key, Iterator records, Output output) + throws Exception { + while (records.hasNext()) { + output.collect(records.next()); + } } - } - }; + }; } }