Index: ql/src/test/org/apache/hadoop/hive/serde2/TestSerDe.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/serde2/TestSerDe.java (revision 901519) +++ ql/src/test/org/apache/hadoop/hive/serde2/TestSerDe.java (working copy) @@ -18,7 +18,6 @@ package org.apache.hadoop.hive.serde2; -import java.io.UnsupportedEncodingException; import java.nio.charset.CharacterCodingException; import java.util.ArrayList; import java.util.Arrays; @@ -37,9 +36,6 @@ import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; -import org.apache.hadoop.hive.serde2.SerDeUtils; -import org.apache.hadoop.hive.serde2.SerDe; -import org.apache.hadoop.hive.serde2.SerDeException; public class TestSerDe implements SerDe { @@ -49,7 +45,6 @@ return shortName(); } - public static String shortName() { return "test_meta"; } @@ -60,20 +55,22 @@ try { SerDeUtils.registerSerDe(shortName(), Class.forName(className)); // For backward compatibility: this class replaces the following class. - SerDeUtils.registerSerDe("org.apache.hadoop.hive.serde.TestSerDe", Class.forName(className)); - } catch(Exception e) { + SerDeUtils.registerSerDe("org.apache.hadoop.hive.serde.TestSerDe", Class + .forName(className)); + } catch (Exception e) { throw new RuntimeException(e); } } - + final public static String DefaultSeparator = "\002"; private String separator; // constant for now, will make it configurable later. - private String nullString = "\\N"; + private final String nullString = "\\N"; private List columnNames; private ObjectInspector cachedObjectInspector; + @Override public String toString() { return "TestSerDe[" + separator + "," + columnNames + "]"; } @@ -82,15 +79,16 @@ separator = DefaultSeparator; } - public void initialize(Configuration job, Properties tbl) throws SerDeException { + public void initialize(Configuration job, Properties tbl) + throws SerDeException { separator = DefaultSeparator; String alt_sep = tbl.getProperty("testserde.default.serialization.format"); - if(alt_sep != null && alt_sep.length() > 0) { + if (alt_sep != null && alt_sep.length() > 0) { try { - byte b [] = new byte[1]; + byte b[] = new byte[1]; b[0] = Byte.valueOf(alt_sep).byteValue(); separator = new String(b); - } catch(NumberFormatException e) { + } catch (NumberFormatException e) { separator = alt_sep; } } @@ -98,25 +96,29 @@ String columnProperty = tbl.getProperty("columns"); if (columnProperty == null || columnProperty.length() == 0) { // Hack for tables with no columns - // Treat it as a table with a single column called "col" - cachedObjectInspector = ObjectInspectorFactory.getReflectionObjectInspector( - ColumnSet.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); + // Treat it as a table with a single column called "col" + cachedObjectInspector = ObjectInspectorFactory + .getReflectionObjectInspector(ColumnSet.class, + ObjectInspectorFactory.ObjectInspectorOptions.JAVA); } else { columnNames = Arrays.asList(columnProperty.split(",")); - cachedObjectInspector = MetadataListStructObjectInspector.getInstance(columnNames); + cachedObjectInspector = MetadataListStructObjectInspector + .getInstance(columnNames); } - LOG.info(getClass().getName() + ": initialized with columnNames: " + columnNames ); + LOG.info(getClass().getName() + ": initialized with columnNames: " + + columnNames); } - public static Object deserialize(ColumnSet c, String row, String sep, String nullString) throws Exception { + public static Object deserialize(ColumnSet c, String row, String sep, + String nullString) throws Exception { if (c.col == null) { c.col = new ArrayList(); } else { c.col.clear(); } - String [] l1 = row.split(sep, -1); + String[] l1 = row.split(sep, -1); - for(String s: l1) { + for (String s : l1) { if (s.equals(nullString)) { c.col.add(null); } else { @@ -125,12 +127,13 @@ } return (c); } - + ColumnSet deserializeCache = new ColumnSet(); + public Object deserialize(Writable field) throws SerDeException { String row = null; if (field instanceof BytesWritable) { - BytesWritable b = (BytesWritable)field; + BytesWritable b = (BytesWritable) field; try { row = Text.decode(b.get(), 0, b.getSize()); } catch (CharacterCodingException e) { @@ -142,17 +145,17 @@ try { deserialize(deserializeCache, row, separator, nullString); if (columnNames != null) { - assert(columnNames.size() == deserializeCache.col.size()); + assert (columnNames.size() == deserializeCache.col.size()); } return deserializeCache; } catch (ClassCastException e) { - throw new SerDeException( this.getClass().getName() + " expects Text or BytesWritable", e); + throw new SerDeException(this.getClass().getName() + + " expects Text or BytesWritable", e); } catch (Exception e) { throw new SerDeException(e); } } - - + public ObjectInspector getObjectInspector() throws SerDeException { return cachedObjectInspector; } @@ -160,27 +163,33 @@ public Class getSerializedClass() { return Text.class; } - + Text serializeCache = new Text(); - public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException { + public Writable serialize(Object obj, ObjectInspector objInspector) + throws SerDeException { + if (objInspector.getCategory() != Category.STRUCT) { - throw new SerDeException(getClass().toString() - + " can only serialize struct types, but we got: " + objInspector.getTypeName()); + throw new SerDeException(getClass().toString() + + " can only serialize struct types, but we got: " + + objInspector.getTypeName()); } StructObjectInspector soi = (StructObjectInspector) objInspector; List fields = soi.getAllStructFieldRefs(); - + StringBuilder sb = new StringBuilder(); - for(int i=0; i0) sb.append(separator); + for (int i = 0; i < fields.size(); i++) { + if (i > 0) { + sb.append(separator); + } Object column = soi.getStructFieldData(obj, fields.get(i)); if (fields.get(i).getFieldObjectInspector().getCategory() == Category.PRIMITIVE) { // For primitive object, serialize to plain string sb.append(column == null ? nullString : column.toString()); } else { // For complex object, serialize to JSON format - sb.append(SerDeUtils.getJSONString(column, fields.get(i).getFieldObjectInspector())); + sb.append(SerDeUtils.getJSONString(column, fields.get(i) + .getFieldObjectInspector())); } } serializeCache.set(sb.toString()); Index: ql/src/test/org/apache/hadoop/hive/scripts/extracturl.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/scripts/extracturl.java (revision 901519) +++ ql/src/test/org/apache/hadoop/hive/scripts/extracturl.java (working copy) @@ -18,17 +18,18 @@ package org.apache.hadoop.hive.scripts; -import java.io.*; - -import java.util.HashMap; +import java.io.BufferedReader; +import java.io.InputStreamReader; import java.util.regex.Matcher; import java.util.regex.Pattern; public class extracturl { - protected static final Pattern pattern = Pattern.compile("link", Pattern.CASE_INSENSITIVE); - static InputStreamReader converter = new InputStreamReader (System.in); - static BufferedReader in = new BufferedReader (converter); + protected static final Pattern pattern = Pattern.compile( + "link", + Pattern.CASE_INSENSITIVE); + static InputStreamReader converter = new InputStreamReader(System.in); + static BufferedReader in = new BufferedReader(converter); public static void main(String[] args) { String input; @@ -36,15 +37,14 @@ while ((input = in.readLine()) != null) { Matcher m = pattern.matcher(input); - while(m.find()) { + while (m.find()) { String url = input.substring(m.start(1), m.end(1)); System.out.println(url + "\t" + "1"); - } + } } - } - catch (Exception e) { + } catch (Exception e) { e.printStackTrace(); System.exit(1); } - } + } } Index: ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java (revision 901519) +++ ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java (working copy) @@ -16,8 +16,6 @@ import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat; import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.mapred.TextInputFormat; -import org.apache.hadoop.mapred.TextOutputFormat; - import org.apache.thrift.TException; public class TestHiveMetaStoreChecker extends TestCase { @@ -26,11 +24,11 @@ private FileSystem fs; private HiveMetaStoreChecker checker = null; - private String dbName = "dbname"; - private String tableName = "tablename"; + private final String dbName = "dbname"; + private final String tableName = "tablename"; - private String partDateName = "partdate"; - private String partCityName = "partcity"; + private final String partDateName = "partdate"; + private final String partCityName = "partcity"; private List partCols; private List> parts; @@ -42,10 +40,8 @@ checker = new HiveMetaStoreChecker(hive); partCols = new ArrayList(); - partCols.add(new FieldSchema(partDateName, Constants.STRING_TYPE_NAME, - "")); - partCols.add(new FieldSchema(partCityName, Constants.STRING_TYPE_NAME, - "")); + partCols.add(new FieldSchema(partDateName, Constants.STRING_TYPE_NAME, "")); + partCols.add(new FieldSchema(partCityName, Constants.STRING_TYPE_NAME, "")); parts = new ArrayList>(); Map part1 = new HashMap(); @@ -140,13 +136,13 @@ assertTrue(result.getTablesNotOnFs().isEmpty()); assertTrue(result.getPartitionsNotOnFs().isEmpty()); assertTrue(result.getPartitionsNotInMs().isEmpty()); - - //create a new external table + + // create a new external table hive.dropTable(dbName, tableName); table.setProperty("EXTERNAL", "TRUE"); hive.createTable(table); - - //should return all ok + + // should return all ok result = new CheckResult(); checker.checkMetastore(dbName, null, null, result); assertTrue(result.getTablesNotInMs().isEmpty()); @@ -188,7 +184,7 @@ fs = partToRemovePath.getFileSystem(hive.getConf()); fs.delete(partToRemovePath, true); - result = new CheckResult(); + result = new CheckResult(); checker.checkMetastore(dbName, tableName, null, result); // missing one partition on fs assertTrue(result.getTablesNotInMs().isEmpty()); Index: ql/src/test/org/apache/hadoop/hive/ql/metadata/TestPartition.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/metadata/TestPartition.java (revision 901519) +++ ql/src/test/org/apache/hadoop/hive/ql/metadata/TestPartition.java (working copy) @@ -27,25 +27,26 @@ public void testPartition() throws HiveException, URISyntaxException { StorageDescriptor sd = new StorageDescriptor(); sd.setLocation("partlocation"); - + Partition tp = new Partition(); tp.setTableName(TABLENAME); tp.setSd(sd); - + List values = new ArrayList(); values.add(PARTITION_VALUE); tp.setValues(values); - + List partCols = new ArrayList(); partCols.add(new FieldSchema(PARTITION_COL, "string", "")); - + Table tbl = new Table(TABLENAME); tbl.setDataLocation(new URI("tmplocation")); tbl.setPartCols(partCols); - - Map spec = new org.apache.hadoop.hive.ql.metadata.Partition(tbl, tp).getSpec(); + + Map spec = new org.apache.hadoop.hive.ql.metadata.Partition( + tbl, tp).getSpec(); assertFalse(spec.isEmpty()); assertEquals(spec.get(PARTITION_COL), PARTITION_VALUE); } - + } Index: ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java (revision 901519) +++ ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java (working copy) @@ -31,15 +31,14 @@ import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; +import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; import org.apache.hadoop.hive.serde2.thrift.ThriftDeserializer; import org.apache.hadoop.hive.serde2.thrift.test.Complex; -import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.mapred.SequenceFileInputFormat; import org.apache.hadoop.mapred.SequenceFileOutputFormat; import org.apache.hadoop.mapred.TextInputFormat; import org.apache.hadoop.util.StringUtils; - import org.apache.thrift.protocol.TBinaryProtocol; public class TestHive extends TestCase { @@ -47,26 +46,32 @@ private HiveConf hiveConf; private FileSystem fs; + @Override protected void setUp() throws Exception { super.setUp(); hiveConf = new HiveConf(this.getClass()); fs = FileSystem.get(hiveConf); try { - this.hm = Hive.get(hiveConf); + hm = Hive.get(hiveConf); } catch (Exception e) { System.err.println(StringUtils.stringifyException(e)); - System.err.println("Unable to initialize Hive Metastore using configruation: \n " + hiveConf); + System.err + .println("Unable to initialize Hive Metastore using configruation: \n " + + hiveConf); throw e; } } + @Override protected void tearDown() throws Exception { try { super.tearDown(); Hive.closeCurrent(); } catch (Exception e) { System.err.println(StringUtils.stringifyException(e)); - System.err.println("Unable to close Hive Metastore using configruation: \n " + hiveConf); + System.err + .println("Unable to close Hive Metastore using configruation: \n " + + hiveConf); throw e; } } @@ -76,23 +81,27 @@ // create a simple table and test create, drop, get String tableName = "table_for_testtable"; try { - this.hm.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName); + hm.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName); } catch (HiveException e1) { e1.printStackTrace(); assertTrue("Unable to drop table", false); } Table tbl = new Table(tableName); - List fields = tbl.getCols(); + List fields = tbl.getCols(); - fields.add(new FieldSchema("col1", Constants.INT_TYPE_NAME, "int -- first column")); - fields.add(new FieldSchema("col2", Constants.STRING_TYPE_NAME, "string -- second column")); - fields.add(new FieldSchema("col3", Constants.DOUBLE_TYPE_NAME, "double -- thrift column")); + fields.add(new FieldSchema("col1", Constants.INT_TYPE_NAME, + "int -- first column")); + fields.add(new FieldSchema("col2", Constants.STRING_TYPE_NAME, + "string -- second column")); + fields.add(new FieldSchema("col3", Constants.DOUBLE_TYPE_NAME, + "double -- thrift column")); tbl.setFields(fields); tbl.setOutputFormatClass(HiveIgnoreKeyTextOutputFormat.class); tbl.setInputFormatClass(SequenceFileInputFormat.class); - tbl.setProperty("comment", "this is a test table created as part junit tests"); + tbl.setProperty("comment", + "this is a test table created as part junit tests"); List bucketCols = tbl.getBucketCols(); bucketCols.add("col1"); @@ -103,16 +112,20 @@ assertTrue("Unable to set bucket column for table: " + tableName, false); } - List partCols = new ArrayList(); - partCols.add(new FieldSchema("ds", Constants.STRING_TYPE_NAME, - "partition column, date but in string format as date type is not yet supported in QL")); + List partCols = new ArrayList(); + partCols + .add(new FieldSchema( + "ds", + Constants.STRING_TYPE_NAME, + "partition column, date but in string format as date type is not yet supported in QL")); tbl.setPartCols(partCols); tbl.setNumBuckets((short) 512); tbl.setOwner("pchakka"); tbl.setRetention(10); - // set output format parameters (these are not supported by QL but only for demo purposes) + // set output format parameters (these are not supported by QL but only + // for demo purposes) tbl.setSerdeParam(Constants.FIELD_DELIM, "1"); tbl.setSerdeParam(Constants.LINE_DELIM, "\n"); tbl.setSerdeParam(Constants.MAPKEY_DELIM, "3"); @@ -135,23 +148,34 @@ try { ft = hm.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName); ft.checkValidity(); - assertEquals("Table names didn't match for table: " + tableName, tbl.getName(), ft.getName()); - assertEquals("Table owners didn't match for table: " + tableName, tbl.getOwner(), ft.getOwner()); - assertEquals("Table retention didn't match for table: " + tableName, tbl.getRetention(), ft.getRetention()); - assertEquals("Data location is not set correctly", wh.getDefaultTablePath(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName).toString(), ft.getDataLocation().toString()); - // now that URI is set correctly, set the original table's uri and then compare the two tables + assertEquals("Table names didn't match for table: " + tableName, tbl + .getName(), ft.getName()); + assertEquals("Table owners didn't match for table: " + tableName, tbl + .getOwner(), ft.getOwner()); + assertEquals("Table retention didn't match for table: " + tableName, + tbl.getRetention(), ft.getRetention()); + assertEquals("Data location is not set correctly", wh + .getDefaultTablePath(MetaStoreUtils.DEFAULT_DATABASE_NAME, + tableName).toString(), ft.getDataLocation().toString()); + // now that URI is set correctly, set the original table's uri and then + // compare the two tables tbl.setDataLocation(ft.getDataLocation()); - assertTrue("Tables doesn't match: " + tableName, ft.getTTable().equals(tbl.getTTable())); - assertEquals("Serde is not set correctly", tbl.getDeserializer().getClass().getName(), ft.getDeserializer().getClass().getName()); - assertEquals("SerializationLib is not set correctly", tbl.getSerializationLib(), LazySimpleSerDe.class.getName()); + assertTrue("Tables doesn't match: " + tableName, ft.getTTable().equals( + tbl.getTTable())); + assertEquals("Serde is not set correctly", tbl.getDeserializer() + .getClass().getName(), ft.getDeserializer().getClass().getName()); + assertEquals("SerializationLib is not set correctly", tbl + .getSerializationLib(), LazySimpleSerDe.class.getName()); } catch (HiveException e) { e.printStackTrace(); assertTrue("Unable to fetch table correctly: " + tableName, false); } try { - hm.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName, true, false); - Table ft2 = hm.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName, false); + hm.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName, true, + false); + Table ft2 = hm.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, + tableName, false); assertNull("Unable to drop table ", ft2); } catch (HiveException e) { assertTrue("Unable to drop table: " + tableName, false); @@ -165,13 +189,14 @@ /** * Tests create and fetch of a thrift based table - * @throws Throwable + * + * @throws Throwable */ public void testThriftTable() throws Throwable { String tableName = "table_for_test_thrifttable"; try { try { - this.hm.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName); + hm.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName); } catch (HiveException e1) { System.err.println(StringUtils.stringifyException(e1)); assertTrue("Unable to drop table", false); @@ -181,7 +206,8 @@ tbl.setOutputFormatClass(SequenceFileOutputFormat.class.getName()); tbl.setSerializationLib(ThriftDeserializer.class.getName()); tbl.setSerdeParam(Constants.SERIALIZATION_CLASS, Complex.class.getName()); - tbl.setSerdeParam(Constants.SERIALIZATION_FORMAT, TBinaryProtocol.class.getName()); + tbl.setSerdeParam(Constants.SERIALIZATION_FORMAT, TBinaryProtocol.class + .getName()); try { hm.createTable(tbl); } catch (HiveException e) { @@ -195,15 +221,24 @@ ft = hm.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName); assertNotNull("Unable to fetch table", ft); ft.checkValidity(); - assertEquals("Table names didn't match for table: " + tableName, tbl.getName(), ft.getName()); - assertEquals("Table owners didn't match for table: " + tableName, tbl.getOwner(), ft.getOwner()); - assertEquals("Table retention didn't match for table: " + tableName, tbl.getRetention(), ft.getRetention()); - assertEquals("Data location is not set correctly", wh.getDefaultTablePath(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName).toString(), ft.getDataLocation().toString()); - // now that URI is set correctly, set the original table's uri and then compare the two tables + assertEquals("Table names didn't match for table: " + tableName, tbl + .getName(), ft.getName()); + assertEquals("Table owners didn't match for table: " + tableName, tbl + .getOwner(), ft.getOwner()); + assertEquals("Table retention didn't match for table: " + tableName, + tbl.getRetention(), ft.getRetention()); + assertEquals("Data location is not set correctly", wh + .getDefaultTablePath(MetaStoreUtils.DEFAULT_DATABASE_NAME, + tableName).toString(), ft.getDataLocation().toString()); + // now that URI is set correctly, set the original table's uri and then + // compare the two tables tbl.setDataLocation(ft.getDataLocation()); - assertTrue("Tables doesn't match: " + tableName, ft.getTTable().equals(tbl.getTTable())); - assertEquals("SerializationLib is not set correctly", tbl.getSerializationLib(), ThriftDeserializer.class.getName()); - assertEquals("Serde is not set correctly", tbl.getDeserializer().getClass().getName(), ft.getDeserializer().getClass().getName()); + assertTrue("Tables doesn't match: " + tableName, ft.getTTable() + .equals(tbl.getTTable())); + assertEquals("SerializationLib is not set correctly", tbl + .getSerializationLib(), ThriftDeserializer.class.getName()); + assertEquals("Serde is not set correctly", tbl.getDeserializer() + .getClass().getName(), ft.getDeserializer().getClass().getName()); } catch (HiveException e) { System.err.println(StringUtils.stringifyException(e)); assertTrue("Unable to fetch table correctly: " + tableName, false); @@ -216,14 +251,16 @@ } } - private static Table createTestTable(String dbName, String tableName) throws HiveException { + private static Table createTestTable(String dbName, String tableName) + throws HiveException { Table tbl = new Table(tableName); tbl.getTTable().setDbName(dbName); tbl.setInputFormatClass(SequenceFileInputFormat.class.getName()); tbl.setOutputFormatClass(SequenceFileOutputFormat.class.getName()); tbl.setSerializationLib(ThriftDeserializer.class.getName()); tbl.setSerdeParam(Constants.SERIALIZATION_CLASS, Complex.class.getName()); - tbl.setSerdeParam(Constants.SERIALIZATION_FORMAT, TBinaryProtocol.class.getName()); + tbl.setSerdeParam(Constants.SERIALIZATION_FORMAT, TBinaryProtocol.class + .getName()); return tbl; } @@ -250,17 +287,17 @@ fts = hm.getTablesForDb(dbName, ".*1"); assertEquals(1, fts.size()); assertEquals(ts.get(0), fts.get(0)); - - //also test getting a table from a specific db + + // also test getting a table from a specific db Table table1 = hm.getTable(dbName, table1Name); assertNotNull(table1); assertEquals(table1Name, table1.getName()); - + assertTrue(fs.exists(table1.getPath())); - //and test dropping this specific table + // and test dropping this specific table hm.dropTable(dbName, table1Name); assertFalse(fs.exists(table1.getPath())); - + hm.dropDatabase(dbName); } catch (Throwable e) { System.err.println(StringUtils.stringifyException(e)); @@ -286,7 +323,8 @@ part_cols.add("ds"); part_cols.add("hr"); try { - hm.createTable(tableName, cols, part_cols, TextInputFormat.class, HiveIgnoreKeyTextOutputFormat.class); + hm.createTable(tableName, cols, part_cols, TextInputFormat.class, + HiveIgnoreKeyTextOutputFormat.class); } catch (HiveException e) { System.err.println(StringUtils.stringifyException(e)); assertTrue("Unable to create table: " + tableName, false); Index: ql/src/test/org/apache/hadoop/hive/ql/history/TestHiveHistory.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/history/TestHiveHistory.java (revision 901519) +++ ql/src/test/org/apache/hadoop/hive/ql/history/TestHiveHistory.java (working copy) @@ -54,7 +54,8 @@ * intialize the tables */ - protected void setUp(){ + @Override + protected void setUp() { try { conf = new HiveConf(HiveHistory.class); @@ -102,13 +103,13 @@ e.printStackTrace(); throw new RuntimeException("Encountered throwable"); } -} + } /** * check history file output for this query.als */ public void testSimpleQuery() { - LineageInfo lep = new LineageInfo(); + new LineageInfo(); try { // NOTE: It is critical to do this here so that log4j is reinitialized @@ -145,8 +146,7 @@ fail("jobInfo Map size not 1"); } - - cmd = (String)jobInfoMap.keySet().toArray()[0]; + cmd = (String) jobInfoMap.keySet().toArray()[0]; QueryInfo ji = jobInfoMap.get(cmd); if (!ji.hm.get(Keys.QUERY_NUM_TASKS.name()).equals("1")) { Index: ql/src/test/org/apache/hadoop/hive/ql/hooks/PostExecutePrinter.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/hooks/PostExecutePrinter.java (revision 901519) +++ ql/src/test/org/apache/hadoop/hive/ql/hooks/PostExecutePrinter.java (working copy) @@ -19,35 +19,36 @@ package org.apache.hadoop.hive.ql.hooks; import java.util.Set; -import org.apache.hadoop.security.UserGroupInformation; + import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; +import org.apache.hadoop.security.UserGroupInformation; /** - * Implementation of a post execute hook that simply prints out its - * parameters to standard output. + * Implementation of a post execute hook that simply prints out its parameters + * to standard output. */ public class PostExecutePrinter implements PostExecute { @Override public void run(SessionState sess, Set inputs, - Set outputs, UserGroupInformation ugi) - throws Exception { + Set outputs, UserGroupInformation ugi) throws Exception { LogHelper console = SessionState.getConsole(); - if (console == null) + if (console == null) { return; + } if (sess != null) { console.printError("POSTHOOK: query: " + sess.getCmd().trim()); console.printError("POSTHOOK: type: " + sess.getCommandType()); } - for(ReadEntity re: inputs) { + for (ReadEntity re : inputs) { console.printError("POSTHOOK: Input: " + re.toString()); } - for(WriteEntity we: outputs) { + for (WriteEntity we : outputs) { console.printError("POSTHOOK: Output: " + we.toString()); } } Index: ql/src/test/org/apache/hadoop/hive/ql/hooks/PreExecutePrinter.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/hooks/PreExecutePrinter.java (revision 901519) +++ ql/src/test/org/apache/hadoop/hive/ql/hooks/PreExecutePrinter.java (working copy) @@ -19,35 +19,36 @@ package org.apache.hadoop.hive.ql.hooks; import java.util.Set; -import org.apache.hadoop.security.UserGroupInformation; + import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; +import org.apache.hadoop.security.UserGroupInformation; /** - * Implementation of a pre execute hook that simply prints out its - * parameters to standard output. + * Implementation of a pre execute hook that simply prints out its parameters to + * standard output. */ public class PreExecutePrinter implements PreExecute { @Override public void run(SessionState sess, Set inputs, - Set outputs, UserGroupInformation ugi) - throws Exception { + Set outputs, UserGroupInformation ugi) throws Exception { LogHelper console = SessionState.getConsole(); - if (console == null) + if (console == null) { return; + } if (sess != null) { console.printError("PREHOOK: query: " + sess.getCmd().trim()); console.printError("PREHOOK: type: " + sess.getCommandType()); } - for(ReadEntity re: inputs) { + for (ReadEntity re : inputs) { console.printError("PREHOOK: Input: " + re.toString()); } - for(WriteEntity we: outputs) { + for (WriteEntity we : outputs) { console.printError("PREHOOK: Output: " + we.toString()); } } Index: ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java (revision 901519) +++ ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java (working copy) @@ -53,16 +53,17 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.mapred.TextInputFormat; - /** - * Mimics the actual query compiler in generating end to end plans and testing them out - * + * Mimics the actual query compiler in generating end to end plans and testing + * them out + * */ public class TestExecDriver extends TestCase { static HiveConf conf; - static private String tmpdir = "/tmp/"+System.getProperty("user.name")+"/"; + static private String tmpdir = "/tmp/" + System.getProperty("user.name") + + "/"; static private Path tmppath = new Path(tmpdir); static private Hive db; static private FileSystem fs; @@ -72,31 +73,33 @@ conf = new HiveConf(ExecDriver.class); fs = FileSystem.get(conf); - if(fs.exists(tmppath) && !fs.getFileStatus(tmppath).isDir()) { - throw new RuntimeException (tmpdir + " exists but is not a directory"); + if (fs.exists(tmppath) && !fs.getFileStatus(tmppath).isDir()) { + throw new RuntimeException(tmpdir + " exists but is not a directory"); } - if(!fs.exists(tmppath)) { - if(!fs.mkdirs(tmppath)) { - throw new RuntimeException ("Could not make scratch directory " + tmpdir); + if (!fs.exists(tmppath)) { + if (!fs.mkdirs(tmppath)) { + throw new RuntimeException("Could not make scratch directory " + + tmpdir); } } - for(Object one: Utilities.makeList("mapplan1.out", "mapplan2.out", - "mapredplan1.out", "mapredplan2.out", "mapredplan3.out", "mapredplan4.out", - "mapredplan5.out", "mapredplan6.out")) { - Path onedir = new Path(tmppath, (String)one); - if(fs.exists(onedir)) { + for (Object one : Utilities.makeList("mapplan1.out", "mapplan2.out", + "mapredplan1.out", "mapredplan2.out", "mapredplan3.out", + "mapredplan4.out", "mapredplan5.out", "mapredplan6.out")) { + Path onedir = new Path(tmppath, (String) one); + if (fs.exists(onedir)) { fs.delete(onedir, true); } } // copy the test files into hadoop if required. int i = 0; - Path [] hadoopDataFile = new Path [2]; - String [] testFiles = {"kv1.txt", "kv2.txt"}; - String testFileDir = "file://" + conf.get("test.data.files").replace('\\', '/').replace("c:", ""); - for(String oneFile: testFiles) { + Path[] hadoopDataFile = new Path[2]; + String[] testFiles = { "kv1.txt", "kv2.txt" }; + String testFileDir = "file://" + + conf.get("test.data.files").replace('\\', '/').replace("c:", ""); + for (String oneFile : testFiles) { Path localDataFile = new Path(testFileDir, oneFile); hadoopDataFile[i] = new Path(tmppath, oneFile); fs.copyFromLocalFile(false, true, localDataFile, hadoopDataFile[i]); @@ -106,78 +109,77 @@ // load the test files into tables i = 0; db = Hive.get(conf); - String [] srctables = {"src", "src2"}; + String[] srctables = { "src", "src2" }; LinkedList cols = new LinkedList(); cols.add("key"); cols.add("value"); - for(String src: srctables) { + for (String src : srctables) { db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, src, true, true); - db.createTable(src, cols, null, TextInputFormat.class, IgnoreKeyTextOutputFormat.class); + db.createTable(src, cols, null, TextInputFormat.class, + IgnoreKeyTextOutputFormat.class); db.loadTable(hadoopDataFile[i], src, false, null); i++; } } catch (Throwable e) { e.printStackTrace(); - throw new RuntimeException ("Encountered throwable"); + throw new RuntimeException("Encountered throwable"); } } - mapredWork mr; protected void setUp() { - mr = PlanUtils.getMapRedWork(); + mr = PlanUtils.getMapRedWork(); } - private static void fileDiff(String datafile, String testdir) throws Exception { + private static void fileDiff(String datafile, String testdir) + throws Exception { String testFileDir = conf.get("test.data.files"); System.out.println(testFileDir); FileInputStream fi_gold = new FileInputStream(new File(testFileDir, - datafile)); + datafile)); // inbuilt assumption that the testdir has only one output file. - Path di_test = new Path (tmppath, testdir); - if(!fs.exists(di_test)) { - throw new RuntimeException (tmpdir + testdir + " does not exist"); + Path di_test = new Path(tmppath, testdir); + if (!fs.exists(di_test)) { + throw new RuntimeException(tmpdir + testdir + " does not exist"); } - if(!fs.getFileStatus(di_test).isDir()) { - throw new RuntimeException (tmpdir + testdir + " is not a directory"); + if (!fs.getFileStatus(di_test).isDir()) { + throw new RuntimeException(tmpdir + testdir + " is not a directory"); } - FSDataInputStream fi_test = fs.open( (fs.listStatus(di_test))[0].getPath() ); + FSDataInputStream fi_test = fs.open((fs.listStatus(di_test))[0].getPath()); - if(!Utilities.contentsEqual(fi_gold, fi_test, false)) { + if (!Utilities.contentsEqual(fi_gold, fi_test, false)) { System.out.println(di_test.toString() + " does not match " + datafile); assertEquals(false, true); } } - private filterDesc getTestFilterDesc(String column) { ArrayList children1 = new ArrayList(); - children1.add(new exprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, column, "", false)); + children1.add(new exprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, + column, "", false)); exprNodeDesc lhs = new exprNodeGenericFuncDesc( - TypeInfoFactory.doubleTypeInfo, - FunctionRegistry.getFunctionInfo(Constants.DOUBLE_TYPE_NAME).getGenericUDF(), - children1); - + TypeInfoFactory.doubleTypeInfo, FunctionRegistry.getFunctionInfo( + Constants.DOUBLE_TYPE_NAME).getGenericUDF(), children1); + ArrayList children2 = new ArrayList(); - children2.add(new exprNodeConstantDesc(TypeInfoFactory.longTypeInfo, Long.valueOf(100))); + children2.add(new exprNodeConstantDesc(TypeInfoFactory.longTypeInfo, Long + .valueOf(100))); exprNodeDesc rhs = new exprNodeGenericFuncDesc( - TypeInfoFactory.doubleTypeInfo, - FunctionRegistry.getFunctionInfo(Constants.DOUBLE_TYPE_NAME).getGenericUDF(), - children2); - + TypeInfoFactory.doubleTypeInfo, FunctionRegistry.getFunctionInfo( + Constants.DOUBLE_TYPE_NAME).getGenericUDF(), children2); + ArrayList children3 = new ArrayList(); children3.add(lhs); children3.add(rhs); - + exprNodeDesc desc = new exprNodeGenericFuncDesc( - TypeInfoFactory.booleanTypeInfo, - FunctionRegistry.getFunctionInfo("<").getGenericUDF(), - children3); - + TypeInfoFactory.booleanTypeInfo, FunctionRegistry.getFunctionInfo("<") + .getGenericUDF(), children3); + return new filterDesc(desc, false); } @@ -185,13 +187,11 @@ private void populateMapPlan1(Table src) { mr.setNumReduceTasks(Integer.valueOf(0)); - Operator op2 = OperatorFactory.get(new fileSinkDesc - (tmpdir + "mapplan1.out", - Utilities.defaultTd, true)); - Operator op1 = - OperatorFactory.get(getTestFilterDesc("key"), op2); + Operator op2 = OperatorFactory.get(new fileSinkDesc(tmpdir + + "mapplan1.out", Utilities.defaultTd, true)); + Operator op1 = OperatorFactory.get(getTestFilterDesc("key"), + op2); - Utilities.addMapWork(mr, src, "a", op1); } @@ -199,50 +199,44 @@ private void populateMapPlan2(Table src) { mr.setNumReduceTasks(Integer.valueOf(0)); - Operator op3 = OperatorFactory.get(new fileSinkDesc - (tmpdir + "mapplan2.out", - Utilities.defaultTd, false)); + Operator op3 = OperatorFactory.get(new fileSinkDesc(tmpdir + + "mapplan2.out", Utilities.defaultTd, false)); - Operator op2 = OperatorFactory.get - (new scriptDesc("/bin/cat", - PlanUtils.getDefaultTableDesc("" + Utilities.tabCode, "key,value"), - TextRecordWriter.class, - PlanUtils.getDefaultTableDesc("" + Utilities.tabCode, "key,value"), - TextRecordReader.class), - op3); + Operator op2 = OperatorFactory.get(new scriptDesc("/bin/cat", + PlanUtils.getDefaultTableDesc("" + Utilities.tabCode, "key,value"), + TextRecordWriter.class, PlanUtils.getDefaultTableDesc("" + + Utilities.tabCode, "key,value"), TextRecordReader.class), op3); + Operator op1 = OperatorFactory.get(getTestFilterDesc("key"), + op2); - Operator op1 = - OperatorFactory.get(getTestFilterDesc("key"), op2); - - Utilities.addMapWork(mr, src, "a", op1); } @SuppressWarnings("unchecked") private void populateMapRedPlan1(Table src) { mr.setNumReduceTasks(Integer.valueOf(1)); - + ArrayList outputColumns = new ArrayList(); - for (int i = 0; i < 2; i++) + for (int i = 0; i < 2; i++) { outputColumns.add("_col" + i); + } // map-side work - Operator op1 = OperatorFactory.get - (PlanUtils.getReduceSinkDesc - (Utilities.makeList(getStringColumn("key")), - Utilities.makeList(getStringColumn("value")), outputColumns, true, -1, 1, -1)); + Operator op1 = OperatorFactory.get(PlanUtils + .getReduceSinkDesc(Utilities.makeList(getStringColumn("key")), + Utilities.makeList(getStringColumn("value")), outputColumns, true, + -1, 1, -1)); Utilities.addMapWork(mr, src, "a", op1); mr.setKeyDesc(op1.getConf().getKeySerializeInfo()); mr.getTagToValueDesc().add(op1.getConf().getValueSerializeInfo()); // reduce side work - Operator op3 = OperatorFactory.get(new fileSinkDesc - (tmpdir + "mapredplan1.out", - Utilities.defaultTd, false)); + Operator op3 = OperatorFactory.get(new fileSinkDesc(tmpdir + + "mapredplan1.out", Utilities.defaultTd, false)); - Operator op2 = OperatorFactory.get - (new extractDesc(getStringColumn(Utilities.ReduceField.VALUE.toString())), op3); + Operator op2 = OperatorFactory.get(new extractDesc( + getStringColumn(Utilities.ReduceField.VALUE.toString())), op3); mr.setReducer(op2); } @@ -251,29 +245,28 @@ private void populateMapRedPlan2(Table src) { mr.setNumReduceTasks(Integer.valueOf(1)); ArrayList outputColumns = new ArrayList(); - for (int i = 0; i < 2; i++) + for (int i = 0; i < 2; i++) { outputColumns.add("_col" + i); + } // map-side work - Operator op1 = OperatorFactory.get - (PlanUtils.getReduceSinkDesc - (Utilities.makeList(getStringColumn("key")), - Utilities.makeList(getStringColumn("key"), - getStringColumn("value")), outputColumns, false, -1, 1, -1)); + Operator op1 = OperatorFactory.get(PlanUtils + .getReduceSinkDesc(Utilities.makeList(getStringColumn("key")), + Utilities + .makeList(getStringColumn("key"), getStringColumn("value")), + outputColumns, false, -1, 1, -1)); Utilities.addMapWork(mr, src, "a", op1); mr.setKeyDesc(op1.getConf().getKeySerializeInfo()); mr.getTagToValueDesc().add(op1.getConf().getValueSerializeInfo()); // reduce side work - Operator op4 = OperatorFactory.get(new fileSinkDesc - (tmpdir + "mapredplan2.out", - Utilities.defaultTd, false)); + Operator op4 = OperatorFactory.get(new fileSinkDesc(tmpdir + + "mapredplan2.out", Utilities.defaultTd, false)); - Operator op3 = - OperatorFactory.get(getTestFilterDesc("0"), op4); + Operator op3 = OperatorFactory.get(getTestFilterDesc("0"), op4); - Operator op2 = OperatorFactory.get - (new extractDesc(getStringColumn(Utilities.ReduceField.VALUE.toString())), op3); + Operator op2 = OperatorFactory.get(new extractDesc( + getStringColumn(Utilities.ReduceField.VALUE.toString())), op3); mr.setReducer(op2); } @@ -282,49 +275,42 @@ * test reduce with multiple tagged inputs */ @SuppressWarnings("unchecked") - private void populateMapRedPlan3(Table src, Table src2) { + private void populateMapRedPlan3(Table src, Table src2) { mr.setNumReduceTasks(Integer.valueOf(5)); mr.setNeedsTagging(true); ArrayList outputColumns = new ArrayList(); - for (int i = 0; i < 2; i++) + for (int i = 0; i < 2; i++) { outputColumns.add("_col" + i); + } // map-side work - Operator op1 = OperatorFactory.get - (PlanUtils.getReduceSinkDesc - (Utilities.makeList(getStringColumn("key")), - Utilities.makeList - (getStringColumn("value")), outputColumns, true, Byte.valueOf((byte)0), 1, -1)); + Operator op1 = OperatorFactory.get(PlanUtils + .getReduceSinkDesc(Utilities.makeList(getStringColumn("key")), + Utilities.makeList(getStringColumn("value")), outputColumns, true, + Byte.valueOf((byte) 0), 1, -1)); Utilities.addMapWork(mr, src, "a", op1); mr.setKeyDesc(op1.getConf().getKeySerializeInfo()); mr.getTagToValueDesc().add(op1.getConf().getValueSerializeInfo()); - Operator op2 = OperatorFactory.get - (PlanUtils.getReduceSinkDesc - (Utilities.makeList(getStringColumn("key")), - Utilities.makeList(getStringColumn("key")), - outputColumns, true, - Byte.valueOf((byte)1), - Integer.MAX_VALUE, -1)); + Operator op2 = OperatorFactory.get(PlanUtils + .getReduceSinkDesc(Utilities.makeList(getStringColumn("key")), + Utilities.makeList(getStringColumn("key")), outputColumns, true, + Byte.valueOf((byte) 1), Integer.MAX_VALUE, -1)); Utilities.addMapWork(mr, src2, "b", op2); mr.getTagToValueDesc().add(op2.getConf().getValueSerializeInfo()); // reduce side work - Operator op4 = OperatorFactory.get(new fileSinkDesc - (tmpdir + "mapredplan3.out", - Utilities.defaultTd, false)); + Operator op4 = OperatorFactory.get(new fileSinkDesc(tmpdir + + "mapredplan3.out", Utilities.defaultTd, false)); - Operator op5 = OperatorFactory.get - (new selectDesc - (Utilities.makeList - (getStringColumn(Utilities.ReduceField.ALIAS.toString()), - new exprNodeFieldDesc(TypeInfoFactory.stringTypeInfo, - new exprNodeColumnDesc(TypeInfoFactory.getListTypeInfo( - TypeInfoFactory.stringTypeInfo), - Utilities.ReduceField.VALUE.toString(), "", false), - "0", - false)), outputColumns), op4); + Operator op5 = OperatorFactory.get(new selectDesc(Utilities + .makeList(getStringColumn(Utilities.ReduceField.ALIAS.toString()), + new exprNodeFieldDesc(TypeInfoFactory.stringTypeInfo, + new exprNodeColumnDesc(TypeInfoFactory + .getListTypeInfo(TypeInfoFactory.stringTypeInfo), + Utilities.ReduceField.VALUE.toString(), "", false), "0", + false)), outputColumns), op4); mr.setReducer(op5); } @@ -335,78 +321,70 @@ // map-side work ArrayList outputColumns = new ArrayList(); - for (int i = 0; i < 2; i++) + for (int i = 0; i < 2; i++) { outputColumns.add("_col" + i); - Operator op1 = OperatorFactory.get - (PlanUtils.getReduceSinkDesc - (Utilities.makeList(getStringColumn("tkey")), - Utilities.makeList(getStringColumn("tkey"), - getStringColumn("tvalue")), - outputColumns, false, - -1, 1, -1)); + } + Operator op1 = OperatorFactory.get(PlanUtils + .getReduceSinkDesc(Utilities.makeList(getStringColumn("tkey")), + Utilities.makeList(getStringColumn("tkey"), + getStringColumn("tvalue")), outputColumns, false, -1, 1, -1)); - Operator op0 = OperatorFactory.get - (new scriptDesc("/bin/cat", + Operator op0 = OperatorFactory.get(new scriptDesc("/bin/cat", PlanUtils.getDefaultTableDesc("" + Utilities.tabCode, "key,value"), - TextRecordWriter.class, - PlanUtils.getDefaultTableDesc("" + Utilities.tabCode, "tkey,tvalue"), - TextRecordReader.class), - op1); + TextRecordWriter.class, PlanUtils.getDefaultTableDesc("" + + Utilities.tabCode, "tkey,tvalue"), TextRecordReader.class), op1); - Operator op4 = OperatorFactory.get(new selectDesc( - Utilities.makeList(getStringColumn("key"), - getStringColumn("value")), outputColumns), op0); + Operator op4 = OperatorFactory.get(new selectDesc(Utilities + .makeList(getStringColumn("key"), getStringColumn("value")), + outputColumns), op0); Utilities.addMapWork(mr, src, "a", op4); mr.setKeyDesc(op1.getConf().getKeySerializeInfo()); mr.getTagToValueDesc().add(op1.getConf().getValueSerializeInfo()); // reduce side work - Operator op3 = OperatorFactory.get(new fileSinkDesc - (tmpdir + "mapredplan4.out", - Utilities.defaultTd, false)); + Operator op3 = OperatorFactory.get(new fileSinkDesc(tmpdir + + "mapredplan4.out", Utilities.defaultTd, false)); - Operator op2 = OperatorFactory.get - (new extractDesc(getStringColumn(Utilities.ReduceField.VALUE.toString())), op3); + Operator op2 = OperatorFactory.get(new extractDesc( + getStringColumn(Utilities.ReduceField.VALUE.toString())), op3); mr.setReducer(op2); } public static exprNodeColumnDesc getStringColumn(String columnName) { - return new exprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, columnName, "", false); + return new exprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, columnName, + "", false); } - + @SuppressWarnings("unchecked") private void populateMapRedPlan5(Table src) { mr.setNumReduceTasks(Integer.valueOf(1)); // map-side work ArrayList outputColumns = new ArrayList(); - for (int i = 0; i < 2; i++) + for (int i = 0; i < 2; i++) { outputColumns.add("_col" + i); - Operator op0 = OperatorFactory.get - (PlanUtils.getReduceSinkDesc - (Utilities.makeList(getStringColumn("0")), - Utilities.makeList(getStringColumn("0"), - getStringColumn("1")), - outputColumns, false, - -1, 1, -1)); + } + Operator op0 = OperatorFactory.get(PlanUtils + .getReduceSinkDesc(Utilities.makeList(getStringColumn("0")), Utilities + .makeList(getStringColumn("0"), getStringColumn("1")), + outputColumns, false, -1, 1, -1)); - Operator op4 = OperatorFactory.get(new selectDesc( - Utilities.makeList(getStringColumn("key"), - getStringColumn("value")), outputColumns), op0); + Operator op4 = OperatorFactory.get(new selectDesc(Utilities + .makeList(getStringColumn("key"), getStringColumn("value")), + outputColumns), op0); Utilities.addMapWork(mr, src, "a", op4); mr.setKeyDesc(op0.getConf().getKeySerializeInfo()); mr.getTagToValueDesc().add(op0.getConf().getValueSerializeInfo()); // reduce side work - Operator op3 = OperatorFactory.get(new fileSinkDesc - (tmpdir + "mapredplan5.out", - Utilities.defaultTd, false)); + Operator op3 = OperatorFactory.get(new fileSinkDesc(tmpdir + + "mapredplan5.out", Utilities.defaultTd, false)); - Operator op2 = OperatorFactory.get - (new extractDesc(getStringColumn(Utilities.ReduceField.VALUE.toString())), op3); + Operator op2 = OperatorFactory.get(new extractDesc( + getStringColumn(Utilities.ReduceField.VALUE.toString())), op3); mr.setReducer(op2); } @@ -417,49 +395,43 @@ // map-side work ArrayList outputColumns = new ArrayList(); - for (int i = 0; i < 2; i++) + for (int i = 0; i < 2; i++) { outputColumns.add("_col" + i); - Operator op1 = OperatorFactory.get - (PlanUtils.getReduceSinkDesc( - Utilities.makeList(getStringColumn("tkey")), - Utilities.makeList(getStringColumn("tkey"), - getStringColumn("tvalue")), - outputColumns, false, - -1, 1, -1)); + } + Operator op1 = OperatorFactory.get(PlanUtils + .getReduceSinkDesc(Utilities.makeList(getStringColumn("tkey")), + Utilities.makeList(getStringColumn("tkey"), + getStringColumn("tvalue")), outputColumns, false, -1, 1, -1)); - Operator op0 = OperatorFactory.get - (new scriptDesc("\'/bin/cat\'", - PlanUtils.getDefaultTableDesc("" + Utilities.tabCode, "tkey,tvalue"), - TextRecordWriter.class, - PlanUtils.getDefaultTableDesc("" + Utilities.tabCode, "tkey,tvalue"), - TextRecordReader.class), - op1); + Operator op0 = OperatorFactory.get(new scriptDesc( + "\'/bin/cat\'", PlanUtils.getDefaultTableDesc("" + Utilities.tabCode, + "tkey,tvalue"), TextRecordWriter.class, PlanUtils + .getDefaultTableDesc("" + Utilities.tabCode, "tkey,tvalue"), + TextRecordReader.class), op1); - Operator op4 = OperatorFactory.get(new selectDesc( - Utilities.makeList(getStringColumn("key"), - getStringColumn("value")), outputColumns), op0); + Operator op4 = OperatorFactory.get(new selectDesc(Utilities + .makeList(getStringColumn("key"), getStringColumn("value")), + outputColumns), op0); Utilities.addMapWork(mr, src, "a", op4); mr.setKeyDesc(op1.getConf().getKeySerializeInfo()); mr.getTagToValueDesc().add(op1.getConf().getValueSerializeInfo()); // reduce side work - Operator op3 = OperatorFactory.get(new fileSinkDesc - (tmpdir + "mapredplan6.out", - Utilities.defaultTd, false)); + Operator op3 = OperatorFactory.get(new fileSinkDesc(tmpdir + + "mapredplan6.out", Utilities.defaultTd, false)); - Operator op2 = - OperatorFactory.get(getTestFilterDesc("0"), op3); + Operator op2 = OperatorFactory.get(getTestFilterDesc("0"), op3); - Operator op5 = OperatorFactory.get - (new extractDesc(getStringColumn(Utilities.ReduceField.VALUE.toString())), op2); + Operator op5 = OperatorFactory.get(new extractDesc( + getStringColumn(Utilities.ReduceField.VALUE.toString())), op2); mr.setReducer(op5); } private File generatePlanFile() throws Exception { - File scratchDir = new File( - (new HiveConf(TestExecDriver.class)).getVar(ConfVars.SCRATCHDIR)); + File scratchDir = new File((new HiveConf(TestExecDriver.class)) + .getVar(ConfVars.SCRATCHDIR)); File planFile = File.createTempFile("plan", ".xml", scratchDir); System.out.println("Generating plan file " + planFile.toString()); FileOutputStream out = new FileOutputStream(planFile); @@ -469,31 +441,34 @@ private void executePlan(File planFile) throws Exception { String testName = new Exception().getStackTrace()[1].getMethodName(); - String cmdLine = conf.getVar(HiveConf.ConfVars.HADOOPBIN) + " jar " + conf.getJar() + - " org.apache.hadoop.hive.ql.exec.ExecDriver -plan " + - planFile.toString() + " " + ExecDriver.generateCmdLine(conf); + String cmdLine = conf.getVar(HiveConf.ConfVars.HADOOPBIN) + " jar " + + conf.getJar() + " org.apache.hadoop.hive.ql.exec.ExecDriver -plan " + + planFile.toString() + " " + ExecDriver.generateCmdLine(conf); System.out.println("Executing: " + cmdLine); Process executor = Runtime.getRuntime().exec(cmdLine); - StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, System.out); - StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, System.err); + StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), + null, System.out); + StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), + null, System.err); outPrinter.start(); errPrinter.start(); int exitVal = executor.waitFor(); - if(exitVal != 0) { - System.out.println(testName + " execution failed with exit status: " + exitVal); + if (exitVal != 0) { + System.out.println(testName + " execution failed with exit status: " + + exitVal); assertEquals(true, false); } System.out.println(testName + " execution completed successfully"); } public void testMapPlan1() throws Exception { - + System.out.println("Beginning testMapPlan1"); - + try { populateMapPlan1(db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, "src")); File planFile = generatePlanFile(); @@ -525,7 +500,8 @@ System.out.println("Beginning testMapRedPlan1"); try { - populateMapRedPlan1(db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, "src")); + populateMapRedPlan1(db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, + "src")); File planFile = generatePlanFile(); executePlan(planFile); fileDiff("kv1.val.sorted.txt", "mapredplan1.out"); @@ -540,7 +516,8 @@ System.out.println("Beginning testMapPlan2"); try { - populateMapRedPlan2(db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, "src")); + populateMapRedPlan2(db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, + "src")); File planFile = generatePlanFile(); executePlan(planFile); fileDiff("lt100.sorted.txt", "mapredplan2.out"); @@ -555,8 +532,8 @@ System.out.println("Beginning testMapPlan3"); try { - populateMapRedPlan3(db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, "src"), - db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, "src2")); + populateMapRedPlan3(db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, + "src"), db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, "src2")); File planFile = generatePlanFile(); executePlan(planFile); fileDiff("kv1kv2.cogroup.txt", "mapredplan3.out"); @@ -566,13 +543,13 @@ } } - public void testMapRedPlan4() throws Exception { System.out.println("Beginning testMapPlan4"); try { - populateMapRedPlan4(db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, "src")); + populateMapRedPlan4(db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, + "src")); File planFile = generatePlanFile(); executePlan(planFile); fileDiff("kv1.string-sorted.txt", "mapredplan4.out"); @@ -587,7 +564,8 @@ System.out.println("Beginning testMapPlan5"); try { - populateMapRedPlan5(db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, "src")); + populateMapRedPlan5(db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, + "src")); File planFile = generatePlanFile(); executePlan(planFile); fileDiff("kv1.string-sorted.txt", "mapredplan5.out"); @@ -602,7 +580,8 @@ System.out.println("Beginning testMapPlan6"); try { - populateMapRedPlan6(db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, "src")); + populateMapRedPlan6(db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, + "src")); File planFile = generatePlanFile(); executePlan(planFile); fileDiff("lt100.sorted.txt", "mapredplan6.out"); Index: ql/src/test/org/apache/hadoop/hive/ql/exec/TestExpressionEvaluator.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/exec/TestExpressionEvaluator.java (revision 901519) +++ ql/src/test/org/apache/hadoop/hive/ql/exec/TestExpressionEvaluator.java (working copy) @@ -18,29 +18,25 @@ package org.apache.hadoop.hive.ql.exec; +import java.util.ArrayList; + import junit.framework.TestCase; -import java.io.*; -import java.util.*; -import org.apache.hadoop.hive.serde.Constants; -import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory; import org.apache.hadoop.hive.ql.plan.exprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.exprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.exprNodeDesc; import org.apache.hadoop.hive.ql.plan.exprNodeGenericFuncDesc; -import org.apache.hadoop.hive.ql.plan.PlanUtils.ExpressionTypes; +import org.apache.hadoop.hive.serde.Constants; +import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; import org.apache.hadoop.io.Text; public class TestExpressionEvaluator extends TestCase { @@ -56,21 +52,19 @@ ArrayList names; ArrayList typeInfos; TypeInfo dataType; - + public TestExpressionEvaluator() { - col1 = new ArrayList (); + col1 = new ArrayList(); col1.add(new Text("0")); col1.add(new Text("1")); col1.add(new Text("2")); col1.add(new Text("3")); - col1Type = TypeInfoFactory.getListTypeInfo( - TypeInfoFactory.stringTypeInfo); - cola = new ArrayList (); + col1Type = TypeInfoFactory.getListTypeInfo(TypeInfoFactory.stringTypeInfo); + cola = new ArrayList(); cola.add(new Text("a")); cola.add(new Text("b")); cola.add(new Text("c")); - colaType = TypeInfoFactory.getListTypeInfo( - TypeInfoFactory.stringTypeInfo); + colaType = TypeInfoFactory.getListTypeInfo(TypeInfoFactory.stringTypeInfo); try { data = new ArrayList(); data.add(col1); @@ -82,30 +76,34 @@ typeInfos.add(col1Type); typeInfos.add(colaType); dataType = TypeInfoFactory.getStructTypeInfo(names, typeInfos); - + r = new InspectableObject(); r.o = data; - r.oi = TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(dataType); + r.oi = TypeInfoUtils + .getStandardWritableObjectInspectorFromTypeInfo(dataType); } catch (Throwable e) { e.printStackTrace(); - throw new RuntimeException (e); + throw new RuntimeException(e); } } + @Override protected void setUp() { } public void testExprNodeColumnEvaluator() throws Throwable { try { // get a evaluator for a simple field expression - exprNodeDesc exprDesc = new exprNodeColumnDesc(colaType, "cola", "", false); + exprNodeDesc exprDesc = new exprNodeColumnDesc(colaType, "cola", "", + false); ExprNodeEvaluator eval = ExprNodeEvaluatorFactory.get(exprDesc); // evaluate on row ObjectInspector resultOI = eval.initialize(r.oi); Object resultO = eval.evaluate(r.o); - - Object standardResult = ObjectInspectorUtils.copyToStandardObject(resultO, resultOI, ObjectInspectorCopyOption.WRITABLE); + + Object standardResult = ObjectInspectorUtils.copyToStandardObject( + resultO, resultOI, ObjectInspectorCopyOption.WRITABLE); assertEquals(cola, standardResult); System.out.println("ExprNodeColumnEvaluator ok"); } catch (Throwable e) { @@ -117,32 +115,35 @@ private static exprNodeDesc getListIndexNode(exprNodeDesc node, int index) { return getListIndexNode(node, new exprNodeConstantDesc(index)); } - - private static exprNodeDesc getListIndexNode(exprNodeDesc node, exprNodeDesc index) { + + private static exprNodeDesc getListIndexNode(exprNodeDesc node, + exprNodeDesc index) { ArrayList children = new ArrayList(2); children.add(node); children.add(index); - return new exprNodeGenericFuncDesc( - ((ListTypeInfo)node.getTypeInfo()).getListElementTypeInfo(), - FunctionRegistry.getGenericUDFForIndex(), - children); + return new exprNodeGenericFuncDesc(((ListTypeInfo) node.getTypeInfo()) + .getListElementTypeInfo(), FunctionRegistry.getGenericUDFForIndex(), + children); } - + public void testExprNodeFuncEvaluator() throws Throwable { try { // get a evaluator for a string concatenation expression - exprNodeDesc col1desc = new exprNodeColumnDesc(col1Type, "col1", "", false); - exprNodeDesc coladesc = new exprNodeColumnDesc(colaType, "cola", "", false); + exprNodeDesc col1desc = new exprNodeColumnDesc(col1Type, "col1", "", + false); + exprNodeDesc coladesc = new exprNodeColumnDesc(colaType, "cola", "", + false); exprNodeDesc col11desc = getListIndexNode(col1desc, 1); exprNodeDesc cola0desc = getListIndexNode(coladesc, 0); - exprNodeDesc func1 = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("concat", col11desc, cola0desc); + exprNodeDesc func1 = TypeCheckProcFactory.DefaultExprProcessor + .getFuncExprNodeDesc("concat", col11desc, cola0desc); ExprNodeEvaluator eval = ExprNodeEvaluatorFactory.get(func1); // evaluate on row ObjectInspector resultOI = eval.initialize(r.oi); Object resultO = eval.evaluate(r.o); - assertEquals("1a", - ObjectInspectorUtils.copyToStandardObject(resultO, resultOI, ObjectInspectorCopyOption.JAVA)); + assertEquals("1a", ObjectInspectorUtils.copyToStandardObject(resultO, + resultOI, ObjectInspectorCopyOption.JAVA)); System.out.println("ExprNodeFuncEvaluator ok"); } catch (Throwable e) { e.printStackTrace(); @@ -153,16 +154,19 @@ public void testExprNodeConversionEvaluator() throws Throwable { try { // get a evaluator for a string concatenation expression - exprNodeDesc col1desc = new exprNodeColumnDesc(col1Type, "col1", "", false); + exprNodeDesc col1desc = new exprNodeColumnDesc(col1Type, "col1", "", + false); exprNodeDesc col11desc = getListIndexNode(col1desc, 1); - exprNodeDesc func1 = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc(Constants.DOUBLE_TYPE_NAME, col11desc); + exprNodeDesc func1 = TypeCheckProcFactory.DefaultExprProcessor + .getFuncExprNodeDesc(Constants.DOUBLE_TYPE_NAME, col11desc); ExprNodeEvaluator eval = ExprNodeEvaluatorFactory.get(func1); // evaluate on row ObjectInspector resultOI = eval.initialize(r.oi); Object resultO = eval.evaluate(r.o); - assertEquals(Double.valueOf("1"), - ObjectInspectorUtils.copyToStandardObject(resultO, resultOI, ObjectInspectorCopyOption.JAVA)); + assertEquals(Double.valueOf("1"), ObjectInspectorUtils + .copyToStandardObject(resultO, resultOI, + ObjectInspectorCopyOption.JAVA)); System.out.println("testExprNodeConversionEvaluator ok"); } catch (Throwable e) { e.printStackTrace(); @@ -170,119 +174,118 @@ } } - private static void measureSpeed(String expr, int times, ExprNodeEvaluator eval, InspectableObject input, Object standardJavaOutput) throws HiveException { + private static void measureSpeed(String expr, int times, + ExprNodeEvaluator eval, InspectableObject input, Object standardJavaOutput) + throws HiveException { System.out.println("Evaluating " + expr + " for " + times + " times"); - // evaluate on row - InspectableObject output = new InspectableObject(); + new InspectableObject(); ObjectInspector resultOI = eval.initialize(input.oi); Object resultO = null; long start = System.currentTimeMillis(); - for (int i=0; i wrapper = - new HashMapWrapper(0); + HashMapWrapper wrapper = new HashMapWrapper( + 0); insertAll(wrapper, mem_map); checkAll(wrapper, mem_map); - wrapper.close(); // clean up temporary files - + wrapper.close(); // clean up temporary files + // cache size = 1 wrapper = new HashMapWrapper(1); insertAll(wrapper, mem_map); checkAll(wrapper, mem_map); - wrapper.close(); // clean up temporary files - + wrapper.close(); // clean up temporary files + // cache size = 2 wrapper = new HashMapWrapper(2); insertAll(wrapper, mem_map); checkAll(wrapper, mem_map); - wrapper.close(); // clean up temporary files - + wrapper.close(); // clean up temporary files + // cache size = 4 wrapper = new HashMapWrapper(4); insertAll(wrapper, mem_map); checkAll(wrapper, mem_map); - wrapper.close(); // clean up temporary files - + wrapper.close(); // clean up temporary files + // default cache size (25000) wrapper = new HashMapWrapper(); insertAll(wrapper, mem_map); checkAll(wrapper, mem_map); - wrapper.close(); // clean up temporary files - + wrapper.close(); // clean up temporary files + // check mixed put/remove/get functions wrapper = new HashMapWrapper(2); insertAll(wrapper, mem_map); @@ -74,35 +75,35 @@ mem_map.remove("k3"); assertTrue(mem_map.size() == 3); checkAll(wrapper, mem_map); - + wrapper.remove("k1"); mem_map.remove("k1"); checkAll(wrapper, mem_map); - + String v4 = wrapper.get("k4"); assertTrue(v4 != null); - assert(v4.equals("v4")); - + assert (v4.equals("v4")); + wrapper.remove("k4"); mem_map.remove("k4"); checkAll(wrapper, mem_map); - - wrapper.put("k5", "v5"); - mem_map.put("k5", "v5"); + + wrapper.put("k5", "v5"); + mem_map.put("k5", "v5"); checkAll(wrapper, mem_map); - - wrapper.put("k6", "v6"); - mem_map.put("k6", "v6"); + + wrapper.put("k6", "v6"); + mem_map.put("k6", "v6"); checkAll(wrapper, mem_map); - - wrapper.put("k6", "v61"); - mem_map.put("k6", "v61"); + + wrapper.put("k6", "v61"); + mem_map.put("k6", "v61"); checkAll(wrapper, mem_map); - + wrapper.remove("k6"); mem_map.remove("k6"); checkAll(wrapper, mem_map); - + // get k1, k2 to main memory wrapper.get("k1"); wrapper.get("k2"); @@ -113,16 +114,16 @@ wrapper.put("k6", "v7"); mem_map.put("k6", "v7"); checkAll(wrapper, mem_map); - + // test clear wrapper.clear(); mem_map.clear(); checkAll(wrapper, mem_map); - wrapper.close(); // clean up temporary files - + wrapper.close(); // clean up temporary files + // insert 3,000 pairs random testing wrapper = new HashMapWrapper(1000); - for ( int i = 0; i < 3000; ++i ) { + for (int i = 0; i < 3000; ++i) { String k = "k" + i; String v = "v" + i; wrapper.put(k, v); @@ -130,14 +131,14 @@ } checkAll(wrapper, mem_map); System.out.println("Finished inserting 3000 pairs."); - + // do 10,000 random get/remove operations Random rand = new Random(12345678); - for ( int i = 0; i < 10000; ++i ) { + for (int i = 0; i < 10000; ++i) { int j = rand.nextInt(3000); String k = "k" + j; String v; - + int command = rand.nextInt(3); switch (command) { case 0: // remove @@ -147,14 +148,16 @@ break; case 1: // get // System.out.println("getting " + k);// uncomment this for debugging - v = wrapper.get(k); + v = wrapper.get(k); String v2 = mem_map.get(k); - assertTrue("one of them doesn't exists or different values from two hash tables", - v == null && v2 == null || v.equals(v2)); + assertTrue( + "one of them doesn't exists or different values from two hash tables", + v == null && v2 == null || v.equals(v2)); break; case 2: // put v = "v" + rand.nextInt(3000); - // System.out.println("putting (" + k + ", " + v);// uncomment this for debugging + // System.out.println("putting (" + k + ", " + v);// uncomment this + // for debugging wrapper.put(k, v); mem_map.put(k, v); break; @@ -162,7 +165,7 @@ // checkAll(wrapper, mem_map); // uncomment this for debugging } checkAll(wrapper, mem_map); - wrapper.close(); // clean up temporary files + wrapper.close(); // clean up temporary files } catch (Exception e) { e.printStackTrace(); System.out.println(e.toString()); @@ -170,39 +173,39 @@ } System.out.println("TestHashMapWrapper successful"); } - - private void insertAll(HashMapWrapper hashTable, - HashMap map) - throws HiveException { - - for (String k: map.keySet()) { + + private void insertAll(HashMapWrapper hashTable, + HashMap map) throws HiveException { + + for (String k : map.keySet()) { String v = map.get(k); hashTable.put(k, v); } } - - private void checkAll(HashMapWrapper hashTable, - HashMap map) - throws HiveException { - + + private void checkAll(HashMapWrapper hashTable, + HashMap map) throws HiveException { + // check each item in the HashMapWrapper was actually inserted - for ( String k: hashTable.keySet() ) { + for (String k : hashTable.keySet()) { String map_val = hashTable.get(k); String val = map.get(k); - assertTrue("some HashMapWrapper value is not in main memory HashMap: map_val = " + map_val + "; val = " + val, - map_val != null && val != null); - assertTrue("value in HashMapWrapper is not the same as MM HashMap: map_val = " + map_val + "; val = " + val, - val.equals(map_val)); + assertTrue( + "some HashMapWrapper value is not in main memory HashMap: map_val = " + + map_val + "; val = " + val, map_val != null && val != null); + assertTrue( + "value in HashMapWrapper is not the same as MM HashMap: map_val = " + + map_val + "; val = " + val, val.equals(map_val)); } - + // check all inserted elements are in HashMapWrapper - for ( String k: map.keySet() ) { + for (String k : map.keySet()) { String map_val = hashTable.get(k); String val = map.get(k); - assertTrue("Some MM HashMap key is not in HashMapWrapper: map_val = " + map_val + "; val = " + val, - map_val != null && val != null); - assertTrue("Value in MM HashMap is not in HashMapWrapper: map_val = " + map_val + "; val = " + val, - val.equals(map_val)); + assertTrue("Some MM HashMap key is not in HashMapWrapper: map_val = " + + map_val + "; val = " + val, map_val != null && val != null); + assertTrue("Value in MM HashMap is not in HashMapWrapper: map_val = " + + map_val + "; val = " + val, val.equals(map_val)); } } } Index: ql/src/test/org/apache/hadoop/hive/ql/exec/TestPlan.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/exec/TestPlan.java (revision 901519) +++ ql/src/test/org/apache/hadoop/hive/ql/exec/TestPlan.java (working copy) @@ -18,22 +18,23 @@ package org.apache.hadoop.hive.ql.exec; +import java.io.ByteArrayOutputStream; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.LinkedHashMap; + import junit.framework.TestCase; -import java.io.*; -import java.util.*; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.TextInputFormat; - -import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory; -import org.apache.hadoop.hive.ql.plan.*; +import org.apache.hadoop.hive.ql.plan.exprNodeColumnDesc; +import org.apache.hadoop.hive.ql.plan.exprNodeDesc; +import org.apache.hadoop.hive.ql.plan.filterDesc; +import org.apache.hadoop.hive.ql.plan.mapredWork; +import org.apache.hadoop.hive.ql.plan.partitionDesc; +import org.apache.hadoop.hive.ql.plan.tableDesc; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; -import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat; -import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe; +import org.apache.hadoop.mapred.JobConf; - public class TestPlan extends TestCase { public void testPlan() throws Exception { @@ -43,25 +44,28 @@ try { // initialize a complete map reduce configuration - exprNodeDesc expr1 = new exprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, F1, "", false); - exprNodeDesc expr2 = new exprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, F2, "", false); - exprNodeDesc filterExpr = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("==", expr1, expr2); + exprNodeDesc expr1 = new exprNodeColumnDesc( + TypeInfoFactory.stringTypeInfo, F1, "", false); + exprNodeDesc expr2 = new exprNodeColumnDesc( + TypeInfoFactory.stringTypeInfo, F2, "", false); + exprNodeDesc filterExpr = TypeCheckProcFactory.DefaultExprProcessor + .getFuncExprNodeDesc("==", expr1, expr2); filterDesc filterCtx = new filterDesc(filterExpr, false); Operator op = OperatorFactory.get(filterDesc.class); op.setConf(filterCtx); - ArrayList aliasList = new ArrayList (); + ArrayList aliasList = new ArrayList(); aliasList.add("a"); - LinkedHashMap> pa = new LinkedHashMap> (); + LinkedHashMap> pa = new LinkedHashMap>(); pa.put("/tmp/testfolder", aliasList); tableDesc tblDesc = Utilities.defaultTd; partitionDesc partDesc = new partitionDesc(tblDesc, null); - LinkedHashMap pt = new LinkedHashMap (); + LinkedHashMap pt = new LinkedHashMap(); pt.put("/tmp/testfolder", partDesc); - LinkedHashMap> ao = new LinkedHashMap> (); + LinkedHashMap> ao = new LinkedHashMap>(); ao.put("a", op); mapredWork mrwork = new mapredWork(); @@ -70,7 +74,7 @@ mrwork.setAliasToWork(ao); // serialize the configuration once .. - ByteArrayOutputStream baos = new ByteArrayOutputStream (); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); Utilities.serializeMapRedWork(mrwork, baos); baos.close(); String v1 = baos.toString(); @@ -82,8 +86,9 @@ mapredWork mrwork2 = Utilities.getMapRedWork(job); Utilities.clearMapRedWork(job); - // over here we should have some checks of the deserialized object against the orginal object - //System.out.println(v1); + // over here we should have some checks of the deserialized object against + // the orginal object + // System.out.println(v1); // serialize again baos.reset(); Index: ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java (revision 901519) +++ ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java (working copy) @@ -46,35 +46,40 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.JobConf; public class TestOperators extends TestCase { // this is our row to test expressions on - protected InspectableObject [] r; + protected InspectableObject[] r; + @Override protected void setUp() { - r = new InspectableObject [5]; + r = new InspectableObject[5]; ArrayList names = new ArrayList(3); names.add("col0"); names.add("col1"); names.add("col2"); - ArrayList objectInspectors = new ArrayList(3); - objectInspectors.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector); - objectInspectors.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector); - objectInspectors.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector); - for(int i=0; i<5; i++) { - ArrayList data = new ArrayList (); - data.add(""+i); - data.add(""+(i+1)); - data.add(""+(i+2)); + ArrayList objectInspectors = new ArrayList( + 3); + objectInspectors + .add(PrimitiveObjectInspectorFactory.javaStringObjectInspector); + objectInspectors + .add(PrimitiveObjectInspectorFactory.javaStringObjectInspector); + objectInspectors + .add(PrimitiveObjectInspectorFactory.javaStringObjectInspector); + for (int i = 0; i < 5; i++) { + ArrayList data = new ArrayList(); + data.add("" + i); + data.add("" + (i + 1)); + data.add("" + (i + 2)); try { r[i] = new InspectableObject(); r[i].o = data; - r[i].oi = ObjectInspectorFactory.getStandardStructObjectInspector(names, objectInspectors); + r[i].oi = ObjectInspectorFactory.getStandardStructObjectInspector( + names, objectInspectors); } catch (Throwable e) { - throw new RuntimeException (e); + throw new RuntimeException(e); } } } @@ -86,10 +91,13 @@ exprNodeDesc col1 = TestExecDriver.getStringColumn("col1"); exprNodeDesc col2 = TestExecDriver.getStringColumn("col2"); exprNodeDesc zero = new exprNodeConstantDesc("0"); - exprNodeDesc func1 = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc(">", col2, col1); - exprNodeDesc func2 = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("==", col0, zero); - exprNodeDesc func3 = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("and", func1, func2); - assert(func3 != null); + exprNodeDesc func1 = TypeCheckProcFactory.DefaultExprProcessor + .getFuncExprNodeDesc(">", col2, col1); + exprNodeDesc func2 = TypeCheckProcFactory.DefaultExprProcessor + .getFuncExprNodeDesc("==", col0, zero); + exprNodeDesc func3 = TypeCheckProcFactory.DefaultExprProcessor + .getFuncExprNodeDesc("and", func1, func2); + assert (func3 != null); filterDesc filterCtx = new filterDesc(func3, false); // Configuration @@ -97,23 +105,26 @@ op.setConf(filterCtx); // runtime initialization - op.initialize(new JobConf(TestOperators.class), new ObjectInspector[]{r[0].oi}); + op.initialize(new JobConf(TestOperators.class), + new ObjectInspector[] { r[0].oi }); - for(InspectableObject oner: r) { + for (InspectableObject oner : r) { op.process(oner.o, 0); } Map, Long> results = op.getStats(); - System.out.println("filtered = " + results.get(FilterOperator.Counter.FILTERED)); - assertEquals(Long.valueOf(4), results.get(FilterOperator.Counter.FILTERED)); - System.out.println("passed = " + results.get(FilterOperator.Counter.PASSED)); + System.out.println("filtered = " + + results.get(FilterOperator.Counter.FILTERED)); + assertEquals(Long.valueOf(4), results + .get(FilterOperator.Counter.FILTERED)); + System.out.println("passed = " + + results.get(FilterOperator.Counter.PASSED)); assertEquals(Long.valueOf(1), results.get(FilterOperator.Counter.PASSED)); /* - for(Enum e: results.keySet()) { - System.out.println(e.toString() + ":" + results.get(e)); - } - */ + * for(Enum e: results.keySet()) { System.out.println(e.toString() + ":" + + * results.get(e)); } + */ System.out.println("Filter Operator ok"); } catch (Throwable e) { @@ -131,28 +142,33 @@ // col2 exprNodeDesc expr1 = TestExecDriver.getStringColumn("col0"); exprNodeDesc expr2 = new exprNodeConstantDesc("1"); - exprNodeDesc exprDesc2 = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("concat", expr1, expr2); + exprNodeDesc exprDesc2 = TypeCheckProcFactory.DefaultExprProcessor + .getFuncExprNodeDesc("concat", expr1, expr2); // select operator to project these two columns - ArrayList earr = new ArrayList (); + ArrayList earr = new ArrayList(); earr.add(exprDesc1); earr.add(exprDesc2); ArrayList outputCols = new ArrayList(); - for (int i = 0; i < earr.size(); i++) - outputCols.add("_col"+i); + for (int i = 0; i < earr.size(); i++) { + outputCols.add("_col" + i); + } selectDesc selectCtx = new selectDesc(earr, outputCols); Operator op = OperatorFactory.get(selectDesc.class); op.setConf(selectCtx); // fileSinkOperator to dump the output of the select - //fileSinkDesc fsd = new fileSinkDesc ("file:///tmp" + File.separator + System.getProperty("user.name") + File.separator + "TestFileSinkOperator", - // Utilities.defaultTd, false); - //Operator flop = OperatorFactory.getAndMakeChild(fsd, op); + // fileSinkDesc fsd = new fileSinkDesc ("file:///tmp" + File.separator + + // System.getProperty("user.name") + File.separator + + // "TestFileSinkOperator", + // Utilities.defaultTd, false); + // Operator flop = OperatorFactory.getAndMakeChild(fsd, op); - op.initialize(new JobConf(TestOperators.class), new ObjectInspector[]{r[0].oi}); + op.initialize(new JobConf(TestOperators.class), + new ObjectInspector[] { r[0].oi }); // evaluate on row - for(int i=0; i<5; i++) { + for (int i = 0; i < 5; i++) { op.process(r[i].o, 0); } op.close(false); @@ -165,7 +181,6 @@ } } - public void testScriptOperator() throws Throwable { try { System.out.println("Testing Script Operator"); @@ -175,50 +190,59 @@ // col2 exprNodeDesc expr1 = TestExecDriver.getStringColumn("col0"); exprNodeDesc expr2 = new exprNodeConstantDesc("1"); - exprNodeDesc exprDesc2 = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("concat", expr1, expr2); + exprNodeDesc exprDesc2 = TypeCheckProcFactory.DefaultExprProcessor + .getFuncExprNodeDesc("concat", expr1, expr2); // select operator to project these two columns - ArrayList earr = new ArrayList (); + ArrayList earr = new ArrayList(); earr.add(exprDesc1); earr.add(exprDesc2); ArrayList outputCols = new ArrayList(); - for (int i = 0; i < earr.size(); i++) - outputCols.add("_col"+i); + for (int i = 0; i < earr.size(); i++) { + outputCols.add("_col" + i); + } selectDesc selectCtx = new selectDesc(earr, outputCols); Operator op = OperatorFactory.get(selectDesc.class); op.setConf(selectCtx); // scriptOperator to echo the output of the select - tableDesc scriptOutput = PlanUtils.getDefaultTableDesc("" + Utilities.tabCode, "a,b"); - tableDesc scriptInput = PlanUtils.getDefaultTableDesc("" + Utilities.tabCode, "a,b"); - scriptDesc sd = new scriptDesc("cat", scriptOutput, TextRecordWriter.class, scriptInput, TextRecordReader.class); + tableDesc scriptOutput = PlanUtils.getDefaultTableDesc("" + + Utilities.tabCode, "a,b"); + tableDesc scriptInput = PlanUtils.getDefaultTableDesc("" + + Utilities.tabCode, "a,b"); + scriptDesc sd = new scriptDesc("cat", scriptOutput, + TextRecordWriter.class, scriptInput, TextRecordReader.class); Operator sop = OperatorFactory.getAndMakeChild(sd, op); // Collect operator to observe the output of the script - collectDesc cd = new collectDesc (Integer.valueOf(10)); - CollectOperator cdop = (CollectOperator) OperatorFactory.getAndMakeChild(cd, sop); + collectDesc cd = new collectDesc(Integer.valueOf(10)); + CollectOperator cdop = (CollectOperator) OperatorFactory.getAndMakeChild( + cd, sop); - op.initialize(new JobConf(TestOperators.class), new ObjectInspector[]{r[0].oi}); + op.initialize(new JobConf(TestOperators.class), + new ObjectInspector[] { r[0].oi }); // evaluate on row - for(int i=0; i<5; i++) { + for (int i = 0; i < 5; i++) { op.process(r[i].o, 0); } op.close(false); InspectableObject io = new InspectableObject(); - for(int i=0; i<5; i++) { + for (int i = 0; i < 5; i++) { cdop.retrieve(io); System.out.println("[" + i + "] io.o=" + io.o); System.out.println("[" + i + "] io.oi=" + io.oi); - StructObjectInspector soi = (StructObjectInspector)io.oi; - assert(soi != null); + StructObjectInspector soi = (StructObjectInspector) io.oi; + assert (soi != null); StructField a = soi.getStructFieldRef("a"); StructField b = soi.getStructFieldRef("b"); - assertEquals(""+(i+1), ((PrimitiveObjectInspector)a.getFieldObjectInspector()) - .getPrimitiveJavaObject(soi.getStructFieldData(io.o, a))); - assertEquals((i) + "1", ((PrimitiveObjectInspector)b.getFieldObjectInspector()) - .getPrimitiveJavaObject(soi.getStructFieldData(io.o, b))); + assertEquals("" + (i + 1), ((PrimitiveObjectInspector) a + .getFieldObjectInspector()).getPrimitiveJavaObject(soi + .getStructFieldData(io.o, a))); + assertEquals((i) + "1", ((PrimitiveObjectInspector) b + .getFieldObjectInspector()).getPrimitiveJavaObject(soi + .getStructFieldData(io.o, b))); } System.out.println("Script Operator ok"); @@ -234,35 +258,37 @@ System.out.println("Testing Map Operator"); // initialize configuration Configuration hconf = new JobConf(TestOperators.class); - HiveConf.setVar(hconf, HiveConf.ConfVars.HADOOPMAPFILENAME, "hdfs:///testDir/testFile"); + HiveConf.setVar(hconf, HiveConf.ConfVars.HADOOPMAPFILENAME, + "hdfs:///testDir/testFile"); // initialize pathToAliases - ArrayList aliases = new ArrayList (); + ArrayList aliases = new ArrayList(); aliases.add("a"); aliases.add("b"); - LinkedHashMap> pathToAliases = new LinkedHashMap> (); + LinkedHashMap> pathToAliases = new LinkedHashMap>(); pathToAliases.put("/testDir", aliases); // initialize pathToTableInfo // Default: treat the table as a single column "col" tableDesc td = Utilities.defaultTd; partitionDesc pd = new partitionDesc(td, null); - LinkedHashMap pathToPartitionInfo = new - LinkedHashMap (); + LinkedHashMap pathToPartitionInfo = new LinkedHashMap(); pathToPartitionInfo.put("/testDir", pd); // initialize aliasToWork - collectDesc cd = new collectDesc (Integer.valueOf(1)); - CollectOperator cdop1 = (CollectOperator) OperatorFactory.get(collectDesc.class); + collectDesc cd = new collectDesc(Integer.valueOf(1)); + CollectOperator cdop1 = (CollectOperator) OperatorFactory + .get(collectDesc.class); cdop1.setConf(cd); - CollectOperator cdop2 = (CollectOperator) OperatorFactory.get(collectDesc.class); + CollectOperator cdop2 = (CollectOperator) OperatorFactory + .get(collectDesc.class); cdop2.setConf(cd); - LinkedHashMap> aliasToWork = new LinkedHashMap> (); + LinkedHashMap> aliasToWork = new LinkedHashMap>(); aliasToWork.put("a", cdop1); aliasToWork.put("b", cdop2); // initialize mapredWork - mapredWork mrwork = new mapredWork (); + mapredWork mrwork = new mapredWork(); mrwork.setPathToAliases(pathToAliases); mrwork.setPathToPartitionInfo(pathToPartitionInfo); mrwork.setAliasToWork(aliasToWork); @@ -274,11 +300,11 @@ Text tw = new Text(); InspectableObject io1 = new InspectableObject(); InspectableObject io2 = new InspectableObject(); - for(int i=0; i<5; i++) { - String answer = "[[" + i + ", " + (i+1) + ", " + (i+2) + "]]"; + for (int i = 0; i < 5; i++) { + String answer = "[[" + i + ", " + (i + 1) + ", " + (i + 2) + "]]"; - tw.set("" + i + "\u0001" + (i+1) + "\u0001"+ (i+2)); - mo.process((Writable)tw); + tw.set("" + i + "\u0001" + (i + 1) + "\u0001" + (i + 2)); + mo.process(tw); cdop1.retrieve(io1); cdop2.retrieve(io2); System.out.println("io1.o.toString() = " + io1.o.toString()); Index: ql/src/test/org/apache/hadoop/hive/ql/io/TestFlatFileInputFormat.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/io/TestFlatFileInputFormat.java (revision 901519) +++ ql/src/test/org/apache/hadoop/hive/ql/io/TestFlatFileInputFormat.java (working copy) @@ -18,33 +18,33 @@ package org.apache.hadoop.hive.ql.io; -import java.io.*; -import java.util.*; +import java.io.Serializable; + import junit.framework.TestCase; -import org.apache.commons.logging.*; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.serializer.JavaSerialization; +import org.apache.hadoop.io.serializer.Serializer; +import org.apache.hadoop.io.serializer.WritableSerialization; +import org.apache.hadoop.mapred.FileInputFormat; +import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RecordReader; +import org.apache.hadoop.mapred.Reporter; -import org.apache.hadoop.fs.*; -import org.apache.hadoop.record.*; -import org.apache.hadoop.io.*; -import org.apache.hadoop.mapred.*; -import org.apache.hadoop.io.serializer.*; -import org.apache.hadoop.conf.*; -import org.apache.hadoop.util.ReflectionUtils; - -import org.apache.thrift.*; -import org.apache.thrift.transport.*; -import org.apache.thrift.protocol.*; - //import org.apache.hadoop.contrib.serialization.thrift.*; -public class TestFlatFileInputFormat extends TestCase { +public class TestFlatFileInputFormat extends TestCase { public void testFlatFileInputJava() throws Exception { Configuration conf; - JobConf job ; + JobConf job; FileSystem fs; - Path dir ; + Path dir; Path file; Reporter reporter; FSDataOutputStream ds; @@ -56,18 +56,20 @@ conf = new Configuration(); job = new JobConf(conf); fs = FileSystem.getLocal(conf); - dir = new Path(System.getProperty("test.data.dir",".") + "/mapred"); + dir = new Path(System.getProperty("test.data.dir", ".") + "/mapred"); file = new Path(dir, "test.txt"); reporter = Reporter.NULL; fs.delete(dir, true); job.setClass(FlatFileInputFormat.SerializationImplKey, - org.apache.hadoop.io.serializer.JavaSerialization.class, - org.apache.hadoop.io.serializer.Serialization.class); - - job.setClass(FlatFileInputFormat.SerializationContextFromConf.SerializationSubclassKey, - JavaTestObjFlatFileInputFormat.class, java.io.Serializable.class); - + org.apache.hadoop.io.serializer.JavaSerialization.class, + org.apache.hadoop.io.serializer.Serialization.class); + + job + .setClass( + FlatFileInputFormat.SerializationContextFromConf.SerializationSubclassKey, + JavaTestObjFlatFileInputFormat.class, java.io.Serializable.class); + // // Write some data out to a flat file // @@ -78,38 +80,40 @@ // construct some data and write it serializer.open(ds); for (int i = 0; i < 10; i++) { - serializer.serialize(new JavaTestObjFlatFileInputFormat("Hello World! " + String.valueOf(i), i)); + serializer.serialize(new JavaTestObjFlatFileInputFormat("Hello World! " + + String.valueOf(i), i)); } serializer.close(); // // Construct the reader // - FileInputFormat> format = - new FlatFileInputFormat(); + FileInputFormat> format = new FlatFileInputFormat(); InputSplit[] splits = format.getSplits(job, 1); // construct the record reader - RecordReader> reader = - format.getRecordReader(splits[0], job, reporter); + RecordReader> reader = format + .getRecordReader(splits[0], job, reporter); // create key/value Void key = reader.createKey(); - FlatFileInputFormat.RowContainer value = reader.createValue(); - + FlatFileInputFormat.RowContainer value = reader + .createValue(); + // // read back the data using the FlatFileRecordReader // int count = 0; while (reader.next(key, value)) { assertTrue(key == null); - assertTrue(((JavaTestObjFlatFileInputFormat)value.row).s.equals("Hello World! " +String.valueOf(count))); - assertTrue(((JavaTestObjFlatFileInputFormat)value.row).num == count); + assertTrue(((JavaTestObjFlatFileInputFormat) value.row).s + .equals("Hello World! " + String.valueOf(count))); + assertTrue(((JavaTestObjFlatFileInputFormat) value.row).num == count); count++; } reader.close(); - } catch(Exception e) { + } catch (Exception e) { System.err.println("caught: " + e); e.printStackTrace(); } finally { @@ -119,9 +123,9 @@ public void testFlatFileInputRecord() throws Exception { Configuration conf; - JobConf job ; + JobConf job; FileSystem fs; - Path dir ; + Path dir; Path file; Reporter reporter; FSDataOutputStream ds; @@ -133,149 +137,124 @@ conf = new Configuration(); job = new JobConf(conf); fs = FileSystem.getLocal(conf); - dir = new Path(System.getProperty("test.data.dir",".") + "/mapred"); + dir = new Path(System.getProperty("test.data.dir", ".") + "/mapred"); file = new Path(dir, "test.txt"); reporter = Reporter.NULL; fs.delete(dir, true); job.setClass(FlatFileInputFormat.SerializationImplKey, - org.apache.hadoop.io.serializer.WritableSerialization.class, - org.apache.hadoop.io.serializer.Serialization.class); - - job.setClass(FlatFileInputFormat.SerializationContextFromConf.SerializationSubclassKey, - RecordTestObj.class, Writable.class); - + org.apache.hadoop.io.serializer.WritableSerialization.class, + org.apache.hadoop.io.serializer.Serialization.class); + + job + .setClass( + FlatFileInputFormat.SerializationContextFromConf.SerializationSubclassKey, + RecordTestObj.class, Writable.class); + // // Write some data out to a flat file // FileInputFormat.setInputPaths(job, dir); ds = fs.create(file); - Serializer serializer = new WritableSerialization().getSerializer(Writable.class); + Serializer serializer = new WritableSerialization() + .getSerializer(Writable.class); // construct some data and write it serializer.open(ds); for (int i = 0; i < 10; i++) { - serializer.serialize(new RecordTestObj("Hello World! " + String.valueOf(i), i)); + serializer.serialize(new RecordTestObj("Hello World! " + + String.valueOf(i), i)); } serializer.close(); // // Construct the reader // - FileInputFormat> format = - new FlatFileInputFormat(); + FileInputFormat> format = new FlatFileInputFormat(); InputSplit[] splits = format.getSplits(job, 1); // construct the record reader - RecordReader> reader = - format.getRecordReader(splits[0], job, reporter); + RecordReader> reader = format + .getRecordReader(splits[0], job, reporter); // create key/value Void key = reader.createKey(); FlatFileInputFormat.RowContainer value = reader.createValue(); - - // - // read back the data using the FlatFileRecordReader - // - int count = 0; - while (reader.next(key, value)) { - assertTrue(key == null); - assertTrue(((RecordTestObj)value.row).getS().equals("Hello World! " +String.valueOf(count))); - assertTrue(((RecordTestObj)value.row).getNum() == count); - count++; - } - reader.close(); - } catch(Exception e) { - System.err.println("caught: " + e); - e.printStackTrace(); - } finally { - } - - } - /* - public void testFlatFileInputThrift() throws Exception { - Configuration conf; - JobConf job ; - FileSystem fs; - Path dir ; - Path file; - Reporter reporter; - FSDataOutputStream ds; - - try { // - // create job and filesystem and reporter and such. - // - conf = new Configuration(); - job = new JobConf(conf); - fs = FileSystem.getLocal(conf); - dir = new Path(System.getProperty("test.data.dir",".") + "/mapred"); - file = new Path(dir, "test.txt"); - reporter = Reporter.NULL; - fs.delete(dir, true); - - job.setClass(FlatFileInputFormat.SerializationContextFromConf.SerializationImplKey, - org.apache.hadoop.contrib.serialization.thrift.ThriftSerialization.class, - org.apache.hadoop.io.serializer.Serialization.class); - - job.setClass(FlatFileInputFormat.SerializationContextFromConf.SerializationSubclassKey, - FlatFileThriftTestObj.class, TBase.class); - - // - // Write some data out to a flat file - // - FileInputFormat.setInputPaths(job, dir); - ds = fs.create(file); - Serializer serializer = new ThriftSerialization().getSerializer(TBase.class); - - // construct some data and write it - serializer.open(ds); - for (int i = 0; i < 10; i++) { - serializer.serialize(new FlatFileThriftTestObj("Hello World! " + String.valueOf(i), i)); - } - serializer.close(); - - // - // Construct the reader - // - FileInputFormat> format = - new FlatFileInputFormat(); - InputSplit[] splits = format.getSplits(job, 1); - - // construct the record reader - RecordReader> reader = - format.getRecordReader(splits[0], job, reporter); - - // create key/value - Void key = reader.createKey(); - FlatFileInputFormat.RowContainer value = reader.createValue(); - - // // read back the data using the FlatFileRecordReader // int count = 0; while (reader.next(key, value)) { assertTrue(key == null); - assertTrue(((FlatFileThriftTestObj)value.row).s.equals("Hello World! " +String.valueOf(count))); - assertTrue(((FlatFileThriftTestObj)value.row).num == count); + assertTrue(((RecordTestObj) value.row).getS().equals( + "Hello World! " + String.valueOf(count))); + assertTrue(((RecordTestObj) value.row).getNum() == count); count++; } reader.close(); - } catch(Exception e) { + } catch (Exception e) { System.err.println("caught: " + e); e.printStackTrace(); } finally { } } - */ + /* + * public void testFlatFileInputThrift() throws Exception { Configuration + * conf; JobConf job ; FileSystem fs; Path dir ; Path file; Reporter reporter; + * FSDataOutputStream ds; + * + * try { // // create job and filesystem and reporter and such. // conf = new + * Configuration(); job = new JobConf(conf); fs = FileSystem.getLocal(conf); + * dir = new Path(System.getProperty("test.data.dir",".") + "/mapred"); file = + * new Path(dir, "test.txt"); reporter = Reporter.NULL; fs.delete(dir, true); + * + * job.setClass(FlatFileInputFormat.SerializationContextFromConf. + * SerializationImplKey, + * org.apache.hadoop.contrib.serialization.thrift.ThriftSerialization.class, + * org.apache.hadoop.io.serializer.Serialization.class); + * + * job.setClass(FlatFileInputFormat.SerializationContextFromConf. + * SerializationSubclassKey, FlatFileThriftTestObj.class, TBase.class); + * + * // // Write some data out to a flat file // + * FileInputFormat.setInputPaths(job, dir); ds = fs.create(file); Serializer + * serializer = new ThriftSerialization().getSerializer(TBase.class); + * + * // construct some data and write it serializer.open(ds); for (int i = 0; i + * < 10; i++) { serializer.serialize(new FlatFileThriftTestObj("Hello World! " + * + String.valueOf(i), i)); } serializer.close(); + * + * // // Construct the reader // FileInputFormat> format = new + * FlatFileInputFormat(); InputSplit[] splits = format.getSplits(job, + * 1); + * + * // construct the record reader RecordReader> reader = + * format.getRecordReader(splits[0], job, reporter); + * + * // create key/value Void key = reader.createKey(); + * FlatFileInputFormat.RowContainer value = reader.createValue(); + * + * // // read back the data using the FlatFileRecordReader // int count = 0; + * while (reader.next(key, value)) { assertTrue(key == null); + * assertTrue(((FlatFileThriftTestObj)value.row).s.equals("Hello World! " + * +String.valueOf(count))); assertTrue(((FlatFileThriftTestObj)value.row).num + * == count); count++; } reader.close(); + * + * } catch(Exception e) { System.err.println("caught: " + e); + * e.printStackTrace(); } finally { } + * + * } + */ public static void main(String[] args) throws Exception { new TestFlatFileInputFormat().testFlatFileInputJava(); new TestFlatFileInputFormat().testFlatFileInputRecord(); - // new TestFlatFileInputFormat().testFlatFileInputThrift(); + // new TestFlatFileInputFormat().testFlatFileInputThrift(); } } Index: ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java (revision 901519) +++ ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java (working copy) @@ -31,8 +31,8 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.serde.Constants; +import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; import org.apache.hadoop.hive.serde2.SerDeException; -import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; import org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable; import org.apache.hadoop.hive.serde2.columnar.BytesRefWritable; import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe; @@ -97,7 +97,8 @@ bytesArray = new byte[][] { "123".getBytes("UTF-8"), "456".getBytes("UTF-8"), "789".getBytes("UTF-8"), "1000".getBytes("UTF-8"), "5.3".getBytes("UTF-8"), - "hive and hadoop".getBytes("UTF-8"), new byte[0], "NULL".getBytes("UTF-8") }; + "hive and hadoop".getBytes("UTF-8"), new byte[0], + "NULL".getBytes("UTF-8") }; s = new BytesRefArrayWritable(bytesArray.length); s.set(0, new BytesRefWritable("123".getBytes("UTF-8"))); s.set(1, new BytesRefWritable("456".getBytes("UTF-8"))); @@ -180,12 +181,14 @@ assertEquals("Field size should be 8", 8, fieldRefs.size()); for (int j = 0; j < fieldRefs.size(); j++) { Object fieldData = oi.getStructFieldData(row, fieldRefs.get(j)); - Object standardWritableData = ObjectInspectorUtils.copyToStandardObject(fieldData, - fieldRefs.get(j).getFieldObjectInspector(), ObjectInspectorCopyOption.WRITABLE); - if (i == 0) + Object standardWritableData = ObjectInspectorUtils + .copyToStandardObject(fieldData, fieldRefs.get(j) + .getFieldObjectInspector(), ObjectInspectorCopyOption.WRITABLE); + if (i == 0) { assertEquals("Field " + i, standardWritableData, expectedRecord_1[j]); - else + } else { assertEquals("Field " + i, standardWritableData, expectedRecord_2[j]); + } } } @@ -307,12 +310,15 @@ assertEquals("Field size should be 8", 8, fieldRefs.size()); for (int i = 0; i < fieldRefs.size(); i++) { Object fieldData = oi.getStructFieldData(row, fieldRefs.get(i)); - Object standardWritableData = ObjectInspectorUtils.copyToStandardObject(fieldData, - fieldRefs.get(i).getFieldObjectInspector(), ObjectInspectorCopyOption.WRITABLE); + Object standardWritableData = ObjectInspectorUtils + .copyToStandardObject(fieldData, fieldRefs.get(i) + .getFieldObjectInspector(), ObjectInspectorCopyOption.WRITABLE); assertEquals("Field " + i, standardWritableData, expectedFieldsData[i]); } // Serialize - assertEquals("Class of the serialized object should be BytesRefArrayWritable", BytesRefArrayWritable.class, serDe.getSerializedClass()); + assertEquals( + "Class of the serialized object should be BytesRefArrayWritable", + BytesRefArrayWritable.class, serDe.getSerializedClass()); BytesRefArrayWritable serializedText = (BytesRefArrayWritable) serDe .serialize(row, oi); assertEquals("Serialized data", s, serializedText); @@ -337,7 +343,7 @@ LongWritable rowID = new LongWritable(); BytesRefArrayWritable cols = new BytesRefArrayWritable(); - + while (reader.next(rowID)) { reader.getCurrentRow(cols); cols.resetValid(8); @@ -350,12 +356,16 @@ for (int i : readCols) { Object fieldData = oi.getStructFieldData(row, fieldRefs.get(i)); - Object standardWritableData = ObjectInspectorUtils.copyToStandardObject(fieldData, - fieldRefs.get(i).getFieldObjectInspector(), ObjectInspectorCopyOption.WRITABLE); - assertEquals("Field " + i, standardWritableData, expectedPartitalFieldsData[i]); + Object standardWritableData = ObjectInspectorUtils + .copyToStandardObject(fieldData, fieldRefs.get(i) + .getFieldObjectInspector(), ObjectInspectorCopyOption.WRITABLE); + assertEquals("Field " + i, standardWritableData, + expectedPartitalFieldsData[i]); } - assertEquals("Class of the serialized object should be BytesRefArrayWritable", BytesRefArrayWritable.class, serDe.getSerializedClass()); + assertEquals( + "Class of the serialized object should be BytesRefArrayWritable", + BytesRefArrayWritable.class, serDe.getSerializedClass()); BytesRefArrayWritable serializedBytes = (BytesRefArrayWritable) serDe .serialize(row, oi); assertEquals("Serialized data", patialS, serializedBytes); Index: ql/src/test/org/apache/hadoop/hive/ql/io/RecordTestObj.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/io/RecordTestObj.java (revision 901519) +++ ql/src/test/org/apache/hadoop/hive/ql/io/RecordTestObj.java (working copy) @@ -6,44 +6,60 @@ private static org.apache.hadoop.record.meta.RecordTypeInfo _rio_rtiFilter; private static int[] _rio_rtiFilterFields; static { - _rio_recTypeInfo = new org.apache.hadoop.record.meta.RecordTypeInfo("RecordTestObj"); - _rio_recTypeInfo.addField("s", org.apache.hadoop.record.meta.TypeID.StringTypeID); - _rio_recTypeInfo.addField("num", org.apache.hadoop.record.meta.TypeID.LongTypeID); + _rio_recTypeInfo = new org.apache.hadoop.record.meta.RecordTypeInfo( + "RecordTestObj"); + _rio_recTypeInfo.addField("s", + org.apache.hadoop.record.meta.TypeID.StringTypeID); + _rio_recTypeInfo.addField("num", + org.apache.hadoop.record.meta.TypeID.LongTypeID); } - + private String s; private long num; - public RecordTestObj() { } - public RecordTestObj( - final String s, - final long num) { + + public RecordTestObj() { + } + + public RecordTestObj(final String s, final long num) { this.s = s; this.num = num; } + public static org.apache.hadoop.record.meta.RecordTypeInfo getTypeInfo() { return _rio_recTypeInfo; } - public static void setTypeFilter(org.apache.hadoop.record.meta.RecordTypeInfo rti) { - if (null == rti) return; + + public static void setTypeFilter( + org.apache.hadoop.record.meta.RecordTypeInfo rti) { + if (null == rti) { + return; + } _rio_rtiFilter = rti; _rio_rtiFilterFields = null; } - private static void setupRtiFields() - { - if (null == _rio_rtiFilter) return; + + private static void setupRtiFields() { + if (null == _rio_rtiFilter) { + return; + } // we may already have done this - if (null != _rio_rtiFilterFields) return; + if (null != _rio_rtiFilterFields) { + return; + } int _rio_i, _rio_j; - _rio_rtiFilterFields = new int [_rio_rtiFilter.getFieldTypeInfos().size()]; - for (_rio_i=0; _rio_i<_rio_rtiFilterFields.length; _rio_i++) { + _rio_rtiFilterFields = new int[_rio_rtiFilter.getFieldTypeInfos().size()]; + for (_rio_i = 0; _rio_i < _rio_rtiFilterFields.length; _rio_i++) { _rio_rtiFilterFields[_rio_i] = 0; } - java.util.Iterator _rio_itFilter = _rio_rtiFilter.getFieldTypeInfos().iterator(); - _rio_i=0; + java.util.Iterator _rio_itFilter = _rio_rtiFilter + .getFieldTypeInfos().iterator(); + _rio_i = 0; while (_rio_itFilter.hasNext()) { - org.apache.hadoop.record.meta.FieldTypeInfo _rio_tInfoFilter = _rio_itFilter.next(); - java.util.Iterator _rio_it = _rio_recTypeInfo.getFieldTypeInfos().iterator(); - _rio_j=1; + org.apache.hadoop.record.meta.FieldTypeInfo _rio_tInfoFilter = _rio_itFilter + .next(); + java.util.Iterator _rio_it = _rio_recTypeInfo + .getFieldTypeInfos().iterator(); + _rio_j = 1; while (_rio_it.hasNext()) { org.apache.hadoop.record.meta.FieldTypeInfo _rio_tInfo = _rio_it.next(); if (_rio_tInfo.equals(_rio_tInfoFilter)) { @@ -55,34 +71,44 @@ _rio_i++; } } + public String getS() { return s; } + public void setS(final String s) { - this.s=s; + this.s = s; } + public long getNum() { return num; } + public void setNum(final long num) { - this.num=num; + this.num = num; } - public void serialize(final org.apache.hadoop.record.RecordOutput _rio_a, final String _rio_tag) - throws java.io.IOException { - _rio_a.startRecord(this,_rio_tag); - _rio_a.writeString(s,"s"); - _rio_a.writeLong(num,"num"); - _rio_a.endRecord(this,_rio_tag); + + @Override + public void serialize(final org.apache.hadoop.record.RecordOutput _rio_a, + final String _rio_tag) throws java.io.IOException { + _rio_a.startRecord(this, _rio_tag); + _rio_a.writeString(s, "s"); + _rio_a.writeLong(num, "num"); + _rio_a.endRecord(this, _rio_tag); } - private void deserializeWithoutFilter(final org.apache.hadoop.record.RecordInput _rio_a, final String _rio_tag) - throws java.io.IOException { + + private void deserializeWithoutFilter( + final org.apache.hadoop.record.RecordInput _rio_a, final String _rio_tag) + throws java.io.IOException { _rio_a.startRecord(_rio_tag); - s=_rio_a.readString("s"); - num=_rio_a.readLong("num"); + s = _rio_a.readString("s"); + num = _rio_a.readLong("num"); _rio_a.endRecord(_rio_tag); } - public void deserialize(final org.apache.hadoop.record.RecordInput _rio_a, final String _rio_tag) - throws java.io.IOException { + + @Override + public void deserialize(final org.apache.hadoop.record.RecordInput _rio_a, + final String _rio_tag) throws java.io.IOException { if (null == _rio_rtiFilter) { deserializeWithoutFilter(_rio_a, _rio_tag); return; @@ -90,32 +116,40 @@ // if we're here, we need to read based on version info _rio_a.startRecord(_rio_tag); setupRtiFields(); - for (int _rio_i=0; _rio_i<_rio_rtiFilter.getFieldTypeInfos().size(); _rio_i++) { + for (int _rio_i = 0; _rio_i < _rio_rtiFilter.getFieldTypeInfos().size(); _rio_i++) { if (1 == _rio_rtiFilterFields[_rio_i]) { - s=_rio_a.readString("s"); + s = _rio_a.readString("s"); + } else if (2 == _rio_rtiFilterFields[_rio_i]) { + num = _rio_a.readLong("num"); + } else { + java.util.ArrayList typeInfos = (java.util.ArrayList) (_rio_rtiFilter + .getFieldTypeInfos()); + org.apache.hadoop.record.meta.Utils.skip(_rio_a, typeInfos.get(_rio_i) + .getFieldID(), typeInfos.get(_rio_i).getTypeID()); } - else if (2 == _rio_rtiFilterFields[_rio_i]) { - num=_rio_a.readLong("num"); - } - else { - java.util.ArrayList typeInfos = (java.util.ArrayList)(_rio_rtiFilter.getFieldTypeInfos()); - org.apache.hadoop.record.meta.Utils.skip(_rio_a, typeInfos.get(_rio_i).getFieldID(), typeInfos.get(_rio_i).getTypeID()); - } } _rio_a.endRecord(_rio_tag); } - public int compareTo (final Object _rio_peer_) throws ClassCastException { + + @Override + public int compareTo(final Object _rio_peer_) throws ClassCastException { if (!(_rio_peer_ instanceof RecordTestObj)) { throw new ClassCastException("Comparing different types of records."); } RecordTestObj _rio_peer = (RecordTestObj) _rio_peer_; int _rio_ret = 0; _rio_ret = s.compareTo(_rio_peer.s); - if (_rio_ret != 0) return _rio_ret; - _rio_ret = (num == _rio_peer.num)? 0 :((num<_rio_peer.num)?-1:1); - if (_rio_ret != 0) return _rio_ret; + if (_rio_ret != 0) { + return _rio_ret; + } + _rio_ret = (num == _rio_peer.num) ? 0 : ((num < _rio_peer.num) ? -1 : 1); + if (_rio_ret != 0) { + return _rio_ret; + } return _rio_ret; } + + @Override public boolean equals(final Object _rio_peer_) { if (!(_rio_peer_ instanceof RecordTestObj)) { return false; @@ -126,53 +160,68 @@ RecordTestObj _rio_peer = (RecordTestObj) _rio_peer_; boolean _rio_ret = false; _rio_ret = s.equals(_rio_peer.s); - if (!_rio_ret) return _rio_ret; - _rio_ret = (num==_rio_peer.num); - if (!_rio_ret) return _rio_ret; + if (!_rio_ret) { + return _rio_ret; + } + _rio_ret = (num == _rio_peer.num); + if (!_rio_ret) { + return _rio_ret; + } return _rio_ret; } + + @Override public Object clone() throws CloneNotSupportedException { RecordTestObj _rio_other = new RecordTestObj(); - _rio_other.s = this.s; - _rio_other.num = this.num; + _rio_other.s = s; + _rio_other.num = num; return _rio_other; } + + @Override public int hashCode() { int _rio_result = 17; int _rio_ret; _rio_ret = s.hashCode(); - _rio_result = 37*_rio_result + _rio_ret; - _rio_ret = (int) (num^(num>>>32)); - _rio_result = 37*_rio_result + _rio_ret; + _rio_result = 37 * _rio_result + _rio_ret; + _rio_ret = (int) (num ^ (num >>> 32)); + _rio_result = 37 * _rio_result + _rio_ret; return _rio_result; } + public static String signature() { return "LRecordTestObj(sl)"; } - public static class Comparator extends org.apache.hadoop.record.RecordComparator { + + public static class Comparator extends + org.apache.hadoop.record.RecordComparator { public Comparator() { super(RecordTestObj.class); } + static public int slurpRaw(byte[] b, int s, int l) { try { int os = s; { int i = org.apache.hadoop.record.Utils.readVInt(b, s); int z = org.apache.hadoop.record.Utils.getVIntSize(i); - s+=(z+i); l-= (z+i); + s += (z + i); + l -= (z + i); } { long i = org.apache.hadoop.record.Utils.readVLong(b, s); int z = org.apache.hadoop.record.Utils.getVIntSize(i); - s+=z; l-=z; + s += z; + l -= z; } return (os - s); - } catch(java.io.IOException e) { + } catch (java.io.IOException e) { throw new RuntimeException(e); } } - static public int compareRaw(byte[] b1, int s1, int l1, - byte[] b2, int s2, int l2) { + + static public int compareRaw(byte[] b1, int s1, int l1, byte[] b2, int s2, + int l2) { try { int os1 = s1; { @@ -180,33 +229,48 @@ int i2 = org.apache.hadoop.record.Utils.readVInt(b2, s2); int z1 = org.apache.hadoop.record.Utils.getVIntSize(i1); int z2 = org.apache.hadoop.record.Utils.getVIntSize(i2); - s1+=z1; s2+=z2; l1-=z1; l2-=z2; - int r1 = org.apache.hadoop.record.Utils.compareBytes(b1,s1,i1,b2,s2,i2); - if (r1 != 0) { return (r1<0)?-1:0; } - s1+=i1; s2+=i2; l1-=i1; l1-=i2; + s1 += z1; + s2 += z2; + l1 -= z1; + l2 -= z2; + int r1 = org.apache.hadoop.record.Utils.compareBytes(b1, s1, i1, b2, + s2, i2); + if (r1 != 0) { + return (r1 < 0) ? -1 : 0; + } + s1 += i1; + s2 += i2; + l1 -= i1; + l1 -= i2; } { long i1 = org.apache.hadoop.record.Utils.readVLong(b1, s1); long i2 = org.apache.hadoop.record.Utils.readVLong(b2, s2); if (i1 != i2) { - return ((i1-i2) < 0) ? -1 : 0; + return ((i1 - i2) < 0) ? -1 : 0; } int z1 = org.apache.hadoop.record.Utils.getVIntSize(i1); int z2 = org.apache.hadoop.record.Utils.getVIntSize(i2); - s1+=z1; s2+=z2; l1-=z1; l2-=z2; + s1 += z1; + s2 += z2; + l1 -= z1; + l2 -= z2; } return (os1 - s1); - } catch(java.io.IOException e) { + } catch (java.io.IOException e) { throw new RuntimeException(e); } } - public int compare(byte[] b1, int s1, int l1, - byte[] b2, int s2, int l2) { - int ret = compareRaw(b1,s1,l1,b2,s2,l2); - return (ret == -1)? -1 : ((ret==0)? 1 : 0);} + + @Override + public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { + int ret = compareRaw(b1, s1, l1, b2, s2, l2); + return (ret == -1) ? -1 : ((ret == 0) ? 1 : 0); + } } - + static { - org.apache.hadoop.record.RecordComparator.define(RecordTestObj.class, new Comparator()); + org.apache.hadoop.record.RecordComparator.define(RecordTestObj.class, + new Comparator()); } } Index: ql/src/test/org/apache/hadoop/hive/ql/io/PerformTestRCFileAndSeqFile.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/io/PerformTestRCFileAndSeqFile.java (revision 901519) +++ ql/src/test/org/apache/hadoop/hive/ql/io/PerformTestRCFileAndSeqFile.java (working copy) @@ -20,7 +20,7 @@ public class PerformTestRCFileAndSeqFile extends TestCase { - private Configuration conf = new Configuration(); + private final Configuration conf = new Configuration(); private Path testRCFile; private Path testSeqFile; @@ -35,23 +35,24 @@ public PerformTestRCFileAndSeqFile(boolean local, String file) throws IOException { - if (local) + if (local) { fs = FileSystem.getLocal(conf); - else + } else { fs = FileSystem.get(conf); + } conf.setInt(RCFile.Writer.COLUMNS_BUFFER_SIZE_CONF_STR, 1 * 1024 * 1024); if (file == null) { Path dir = new Path(System.getProperty("test.data.dir", ".") + "/mapred"); - this.testRCFile = new Path(dir, "test_rcfile"); - this.testSeqFile = new Path(dir, "test_seqfile"); + testRCFile = new Path(dir, "test_rcfile"); + testSeqFile = new Path(dir, "test_seqfile"); } else { - this.testRCFile = new Path(file + "-rcfile"); - this.testSeqFile = new Path(file + "-seqfile"); + testRCFile = new Path(file + "-rcfile"); + testSeqFile = new Path(file + "-seqfile"); } fs.delete(testRCFile, true); - fs.delete(this.testSeqFile, true); - System.out.println("RCFile:" + this.testRCFile.toString()); - System.out.println("SequenceFile:" + this.testSeqFile.toString()); + fs.delete(testSeqFile, true); + System.out.println("RCFile:" + testRCFile.toString()); + System.out.println("SequenceFile:" + testSeqFile.toString()); } private void writeSeqenceFileTest(FileSystem fs, int rowCount, Path file, @@ -115,10 +116,11 @@ private void nextRandomRow(byte[][] row, BytesRefArrayWritable bytes) { bytes.resetValid(row.length); for (int i = 0; i < row.length; i++) { - int len = Math.abs(randColLenGenerator.nextInt(this.columnMaxSize)); + int len = Math.abs(randColLenGenerator.nextInt(columnMaxSize)); row[i] = new byte[len]; - for (int j = 0; j < len; j++) - row[i][j] = getRandomChar(this.randomCharGenerator); + for (int j = 0; j < len; j++) { + row[i][j] = getRandomChar(randomCharGenerator); + } bytes.get(i).set(row[i], 0, len); } } @@ -130,14 +132,12 @@ do { b = (byte) random.nextInt(CHAR_END); } while ((b < 65)); - if (b > 90) + if (b > 90) { b += 7; + } return b; } - private static String usage = "Usage: PerformTestRCFileAndSeqFile " - + "[-count N]" + " [file]"; - public static void main(String[] args) throws Exception { int count = 1000; String file = null; @@ -191,9 +191,9 @@ // sequence file write start = System.currentTimeMillis(); - writeSeqenceFileTest(fs, rowCount, this.testSeqFile, columnNum, codec); + writeSeqenceFileTest(fs, rowCount, testSeqFile, columnNum, codec); cost = System.currentTimeMillis() - start; - fileLen = fs.getFileStatus(this.testSeqFile).getLen(); + fileLen = fs.getFileStatus(testSeqFile).getLen(); System.out.println("Write SequenceFile with " + columnNum + " random string columns and " + rowCount + " rows cost " + cost + " milliseconds. And the file's on disk size is " + fileLen); @@ -206,13 +206,14 @@ System.out.println("Read only one column of a RCFile with " + columnNum + " random string columns and " + rowCount + " rows cost " + cost + " milliseconds."); - if (rowCount != readRows) + if (rowCount != readRows) { throw new IllegalStateException("Compare read and write row count error."); + } assertEquals("", rowCount, readRows); if (isLocalFileSystem() && !checkCorrect) { // make some noisy to avoid disk caches data. - performSequenceFileRead(fs, rowCount, this.testSeqFile); + performSequenceFileRead(fs, rowCount, testSeqFile); } start = System.currentTimeMillis(); @@ -222,13 +223,14 @@ System.out.println("Read only first and last columns of a RCFile with " + columnNum + " random string columns and " + rowCount + " rows cost " + cost + " milliseconds."); - if (rowCount != readRows) + if (rowCount != readRows) { throw new IllegalStateException("Compare read and write row count error."); + } assertEquals("", rowCount, readRows); if (isLocalFileSystem() && !checkCorrect) { // make some noisy to avoid disk caches data. - performSequenceFileRead(fs, rowCount, this.testSeqFile); + performSequenceFileRead(fs, rowCount, testSeqFile); } start = System.currentTimeMillis(); @@ -237,13 +239,14 @@ System.out.println("Read all columns of a RCFile with " + columnNum + " random string columns and " + rowCount + " rows cost " + cost + " milliseconds."); - if (rowCount != readRows) + if (rowCount != readRows) { throw new IllegalStateException("Compare read and write row count error."); + } assertEquals("", rowCount, readRows); // sequence file read start = System.currentTimeMillis(); - performSequenceFileRead(fs, rowCount, this.testSeqFile); + performSequenceFileRead(fs, rowCount, testSeqFile); cost = System.currentTimeMillis() - start; System.out.println("Read SequenceFile with " + columnNum + " random string columns and " + rowCount + " rows cost " + cost @@ -259,8 +262,9 @@ SequenceFile.Reader reader = new SequenceFile.Reader(fs, file, conf); ByteWritable key = new ByteWritable(); BytesRefArrayWritable val = new BytesRefArrayWritable(); - for (int i = 0; i < count; i++) + for (int i = 0; i < count; i++) { reader.next(key, val); + } } public int performRCFileReadFirstColumnTest(FileSystem fs, Path file, @@ -269,7 +273,7 @@ byte[][] checkBytes = null; BytesRefArrayWritable checkRow = new BytesRefArrayWritable(allColumnsNumber); if (chechCorrect) { - this.resetRandomGenerators(); + resetRandomGenerators(); checkBytes = new byte[allColumnsNumber][]; } @@ -286,11 +290,12 @@ reader.getCurrentRow(cols); boolean ok = true; if (chechCorrect) { - this.nextRandomRow(checkBytes, checkRow); + nextRandomRow(checkBytes, checkRow); ok = ok && (checkRow.get(0).equals(cols.get(0))); } - if (!ok) + if (!ok) { throw new IllegalStateException("Compare read and write error."); + } actualReadCount++; } return actualReadCount; @@ -302,7 +307,7 @@ byte[][] checkBytes = null; BytesRefArrayWritable checkRow = new BytesRefArrayWritable(allColumnsNumber); if (chechCorrect) { - this.resetRandomGenerators(); + resetRandomGenerators(); checkBytes = new byte[allColumnsNumber][]; } @@ -320,14 +325,15 @@ reader.getCurrentRow(cols); boolean ok = true; if (chechCorrect) { - this.nextRandomRow(checkBytes, checkRow); + nextRandomRow(checkBytes, checkRow); ok = ok && (checkRow.get(0).equals(cols.get(0))); ok = ok && checkRow.get(allColumnsNumber - 1).equals( cols.get(allColumnsNumber - 1)); } - if (!ok) + if (!ok) { throw new IllegalStateException("Compare read and write error."); + } actualReadCount++; } return actualReadCount; @@ -339,7 +345,7 @@ byte[][] checkBytes = null; BytesRefArrayWritable checkRow = new BytesRefArrayWritable(allColumnsNumber); if (chechCorrect) { - this.resetRandomGenerators(); + resetRandomGenerators(); checkBytes = new byte[allColumnsNumber][]; } @@ -354,11 +360,12 @@ reader.getCurrentRow(cols); boolean ok = true; if (chechCorrect) { - this.nextRandomRow(checkBytes, checkRow); + nextRandomRow(checkBytes, checkRow); ok = ok && checkRow.equals(cols); } - if (!ok) + if (!ok) { throw new IllegalStateException("Compare read and write error."); + } actualReadCount++; } return actualReadCount; Index: ql/src/test/org/apache/hadoop/hive/ql/io/JavaTestObjFlatFileInputFormat.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/io/JavaTestObjFlatFileInputFormat.java (revision 901519) +++ ql/src/test/org/apache/hadoop/hive/ql/io/JavaTestObjFlatFileInputFormat.java (working copy) @@ -26,11 +26,12 @@ public class JavaTestObjFlatFileInputFormat implements Serializable { public String s; public int num; + public JavaTestObjFlatFileInputFormat(String s, int num) { this.s = s; this.num = num; } - public JavaTestObjFlatFileInputFormat() { + + public JavaTestObjFlatFileInputFormat() { } } - Index: ql/src/test/org/apache/hadoop/hive/ql/tool/TestLineageInfo.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/tool/TestLineageInfo.java (revision 901519) +++ ql/src/test/org/apache/hadoop/hive/ql/tool/TestLineageInfo.java (working copy) @@ -20,10 +20,10 @@ import java.util.TreeSet; +import junit.framework.TestCase; + import org.apache.hadoop.hive.ql.tools.LineageInfo; -import junit.framework.TestCase; - public class TestLineageInfo extends TestCase { /** Index: ql/src/test/org/apache/hadoop/hive/ql/udf/UDFTestLength.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/udf/UDFTestLength.java (revision 901519) +++ ql/src/test/org/apache/hadoop/hive/ql/udf/UDFTestLength.java (working copy) @@ -26,8 +26,9 @@ * A UDF for testing, which evaluates the length of a string. */ public class UDFTestLength extends UDF { - + IntWritable result = new IntWritable(); + public IntWritable evaluate(Text s) { if (s == null) { return null; Index: ql/src/test/org/apache/hadoop/hive/ql/udf/UDFTestLength2.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/udf/UDFTestLength2.java (revision 901519) +++ ql/src/test/org/apache/hadoop/hive/ql/udf/UDFTestLength2.java (working copy) @@ -19,15 +19,13 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.Text; /** - * A UDF for testing, which evaluates the length of a string. - * This UDF uses Java Primitive classes for parameters. + * A UDF for testing, which evaluates the length of a string. This UDF uses Java + * Primitive classes for parameters. */ public class UDFTestLength2 extends UDF { - + public Integer evaluate(String s) { if (s == null) { return null; Index: ql/src/test/org/apache/hadoop/hive/ql/udf/UDAFTestMax.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/udf/UDAFTestMax.java (revision 901519) +++ ql/src/test/org/apache/hadoop/hive/ql/udf/UDAFTestMax.java (working copy) @@ -28,8 +28,6 @@ import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; - - public class UDAFTestMax extends UDAF { static public class MaxShortEvaluator implements UDAFEvaluator { @@ -266,5 +264,4 @@ } } - } Index: ql/src/test/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTestTranslate.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTestTranslate.java (revision 901519) +++ ql/src/test/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTestTranslate.java (working copy) @@ -25,7 +25,6 @@ import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; @@ -33,7 +32,7 @@ import org.apache.hadoop.io.Text; /** - * Mimics oracle's function translate(str1, str2, str3) + * Mimics oracle's function translate(str1, str2, str3) */ public class GenericUDFTestTranslate extends GenericUDF { ObjectInspector[] argumentOIs; @@ -43,43 +42,48 @@ */ static String getOrdinal(int i) { int unit = i % 10; - return (i <= 0) ? "" - : (i != 11 && unit == 1) ? i + "st" - : (i != 12 && unit == 2) ? i + "nd" - : (i != 13 && unit == 3) ? i + "rd" - : i + "th"; + return (i <= 0) ? "" : (i != 11 && unit == 1) ? i + "st" + : (i != 12 && unit == 2) ? i + "nd" : (i != 13 && unit == 3) ? i + "rd" + : i + "th"; } @Override public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { - if(arguments.length != 3) { + if (arguments.length != 3) { throw new UDFArgumentLengthException( - "The function TRANSLATE(expr,from_string,to_string) accepts exactly 3 arguments, but " - + arguments.length + " arguments is found."); + "The function TRANSLATE(expr,from_string,to_string) accepts exactly 3 arguments, but " + + arguments.length + " arguments is found."); } - for(int i = 0; i < 3; i++) { - if(arguments[i].getTypeName() != Constants.STRING_TYPE_NAME + for (int i = 0; i < 3; i++) { + if (arguments[i].getTypeName() != Constants.STRING_TYPE_NAME && arguments[i].getTypeName() != Constants.VOID_TYPE_NAME) { - throw new UDFArgumentTypeException(i, - "The " + getOrdinal(i + 1) + " argument of function TRANSLATE is expected to \"" + Constants.STRING_TYPE_NAME - + "\", but \"" + arguments[i].getTypeName() + "\" is found"); + throw new UDFArgumentTypeException(i, "The " + getOrdinal(i + 1) + + " argument of function TRANSLATE is expected to \"" + + Constants.STRING_TYPE_NAME + "\", but \"" + + arguments[i].getTypeName() + "\" is found"); } } - - this.argumentOIs = arguments; + + argumentOIs = arguments; return PrimitiveObjectInspectorFactory.writableStringObjectInspector; } - private Text resultText = new Text(); + private final Text resultText = new Text(); + @Override public Object evaluate(DeferredObject[] arguments) throws HiveException { - if(arguments[0].get() == null || arguments[1].get() == null || arguments[2].get() == null) + if (arguments[0].get() == null || arguments[1].get() == null + || arguments[2].get() == null) { return null; - String exprString = ((StringObjectInspector)argumentOIs[0]).getPrimitiveJavaObject(arguments[0].get()); - String fromString = ((StringObjectInspector)argumentOIs[1]).getPrimitiveJavaObject(arguments[1].get()); - String toString = ((StringObjectInspector)argumentOIs[2]).getPrimitiveJavaObject(arguments[2].get()); + } + String exprString = ((StringObjectInspector) argumentOIs[0]) + .getPrimitiveJavaObject(arguments[0].get()); + String fromString = ((StringObjectInspector) argumentOIs[1]) + .getPrimitiveJavaObject(arguments[1].get()); + String toString = ((StringObjectInspector) argumentOIs[2]) + .getPrimitiveJavaObject(arguments[2].get()); char[] expr = exprString.toCharArray(); char[] from = fromString.toCharArray(); @@ -88,21 +92,23 @@ System.arraycopy(expr, 0, result, 0, expr.length); Set seen = new HashSet(); - for(int i = 0; i < from.length; i++) { - if(seen.contains(from[i])) + for (int i = 0; i < from.length; i++) { + if (seen.contains(from[i])) { continue; + } seen.add(from[i]); - for(int j = 0; j < expr.length; j++) { - if(expr[j] == from[i]) { + for (int j = 0; j < expr.length; j++) { + if (expr[j] == from[i]) { result[j] = (i < to.length) ? to[i] : 0; } } } int pos = 0; - for(int i = 0; i < result.length; i++) { - if(result[i] != 0) - result[pos ++] = result[i]; + for (int i = 0; i < result.length; i++) { + if (result[i] != 0) { + result[pos++] = result[i]; + } } resultText.set(new String(result, 0, pos)); return resultText; @@ -110,7 +116,8 @@ @Override public String getDisplayString(String[] children) { - assert(children.length == 3); - return "translate(" + children[0] + "," + children[1] + "," + children[2] + ")"; + assert (children.length == 3); + return "translate(" + children[0] + "," + children[1] + "," + children[2] + + ")"; } } Index: ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (revision 901519) +++ ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (working copy) @@ -38,11 +38,9 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.cli.CliDriver; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.Task; @@ -62,28 +60,27 @@ import org.apache.hadoop.hive.serde2.thrift.test.Complex; import org.apache.hadoop.hive.shims.HadoopShims; import org.apache.hadoop.hive.shims.ShimLoader; +import org.apache.hadoop.mapred.MiniMRCluster; import org.apache.hadoop.mapred.SequenceFileInputFormat; import org.apache.hadoop.mapred.SequenceFileOutputFormat; import org.apache.hadoop.mapred.TextInputFormat; -import org.apache.hadoop.mapred.MiniMRCluster; - import org.apache.thrift.protocol.TBinaryProtocol; public class QTestUtil { private String testWarehouse; - private String tmpdir = System.getProperty("test.tmp.dir"); - private Path tmppath = new Path(tmpdir); + private final String tmpdir = System.getProperty("test.tmp.dir"); + private final Path tmppath = new Path(tmpdir); - private String testFiles; - private String outDir; - private String logDir; - private TreeMap qMap; - private LinkedList srcTables; + private final String testFiles; + private final String outDir; + private final String logDir; + private final TreeMap qMap; + private final LinkedList srcTables; private ParseDriver pd; private Hive db; - private HiveConf conf; + private final HiveConf conf; private Driver drv; private SemanticAnalyzer sem; private FileSystem fs; @@ -96,16 +93,15 @@ public boolean deleteDirectory(File path) { if (path.exists()) { File[] files = path.listFiles(); - for(int i=0; i part_spec = new HashMap(); - for (String ds: new String[]{"2008-04-08", "2008-04-09"}) { - for (String hr: new String[]{"11", "12"}) { + for (String ds : new String[] { "2008-04-08", "2008-04-09" }) { + for (String hr : new String[] { "11", "12" }) { part_spec.clear(); part_spec.put("ds", ds); part_spec.put("hr", hr); // System.out.println("Loading partition with spec: " + part_spec); - //db.createPartition(srcpart, part_spec); + // db.createPartition(srcpart, part_spec); fpath = new Path(testFiles, "kv1.txt"); newfpath = new Path(tmppath, "kv1.txt"); fs.copyFromLocalFile(false, true, fpath, newfpath); fpath = newfpath; - //db.loadPartition(fpath, srcpart.getName(), part_spec, true); - runLoadCmd("LOAD DATA INPATH '" + newfpath.toString() + - "' OVERWRITE INTO TABLE srcpart PARTITION (ds='" + ds + "',hr='" + hr +"')"); + // db.loadPartition(fpath, srcpart.getName(), part_spec, true); + runLoadCmd("LOAD DATA INPATH '" + newfpath.toString() + + "' OVERWRITE INTO TABLE srcpart PARTITION (ds='" + ds + "',hr='" + + hr + "')"); } } ArrayList bucketCols = new ArrayList(); bucketCols.add("key"); runCreateTableCmd("CREATE TABLE srcbucket(key int, value string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE"); - //db.createTable("srcbucket", cols, null, TextInputFormat.class, IgnoreKeyTextOutputFormat.class, 2, bucketCols); + // db.createTable("srcbucket", cols, null, TextInputFormat.class, + // IgnoreKeyTextOutputFormat.class, 2, bucketCols); srcTables.add("srcbucket"); - for (String fname: new String [] {"srcbucket0.txt", "srcbucket1.txt"}) { + for (String fname : new String[] { "srcbucket0.txt", "srcbucket1.txt" }) { fpath = new Path(testFiles, fname); newfpath = new Path(tmppath, fname); fs.copyFromLocalFile(false, true, fpath, newfpath); - runLoadCmd("LOAD DATA INPATH '" + newfpath.toString() + "' INTO TABLE srcbucket"); + runLoadCmd("LOAD DATA INPATH '" + newfpath.toString() + + "' INTO TABLE srcbucket"); } - + runCreateTableCmd("CREATE TABLE srcbucket2(key int, value string) CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE"); - //db.createTable("srcbucket", cols, null, TextInputFormat.class, IgnoreKeyTextOutputFormat.class, 2, bucketCols); + // db.createTable("srcbucket", cols, null, TextInputFormat.class, + // IgnoreKeyTextOutputFormat.class, 2, bucketCols); srcTables.add("srcbucket2"); - for (String fname: new String [] {"srcbucket20.txt", "srcbucket21.txt", "srcbucket22.txt", "srcbucket23.txt"}) { + for (String fname : new String[] { "srcbucket20.txt", "srcbucket21.txt", + "srcbucket22.txt", "srcbucket23.txt" }) { fpath = new Path(testFiles, fname); newfpath = new Path(tmppath, fname); fs.copyFromLocalFile(false, true, fpath, newfpath); - runLoadCmd("LOAD DATA INPATH '" + newfpath.toString() + "' INTO TABLE srcbucket2"); + runLoadCmd("LOAD DATA INPATH '" + newfpath.toString() + + "' INTO TABLE srcbucket2"); } - for (String tname: new String [] {"src", "src1"}) { - db.createTable(tname, cols, null, TextInputFormat.class, IgnoreKeyTextOutputFormat.class); + for (String tname : new String[] { "src", "src1" }) { + db.createTable(tname, cols, null, TextInputFormat.class, + IgnoreKeyTextOutputFormat.class); srcTables.add(tname); } - db.createTable("src_sequencefile", cols, null, SequenceFileInputFormat.class, SequenceFileOutputFormat.class); + db.createTable("src_sequencefile", cols, null, + SequenceFileInputFormat.class, SequenceFileOutputFormat.class); srcTables.add("src_sequencefile"); Table srcThrift = new Table("src_thrift"); srcThrift.setInputFormatClass(SequenceFileInputFormat.class.getName()); srcThrift.setOutputFormatClass(SequenceFileOutputFormat.class.getName()); srcThrift.setSerializationLib(ThriftDeserializer.class.getName()); - srcThrift.setSerdeParam(Constants.SERIALIZATION_CLASS, Complex.class.getName()); - srcThrift.setSerdeParam(Constants.SERIALIZATION_FORMAT, TBinaryProtocol.class.getName()); + srcThrift.setSerdeParam(Constants.SERIALIZATION_CLASS, Complex.class + .getName()); + srcThrift.setSerdeParam(Constants.SERIALIZATION_FORMAT, + TBinaryProtocol.class.getName()); db.createTable(srcThrift); srcTables.add("src_thrift"); LinkedList json_cols = new LinkedList(); json_cols.add("json"); - db.createTable("src_json", json_cols, null, TextInputFormat.class, IgnoreKeyTextOutputFormat.class); + db.createTable("src_json", json_cols, null, TextInputFormat.class, + IgnoreKeyTextOutputFormat.class); srcTables.add("src_json"); // load the input data into the src table fpath = new Path(testFiles, "kv1.txt"); newfpath = new Path(tmppath, "kv1.txt"); fs.copyFromLocalFile(false, true, fpath, newfpath); - //db.loadTable(newfpath, "src", false); - runLoadCmd("LOAD DATA INPATH '" + newfpath.toString() + "' INTO TABLE src"); + // db.loadTable(newfpath, "src", false); + runLoadCmd("LOAD DATA INPATH '" + newfpath.toString() + "' INTO TABLE src"); // load the input data into the src table fpath = new Path(testFiles, "kv3.txt"); newfpath = new Path(tmppath, "kv3.txt"); fs.copyFromLocalFile(false, true, fpath, newfpath); - //db.loadTable(newfpath, "src1", false); - runLoadCmd("LOAD DATA INPATH '" + newfpath.toString() + "' INTO TABLE src1"); + // db.loadTable(newfpath, "src1", false); + runLoadCmd("LOAD DATA INPATH '" + newfpath.toString() + "' INTO TABLE src1"); // load the input data into the src_sequencefile table fpath = new Path(testFiles, "kv1.seq"); newfpath = new Path(tmppath, "kv1.seq"); fs.copyFromLocalFile(false, true, fpath, newfpath); - //db.loadTable(newfpath, "src_sequencefile", true); - runLoadCmd("LOAD DATA INPATH '" + newfpath.toString() + "' INTO TABLE src_sequencefile"); + // db.loadTable(newfpath, "src_sequencefile", true); + runLoadCmd("LOAD DATA INPATH '" + newfpath.toString() + + "' INTO TABLE src_sequencefile"); // load the input data into the src_thrift table fpath = new Path(testFiles, "complex.seq"); newfpath = new Path(tmppath, "complex.seq"); fs.copyFromLocalFile(false, true, fpath, newfpath); - //db.loadTable(newfpath, "src_thrift", true); - runLoadCmd("LOAD DATA INPATH '" + newfpath.toString() + "' INTO TABLE src_thrift"); + // db.loadTable(newfpath, "src_thrift", true); + runLoadCmd("LOAD DATA INPATH '" + newfpath.toString() + + "' INTO TABLE src_thrift"); // load the json data into the src_json table fpath = new Path(testFiles, "json.txt"); newfpath = new Path(tmppath, "json.txt"); fs.copyFromLocalFile(false, true, fpath, newfpath); - //db.loadTable(newfpath, "src_json", false); - runLoadCmd("LOAD DATA INPATH '" + newfpath.toString() + "' INTO TABLE src_json"); + // db.loadTable(newfpath, "src_json", false); + runLoadCmd("LOAD DATA INPATH '" + newfpath.toString() + + "' INTO TABLE src_json"); } @@ -410,10 +423,13 @@ part_cols.add("ds"); part_cols.add("hr"); - db.createTable("dest1", cols, null, TextInputFormat.class, IgnoreKeyTextOutputFormat.class); - db.createTable("dest2", cols, null, TextInputFormat.class, IgnoreKeyTextOutputFormat.class); + db.createTable("dest1", cols, null, TextInputFormat.class, + IgnoreKeyTextOutputFormat.class); + db.createTable("dest2", cols, null, TextInputFormat.class, + IgnoreKeyTextOutputFormat.class); - db.createTable("dest3", cols, part_cols, TextInputFormat.class, IgnoreKeyTextOutputFormat.class); + db.createTable("dest3", cols, part_cols, TextInputFormat.class, + IgnoreKeyTextOutputFormat.class); Table dest3 = db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, "dest3"); HashMap part_spec = new HashMap(); @@ -421,8 +437,10 @@ part_spec.put("hr", "12"); db.createPartition(dest3, part_spec); - db.createTable("dest4", cols, null, TextInputFormat.class, IgnoreKeyTextOutputFormat.class); - db.createTable("dest4_sequencefile", cols, null, SequenceFileInputFormat.class, SequenceFileOutputFormat.class); + db.createTable("dest4", cols, null, TextInputFormat.class, + IgnoreKeyTextOutputFormat.class); + db.createTable("dest4_sequencefile", cols, null, + SequenceFileInputFormat.class, SequenceFileOutputFormat.class); } public void cliInit(String tname) throws Exception { @@ -436,7 +454,7 @@ } CliSessionState ss = new CliSessionState(conf); - assert ss!= null; + assert ss != null; ss.in = System.in; File qf = new File(outDir, tname); @@ -454,11 +472,12 @@ public int executeOne(String tname) { String q = qMap.get(tname); - if(q.indexOf(";") == -1) + if (q.indexOf(";") == -1) { return -1; + } String q1 = q.substring(0, q.indexOf(";") + 1); - String qrest = q.substring(q.indexOf(";")+1); + String qrest = q.substring(q.indexOf(";") + 1); qMap.put(tname, qrest); System.out.println("Executing " + q1); @@ -483,10 +502,12 @@ cols.add("value"); // Move all data from dest4_sequencefile to dest4 - drv.run("FROM dest4_sequencefile INSERT OVERWRITE TABLE dest4 SELECT dest4_sequencefile.*"); + drv + .run("FROM dest4_sequencefile INSERT OVERWRITE TABLE dest4 SELECT dest4_sequencefile.*"); // Drop dest4_sequencefile - db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, "dest4_sequencefile", true, true); + db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, "dest4_sequencefile", + true, true); } public int checkNegativeResults(String tname, Exception e) throws Exception { @@ -502,11 +523,9 @@ FileWriter outfd = new FileWriter(outf); if (e instanceof ParseException) { outfd.write("Parse Error: "); - } - else if (e instanceof SemanticException) { + } else if (e instanceof SemanticException) { outfd.write("Semantic Exception: \n"); - } - else { + } else { throw e; } @@ -518,15 +537,17 @@ Process executor = Runtime.getRuntime().exec(cmdLine); - StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, System.out); - StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, System.err); + StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), + null, System.out); + StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), + null, System.err); outPrinter.start(); errPrinter.start(); int exitVal = executor.waitFor(); - if(exitVal != 0 && overWrite) { + if (exitVal != 0 && overWrite) { System.out.println("Overwriting results"); cmdLine = "cp " + outf.getPath() + " " + expf.getPath(); executor = Runtime.getRuntime().exec(cmdLine); @@ -555,15 +576,17 @@ Process executor = Runtime.getRuntime().exec(cmdLine); - StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, System.out); - StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, System.err); + StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), + null, System.out); + StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), + null, System.err); outPrinter.start(); errPrinter.start(); int exitVal = executor.waitFor(); - if(exitVal != 0 && overWrite) { + if (exitVal != 0 && overWrite) { System.out.println("Overwriting results"); cmdLine = "cp " + outf.getPath() + " " + expf.getPath(); executor = Runtime.getRuntime().exec(cmdLine); @@ -571,13 +594,13 @@ } return exitVal; - } - else { + } else { throw new Exception("Parse tree is null"); } } - public int checkPlan(String tname, List> tasks) throws Exception { + public int checkPlan(String tname, List> tasks) + throws Exception { if (tasks != null) { File planDir = new File(outDir, "plan"); @@ -588,35 +611,37 @@ outf = new File(outf, tname.concat(".xml")); FileOutputStream ofs = new FileOutputStream(outf); - for(Task plan: tasks) { + for (Task plan : tasks) { Utilities.serializeTasks(plan, ofs); } - String [] cmdArray = new String[6]; + String[] cmdArray = new String[6]; cmdArray[0] = "diff"; cmdArray[1] = "-b"; cmdArray[2] = "-I"; - cmdArray[3] = "\\(\\(\\)" + - "\\|\\(.*/tmp/.*\\)" + - "\\|\\(file:.*\\)" + - "\\|\\([0-9]\\{10\\}\\)" + - "\\|\\(/.*/warehouse/.*\\)\\)"; + cmdArray[3] = "\\(\\(\\)" + + "\\|\\(.*/tmp/.*\\)" + + "\\|\\(file:.*\\)" + + "\\|\\([0-9]\\{10\\}\\)" + + "\\|\\(/.*/warehouse/.*\\)\\)"; cmdArray[4] = outf.getPath(); cmdArray[5] = planFile.getPath(); - System.out.println(cmdArray[0] + " " + cmdArray[1] + " " + cmdArray[2] + "\'" + cmdArray[3] + "\'" + - " " + cmdArray[4] + " " + cmdArray[5]); + System.out.println(cmdArray[0] + " " + cmdArray[1] + " " + cmdArray[2] + + "\'" + cmdArray[3] + "\'" + " " + cmdArray[4] + " " + cmdArray[5]); Process executor = Runtime.getRuntime().exec(cmdArray); - StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, System.out); - StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, System.err); + StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), + null, System.out); + StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), + null, System.err); outPrinter.start(); errPrinter.start(); int exitVal = executor.waitFor(); - if(exitVal != 0 && overWrite) { + if (exitVal != 0 && overWrite) { System.out.println("Overwriting results"); String cmdLine = "cp " + outf.getPath() + " " + planFile.getPath(); executor = Runtime.getRuntime().exec(cmdLine); @@ -624,8 +649,7 @@ } return exitVal; - } - else { + } else { throw new Exception("Plan is null"); } @@ -638,7 +662,8 @@ Path localPath = new Path(FileSystem.getLocal(conf).getUri().getPath()); localPath = new Path(localPath, logDir); localPath = new Path(localPath, "warehouse_local_copy"); - System.out.println("warehousePath = " + warehousePath.toString() + " localPath = " + localPath.toString()); + System.out.println("warehousePath = " + warehousePath.toString() + + " localPath = " + localPath.toString()); if (FileSystem.getLocal(conf).exists(localPath)) { FileSystem.getLocal(conf).delete(localPath, true); @@ -647,7 +672,7 @@ copyDirectoryToLocal(warehousePath, localPath); normalizeNames(new File(localPath.toUri().getPath())); - String [] cmdArray; + String[] cmdArray; if (overWrite == false) { cmdArray = new String[6]; cmdArray[0] = "diff"; @@ -656,22 +681,24 @@ cmdArray[3] = "--exclude=.svn"; cmdArray[4] = localPath.toUri().getPath(); cmdArray[5] = (new File(outDir, tname)).getPath() + "/warehouse"; - System.out.println(cmdArray[0] + " " + cmdArray[1] + " " + cmdArray[2] + " " + - cmdArray[3] + " " + cmdArray[4] + " " + cmdArray[5]); - } - else { + System.out.println(cmdArray[0] + " " + cmdArray[1] + " " + cmdArray[2] + + " " + cmdArray[3] + " " + cmdArray[4] + " " + cmdArray[5]); + } else { System.out.println("overwritting"); // Remove any existing output - String [] cmdArray1 = new String[5]; + String[] cmdArray1 = new String[5]; cmdArray1[0] = "rm"; cmdArray1[1] = "-rf"; cmdArray1[2] = (new File(outDir, tname)).getPath(); - System.out.println(cmdArray1[0] + " " + cmdArray1[1] + " " + cmdArray1[2]); + System.out + .println(cmdArray1[0] + " " + cmdArray1[1] + " " + cmdArray1[2]); Process executor = Runtime.getRuntime().exec(cmdArray1); - StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, System.out); - StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, System.err); + StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), + null, System.out); + StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), + null, System.err); outPrinter.start(); errPrinter.start(); @@ -686,13 +713,16 @@ cmdArray[1] = "-r"; cmdArray[2] = localPath.toUri().getPath(); cmdArray[3] = (new File(outDir, tname)).getPath(); - System.out.println(cmdArray[0] + " " + cmdArray[1] + " " + cmdArray[2] + " " + cmdArray[3]); + System.out.println(cmdArray[0] + " " + cmdArray[1] + " " + cmdArray[2] + + " " + cmdArray[3]); } Process executor = Runtime.getRuntime().exec(cmdArray); - StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, System.out); - StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, System.err); + StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), + null, System.out); + StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), + null, System.err); outPrinter.start(); errPrinter.start(); @@ -703,42 +733,29 @@ } public int checkCliDriverResults(String tname) throws Exception { - String [] cmdArray; + String[] cmdArray; - cmdArray = new String[] { - "diff", - "-a", - "-I", - "file:", - "-I", - "/tmp/", - "-I", - "invalidscheme:", - "-I", - "lastUpdateTime", - "-I", - "lastAccessTime", - "-I", - "owner", - "-I", - "transient_lastDdlTime", + cmdArray = new String[] { "diff", "-a", "-I", "file:", "-I", "/tmp/", "-I", + "invalidscheme:", "-I", "lastUpdateTime", "-I", "lastAccessTime", "-I", + "owner", "-I", "transient_lastDdlTime", (new File(logDir, tname + ".out")).getPath(), - (new File(outDir, tname + ".out")).getPath() - }; + (new File(outDir, tname + ".out")).getPath() }; System.out.println(org.apache.commons.lang.StringUtils.join(cmdArray, ' ')); Process executor = Runtime.getRuntime().exec(cmdArray); - StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, System.out); - StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, System.err); + StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), + null, System.out); + StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), + null, System.err); outPrinter.start(); errPrinter.start(); int exitVal = executor.waitFor(); - if(exitVal != 0 && overWrite) { + if (exitVal != 0 && overWrite) { System.out.println("Overwriting results"); cmdArray = new String[3]; cmdArray[0] = "cp"; @@ -756,12 +773,13 @@ return pd.parse(qMap.get(tname)); } - public List> analyzeAST(ASTNode ast) throws Exception { + public List> analyzeAST(ASTNode ast) + throws Exception { // Do semantic analysis and plan generation Context ctx = new Context(conf); - while((ast.getToken() == null) && (ast.getChildCount() > 0)) { - ast = (ASTNode)ast.getChild(0); + while ((ast.getToken() == null) && (ast.getChildCount() > 0)) { + ast = (ASTNode) ast.getChild(0); } sem.analyze(ast, ctx); @@ -769,19 +787,17 @@ return sem.getRootTasks(); } - public TreeMap getQMap() { return qMap; } - /** * QTRunner: Runnable class for running a a single query file - * + * **/ public static class QTRunner implements Runnable { - private QTestUtil qt; - private String fname; + private final QTestUtil qt; + private final String fname; public QTRunner(QTestUtil qt, String fname) { this.qt = qt; @@ -795,7 +811,8 @@ qt.cliInit(fname, false); qt.executeClient(fname); } catch (Throwable e) { - System.err.println("Query file " + fname + " failed with exception " + e.getMessage()); + System.err.println("Query file " + fname + " failed with exception " + + e.getMessage()); e.printStackTrace(); System.err.flush(); } @@ -803,26 +820,33 @@ } /** - * executes a set of query files either in sequence or in parallel. - * Uses QTestUtil to do so - * - * @param qfiles array of input query files containing arbitrary number of hive queries - * @param resDirs array of output directories one corresponding to each input query file - * @param mt whether to run in multithreaded mode or not + * executes a set of query files either in sequence or in parallel. Uses + * QTestUtil to do so + * + * @param qfiles + * array of input query files containing arbitrary number of hive + * queries + * @param resDirs + * array of output directories one corresponding to each input query + * file + * @param mt + * whether to run in multithreaded mode or not * @return true if all the query files were executed successfully, else false - * - * In multithreaded mode each query file is run in a separate thread. the caller has to - * arrange that different query files do not collide (in terms of destination tables) + * + * In multithreaded mode each query file is run in a separate thread. + * the caller has to arrange that different query files do not collide + * (in terms of destination tables) */ - public static boolean queryListRunner(File [] qfiles, String [] resDirs, String[] logDirs, boolean mt) { + public static boolean queryListRunner(File[] qfiles, String[] resDirs, + String[] logDirs, boolean mt) { - assert(qfiles.length == resDirs.length); - assert(qfiles.length == logDirs.length); + assert (qfiles.length == resDirs.length); + assert (qfiles.length == logDirs.length); boolean failed = false; try { - QTestUtil[] qt = new QTestUtil [qfiles.length]; - for(int i=0; i