diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java index 875d80a..28ffdd8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java @@ -272,7 +272,7 @@ protected void initializeOp(Configuration hconf) throws HiveException { rowContainerStandardObjectInspectors[pos], alias, 1, spillTableDesc, conf, !hasFilter(pos), reporter); - values.add(dummyObj[pos]); + values.addRow(dummyObj[pos]); dummyObjVectors[pos] = values; // if serde is null, the input doesn't need to be spilled out @@ -690,7 +690,7 @@ protected void checkAndGenObject() throws HiveException { } if (alw.rowCount() == 0) { - alw.add(dummyObj[i]); + alw.addRow(dummyObj[i]); hasNulls = true; } else if (condn[i].getPreserved()) { preserve = true; @@ -728,7 +728,7 @@ protected void checkAndGenObject() throws HiveException { } else { if (alw.rowCount() == 0) { hasEmpty = true; - alw.add(dummyObj[i]); + alw.addRow(dummyObj[i]); } else if (!hasEmpty && alw.rowCount() == 1) { if (hasAnyFiltered(alias, alw.rowIter().first())) { hasEmpty = true; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java index 377f01d..e7d9338 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java @@ -91,7 +91,6 @@ private transient Byte[] order; // order in which the results should private Configuration hconf; - private transient Byte alias; private transient MapJoinTableContainer[] mapJoinTables; private transient MapJoinTableContainerSerDe[] mapJoinTableSerdes; @@ -99,11 +98,9 @@ private static final Object[] EMPTY_OBJECT_ARRAY = new Object[0]; private static final MapJoinEagerRowContainer EMPTY_ROW_CONTAINER = new MapJoinEagerRowContainer(); static { - EMPTY_ROW_CONTAINER.add(EMPTY_OBJECT_ARRAY); + EMPTY_ROW_CONTAINER.addRow(EMPTY_OBJECT_ARRAY); } - private transient boolean noOuterJoin; - private long rowNumber = 0; private transient LogHelper console; private long hashTableScale; @@ -132,7 +129,6 @@ protected void initializeOp(Configuration hconf) throws HiveException { // initialize some variables, which used to be initialized in CommonJoinOperator this.hconf = hconf; - noOuterJoin = conf.isNoOuterJoin(); filterMaps = conf.getFilterMap(); int tagLen = conf.getTagLength(); @@ -155,7 +151,7 @@ protected void initializeOp(Configuration hconf) throws HiveException { joinFilterObjectInspectors = JoinUtil.getObjectInspectorsFromEvaluators(joinFilters, inputObjInspectors, posBigTableAlias, tagLen); - if (!noOuterJoin) { + if (!conf.isNoOuterJoin()) { for (Byte alias : order) { if (alias == posBigTableAlias || joinValues[alias] == null) { continue; @@ -228,7 +224,7 @@ protected void initializeOp(Configuration hconf) throws HiveException { */ @Override public void processOp(Object row, int tag) throws HiveException { - alias = (byte)tag; + byte alias = (byte)tag; // compute keys and values as StandardObjects. Use non-optimized key (MR). MapJoinKey key = MapJoinKey.readFromRow(null, new MapJoinKeyObject(), row, joinKeys[alias], joinKeysObjectInspectors[alias], true); @@ -243,7 +239,7 @@ public void processOp(Object row, int tag) throws HiveException { if (rowContainer == null) { if(value.length != 0) { rowContainer = new MapJoinEagerRowContainer(); - rowContainer.add(value); + rowContainer.addRow(value); } else { rowContainer = EMPTY_ROW_CONTAINER; } @@ -254,10 +250,10 @@ public void processOp(Object row, int tag) throws HiveException { tableContainer.put(key, rowContainer); } else if (rowContainer == EMPTY_ROW_CONTAINER) { rowContainer = rowContainer.copy(); - rowContainer.add(value); + rowContainer.addRow(value); tableContainer.put(key, rowContainer); } else { - rowContainer.add(value); + rowContainer.addRow(value); } } private boolean hasFilter(int alias) { diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java index 6894245..c747099 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java @@ -122,7 +122,7 @@ public void processOp(Object row, int tag) throws HiveException { endGroup(); startGroup(); } - storage[alias].add(nr); + storage[alias].addRow(nr); } catch (Exception e) { e.printStackTrace(); throw new HiveException(e); @@ -208,7 +208,6 @@ private void moveUpFiles(Path specPath, Configuration hconf, Log log) * @param hconf * @param success * @param log - * @param dpCtx * @throws IOException * @throws HiveException */ diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java index 9fb473c..3daf7a5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java @@ -210,7 +210,7 @@ public void processOp(Object row, int tag) throws HiveException { if (joinNeeded) { List value = getFilteredValue(alias, row); // Add the value to the ArrayList - storage[alias].add(value); + storage[alias].addRow(value); // generate the output records checkAndGenObject(); } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/PTFPartition.java ql/src/java/org/apache/hadoop/hive/ql/exec/PTFPartition.java index c4a4fa2..3c717d6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/PTFPartition.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/PTFPartition.java @@ -90,11 +90,11 @@ public void append(Object o) throws HiveException { @SuppressWarnings("unchecked") List l = (List) ObjectInspectorUtils.copyToStandardObject(o, inputOI, ObjectInspectorCopyOption.WRITABLE); - elems.add(l); + elems.addRow(l); } public int size() { - return (int) elems.rowCount(); + return elems.rowCount(); } public PTFPartitionIterator iterator() throws HiveException { diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java index 5046a45..487bb33 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java @@ -274,7 +274,7 @@ public void processOp(Object row, int tag) throws HiveException { boolean nextKeyGroup = processKey(alias, key); if (nextKeyGroup) { //assert this.nextGroupStorage[alias].size() == 0; - this.nextGroupStorage[alias].add(value); + this.nextGroupStorage[alias].addRow(value); foundNextKeyGroup[tag] = true; if (tag != posBigTable) { return; @@ -299,7 +299,7 @@ public void processOp(Object row, int tag) throws HiveException { } assert !nextKeyGroup; - candidateStorage[tag].add(value); + candidateStorage[tag].addRow(value); } /* diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/AbstractRowContainer.java ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/AbstractRowContainer.java index f0fa59f..7ef5ebd 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/AbstractRowContainer.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/AbstractRowContainer.java @@ -29,7 +29,12 @@ public RowIterator rowIter() throws HiveException; - public void add(ROW t) throws HiveException; + /** + * add a row into the RowContainer + * + * @param t row + */ + public void addRow(ROW t) throws HiveException; /** * @return number of elements in the RowContainer diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/LazyFlatRowContainer.java ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/LazyFlatRowContainer.java index ffeb2ab..089871c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/LazyFlatRowContainer.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/LazyFlatRowContainer.java @@ -18,10 +18,8 @@ package org.apache.hadoop.hive.ql.exec.persistence; - import java.io.DataInput; import java.io.DataOutput; -import java.io.IOException; import java.io.ObjectOutputStream; import java.util.AbstractCollection; import java.util.AbstractList; @@ -32,8 +30,6 @@ import java.util.ListIterator; import java.util.NoSuchElementException; -import javax.naming.OperationNotSupportedException; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -115,7 +111,7 @@ private void addLazy(MapJoinObjectSerDeContext valueContext, BytesWritable curre // Implementation of AbstractRowContainer and assorted methods @Override - public void add(List t) throws HiveException { + public void addRow(List t) throws HiveException { LOG.debug("Add is called with " + t.size() + " objects"); // This is not called when building HashTable; we don't expect it to be called ever. int offset = prepareForAdd(t.size()); @@ -126,7 +122,7 @@ public void add(List t) throws HiveException { } @Override - public void add(Object[] value) throws HiveException { + public void addRow(Object[] value) throws HiveException { LOG.debug("Add is called with " + value.length + " objects"); // This is not called when building HashTable; we don't expect it to be called ever. int offset = prepareForAdd(value.length); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinEagerRowContainer.java ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinEagerRowContainer.java index 5a18464..65bb1b7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinEagerRowContainer.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinEagerRowContainer.java @@ -49,13 +49,13 @@ public MapJoinEagerRowContainer() { } @Override - public void add(List t) { + public void addRow(List t) { list.add(t); } @Override - public void add(Object[] t) { - add(toList(t)); + public void addRow(Object[] t) { + addRow(toList(t)); } @Override @@ -108,7 +108,7 @@ public byte getAliasFilter() { public MapJoinRowContainer copy() { MapJoinEagerRowContainer result = new MapJoinEagerRowContainer(); for(List item : list) { - result.add(item); + result.addRow(item); } return result; } @@ -129,13 +129,13 @@ public void read(MapJoinObjectSerDeContext context, Writable currentValue) throw List value = (List)ObjectInspectorUtils.copyToStandardObject(serde.deserialize(currentValue), serde.getObjectInspector(), ObjectInspectorCopyOption.WRITABLE); if(value == null) { - add(toList(EMPTY_OBJECT_ARRAY)); + addRow(toList(EMPTY_OBJECT_ARRAY)); } else { Object[] valuesArray = value.toArray(); if (context.hasFilterTag()) { aliasFilter &= ((ShortWritable)valuesArray[valuesArray.length - 1]).get(); } - add(toList(valuesArray)); + addRow(toList(valuesArray)); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinRowContainer.java ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinRowContainer.java index fe6a3c5..008a8db 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinRowContainer.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinRowContainer.java @@ -31,7 +31,7 @@ public MapJoinRowContainer copy() throws HiveException; - public void add(Object[] value) throws HiveException; + public void addRow(Object[] value) throws HiveException; public void write(MapJoinObjectSerDeContext valueContext, ObjectOutputStream out) throws IOException, SerDeException; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/PTFRowContainer.java ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/PTFRowContainer.java index 6a31189..4ab5516 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/PTFRowContainer.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/PTFRowContainer.java @@ -91,7 +91,7 @@ public PTFRowContainer(int bs, Configuration jc, Reporter reporter } @Override - public void add(Row t) throws HiveException { + public void addRow(Row t) throws HiveException { if ( willSpill() ) { setupWriter(); PTFRecordWriter rw = (PTFRecordWriter) getRecordWriter(); @@ -105,7 +105,7 @@ public void add(Row t) throws HiveException { throw new HiveException(e); } } - super.add(t); + super.addRow(t); } @Override diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java index bfc13c1..768467e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java @@ -159,7 +159,7 @@ public void setSerDe(SerDe sd, ObjectInspector oi) { } @Override - public void add(ROW t) throws HiveException { + public void addRow(ROW t) throws HiveException { if (this.tblDesc != null) { if (willSpill()) { // spill the current block to tmp file spillBlock(currentWriteBlock, addCursor); @@ -279,9 +279,9 @@ public ROW next() throws HiveException { private void removeKeys(ROW ret) { if (this.keyObject != null && this.currentReadBlock != this.currentWriteBlock) { int len = this.keyObject.size(); - int rowSize = ((ArrayList) ret).size(); + int rowSize = ret.size(); for (int i = 0; i < len; i++) { - ((ArrayList) ret).remove(rowSize - i - 1); + ret.remove(rowSize - i - 1); } } } diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestMapJoinEqualityTableContainer.java ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestMapJoinEqualityTableContainer.java index 14180bd..65e3779 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestMapJoinEqualityTableContainer.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestMapJoinEqualityTableContainer.java @@ -39,7 +39,7 @@ @Before public void setup() throws Exception { rowContainer = new MapJoinEagerRowContainer(); - rowContainer.add(VALUE); + rowContainer.addRow(VALUE); container = new HashMapWrapper(); } @Test diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestMapJoinRowContainer.java ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestMapJoinRowContainer.java index 091fd89..a666fbf 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestMapJoinRowContainer.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestMapJoinRowContainer.java @@ -30,10 +30,10 @@ @Test public void testSerialization() throws Exception { MapJoinRowContainer container1 = new MapJoinEagerRowContainer(); - container1.add(new Object[]{ new Text("f0"), null, new ShortWritable((short)0xf)}); - container1.add(Arrays.asList(new Object[]{ null, new Text("f1"), new ShortWritable((short)0xf)})); - container1.add(new Object[]{ null, null, new ShortWritable((short)0xf)}); - container1.add(Arrays.asList(new Object[]{ new Text("f0"), new Text("f1"), new ShortWritable((short)0x1)})); + container1.addRow(new Object[]{ new Text("f0"), null, new ShortWritable((short)0xf)}); + container1.addRow(Arrays.asList(new Object[]{ null, new Text("f1"), new ShortWritable((short)0xf)})); + container1.addRow(new Object[]{ null, null, new ShortWritable((short)0xf)}); + container1.addRow(Arrays.asList(new Object[]{ new Text("f0"), new Text("f1"), new ShortWritable((short)0x1)})); MapJoinRowContainer container2 = Utilities.serde(container1, "f0,f1,filter", "string,string,smallint"); Utilities.testEquality(container1, container2); Assert.assertEquals(4, container1.rowCount()); diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestMapJoinTableContainer.java ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestMapJoinTableContainer.java index d0701d6..755d783 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestMapJoinTableContainer.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestMapJoinTableContainer.java @@ -46,7 +46,7 @@ public void setup() throws Exception { key = new MapJoinKeyObject(KEY); rowContainer = new MapJoinEagerRowContainer(); - rowContainer.add(VALUE); + rowContainer.addRow(VALUE); baos = new ByteArrayOutputStream(); out = new ObjectOutputStream(baos); diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestPTFRowContainer.java ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestPTFRowContainer.java index 37855bc..cea3529 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestPTFRowContainer.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestPTFRowContainer.java @@ -81,7 +81,7 @@ private void runTest(int sz, int blockSize) throws SerDeException, HiveException row.add(new DoubleWritable(i)); row.add(new IntWritable(i)); row.add(new Text("def " + i)); - rc.add(row); + rc.addRow(row); } // test forward scan