diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 2b8280e..1b4c96d 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -710,8 +710,8 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { HIVEMAPJOINUSEOPTIMIZEDTABLE("hive.mapjoin.optimized.hashtable", true, "Whether Hive should use memory-optimized hash table for MapJoin. Only works on Tez,\n" + "because memory-optimized hashtable cannot be serialized."), - HIVEUSEHYBRIDGRACEHASHJOIN("hive.mapjoin.hybridgrace.hashtable", false, "Whether to use hybrid" + - "grace hash join as the join method for mapjoin."), + HIVEUSEHYBRIDGRACEHASHJOIN("hive.mapjoin.hybridgrace.hashtable", true, "Whether to use hybrid" + + "grace hash join as the join method for mapjoin. Tez only."), HIVEHYBRIDGRACEHASHJOINMEMCHECKFREQ("hive.mapjoin.hybridgrace.memcheckfrequency", 1024, "For " + "hybrid grace hash join, how often (how many rows apart) we check if memory is full. " + "This number should be power of 2."), diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java index 9c3ec8e..2289a35 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java @@ -37,18 +37,20 @@ import org.apache.hadoop.hive.ql.HashTableLoaderFactory; import org.apache.hadoop.hive.ql.exec.mapjoin.MapJoinMemoryExhaustionHandler; import org.apache.hadoop.hive.ql.exec.mr.ExecMapperContext; +import org.apache.hadoop.hive.ql.exec.persistence.BytesBytesMultiHashMap; +import org.apache.hadoop.hive.ql.exec.persistence.HybridHashTableContainer; +import org.apache.hadoop.hive.ql.exec.persistence.HybridHashTableContainer.HashPartition; +import org.apache.hadoop.hive.ql.exec.persistence.KeyValueContainer; +import org.apache.hadoop.hive.ql.exec.persistence.MapJoinBytesTableContainer; +import org.apache.hadoop.hive.ql.exec.persistence.MapJoinBytesTableContainer.KeyValueHelper; import org.apache.hadoop.hive.ql.exec.persistence.MapJoinKey; import org.apache.hadoop.hive.ql.exec.persistence.MapJoinObjectSerDeContext; import org.apache.hadoop.hive.ql.exec.persistence.MapJoinRowContainer; import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer; -import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainerSerDe; import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer.ReusableGetAdaptor; -import org.apache.hadoop.hive.ql.exec.persistence.UnwrapRowContainer; -import org.apache.hadoop.hive.ql.exec.persistence.BytesBytesMultiHashMap; -import org.apache.hadoop.hive.ql.exec.persistence.MapJoinBytesTableContainer; -import org.apache.hadoop.hive.ql.exec.persistence.HybridHashTableContainer; -import org.apache.hadoop.hive.ql.exec.persistence.KeyValueContainer; +import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainerSerDe; import org.apache.hadoop.hive.ql.exec.persistence.ObjectContainer; +import org.apache.hadoop.hive.ql.exec.persistence.UnwrapRowContainer; import org.apache.hadoop.hive.ql.io.HiveKey; import org.apache.hadoop.hive.ql.log.PerfLogger; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -63,9 +65,6 @@ import org.apache.hadoop.io.Writable; import org.apache.hadoop.util.ReflectionUtils; -import static org.apache.hadoop.hive.ql.exec.persistence.HybridHashTableContainer.HashPartition; -import static org.apache.hadoop.hive.ql.exec.persistence.MapJoinBytesTableContainer.KeyValueHelper; - /** * Map side Join operator implementation. */ @@ -80,6 +79,7 @@ private transient ObjectCache cache; protected HashTableLoader loader; + private boolean loadCalled; protected transient MapJoinTableContainer[] mapJoinTables; private transient MapJoinTableContainerSerDe[] mapJoinTableSerdes; @@ -88,7 +88,6 @@ private UnwrapRowContainer[] unwrapContainer; private transient Configuration hconf; - private transient boolean useHybridGraceHashJoin; // whether Hybrid Grace Hash Join is enabled private transient boolean hybridMapJoinLeftover; // whether there's spilled data to be processed private transient MapJoinBytesTableContainer currentSmallTable; // reloaded hashmap from disk private transient int tag; // big table alias @@ -141,15 +140,13 @@ public void startGroup() throws HiveException { mapJoinTables = new MapJoinTableContainer[tagLen]; mapJoinTableSerdes = new MapJoinTableContainerSerDe[tagLen]; hashTblInitedOnce = false; - useHybridGraceHashJoin = - HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEUSEHYBRIDGRACEHASHJOIN); generateMapMetaData(); final ExecMapperContext mapContext = getExecContext(); final MapredContext mrContext = MapredContext.get(); - if (!conf.isBucketMapJoin() && !useHybridGraceHashJoin) { + if (!conf.isBucketMapJoin()) { /* * The issue with caching in case of bucket map join is that different tasks * process different buckets and if the container is reused to join a different bucket, @@ -187,8 +184,22 @@ protected final void completeInitializationOp(Object[] os) throws HiveException if (os.length != 0) { Pair pair = (Pair) os[0]; - mapJoinTables = pair.getLeft(); - mapJoinTableSerdes = pair.getRight(); + + boolean spilled = false; + for (MapJoinTableContainer container : pair.getLeft()) { + if (container != null) { + spilled = spilled || container.hasSpill(); + } + } + + if (!loadCalled && spilled) { + // we can't use the cached table because it has spilled. + loadHashTable(getExecContext(), MapredContext.get()); + } else { + // let's use the table from the cache. + mapJoinTables = pair.getLeft(); + mapJoinTableSerdes = pair.getRight(); + } hashTblInitedOnce = true; } @@ -258,6 +269,8 @@ public void generateMapMetaData() throws HiveException { private Pair loadHashTable( ExecMapperContext mapContext, MapredContext mrContext) throws HiveException { + loadCalled = true; + if (this.hashTblInitedOnce && ((mapContext == null) || (mapContext.getLocalWork() == null) || (mapContext .getLocalWork().getInputFileChangeSensitive() == false))) { @@ -359,8 +372,8 @@ public void process(Object row, int tag) throws HiveException { if (!noOuterJoin) { // For Hybrid Grace Hash Join, during the 1st round processing, // we only keep the LEFT side if the row is not spilled - if (!useHybridGraceHashJoin || hybridMapJoinLeftover || - (!hybridMapJoinLeftover && joinResult != JoinUtil.JoinResult.SPILL)) { + if (!conf.isHybridHashJoin() || hybridMapJoinLeftover + || (!hybridMapJoinLeftover && joinResult != JoinUtil.JoinResult.SPILL)) { joinNeeded = true; storage[pos] = dummyObjVectors[pos]; } @@ -414,47 +427,57 @@ protected void spillBigTableRow(MapJoinTableContainer hybridHtContainer, Object @Override public void closeOp(boolean abort) throws HiveException { - for (MapJoinTableContainer tableContainer : mapJoinTables) { - if (tableContainer != null) { - tableContainer.dumpMetrics(); - - if (tableContainer instanceof HybridHashTableContainer) { - HybridHashTableContainer hybridHtContainer = (HybridHashTableContainer) tableContainer; - hybridHtContainer.dumpStats(); - - HashPartition[] hashPartitions = hybridHtContainer.getHashPartitions(); - // Clear all in memory partitions first - for (int i = 0; i < hashPartitions.length; i++) { - if (!hashPartitions[i].isHashMapOnDisk()) { - hybridHtContainer.setTotalInMemRowCount( - hybridHtContainer.getTotalInMemRowCount() - - hashPartitions[i].getHashMapFromMemory().getNumValues()); - hashPartitions[i].getHashMapFromMemory().clear(); + + boolean spilled = false; + for (MapJoinTableContainer container: mapJoinTables) { + if (container != null) { + spilled = spilled || container.hasSpill(); + container.dumpMetrics(); + } + } + + if (spilled) { + for (MapJoinTableContainer tableContainer : mapJoinTables) { + if (tableContainer != null) { + if (tableContainer instanceof HybridHashTableContainer) { + HybridHashTableContainer hybridHtContainer = (HybridHashTableContainer) tableContainer; + hybridHtContainer.dumpStats(); + + HashPartition[] hashPartitions = hybridHtContainer.getHashPartitions(); + // Clear all in memory partitions first + for (int i = 0; i < hashPartitions.length; i++) { + if (!hashPartitions[i].isHashMapOnDisk()) { + hybridHtContainer.setTotalInMemRowCount( + hybridHtContainer.getTotalInMemRowCount() - + hashPartitions[i].getHashMapFromMemory().getNumValues()); + hashPartitions[i].getHashMapFromMemory().clear(); + } } - } - assert hybridHtContainer.getTotalInMemRowCount() == 0; - - for (int i = 0; i < hashPartitions.length; i++) { - if (hashPartitions[i].isHashMapOnDisk()) { - // Recursively process on-disk triplets (hash partition, sidefile, matchfile) - try { - hybridMapJoinLeftover = true; - hashMapRowGetters[smallTable] = null; - continueProcess(hashPartitions[i], hybridHtContainer); - } catch (IOException e) { - e.printStackTrace(); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } catch (SerDeException e) { - e.printStackTrace(); + assert hybridHtContainer.getTotalInMemRowCount() == 0; + + for (int i = 0; i < hashPartitions.length; i++) { + if (hashPartitions[i].isHashMapOnDisk()) { + // Recursively process on-disk triplets (hash partition, sidefile, matchfile) + try { + hybridMapJoinLeftover = true; + hashMapRowGetters[smallTable] = null; + continueProcess(hashPartitions[i], hybridHtContainer); + } catch (IOException e) { + e.printStackTrace(); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } catch (SerDeException e) { + e.printStackTrace(); + } } + hybridMapJoinLeftover = false; + currentSmallTable = null; } - hybridMapJoinLeftover = false; - currentSmallTable = null; } } } } + if ((this.getExecContext() != null) && (this.getExecContext().getLocalWork() != null) && (this.getExecContext().getLocalWork().getInputFileChangeSensitive()) && mapJoinTables != null) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java index c2abba2..23deff0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java @@ -19,7 +19,14 @@ package org.apache.hadoop.hive.ql.exec.persistence; -import com.esotericsoftware.kryo.Kryo; +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectOutputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -33,26 +40,20 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter; import org.apache.hadoop.hive.ql.io.HiveKey; import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.ByteStream.Output; import org.apache.hadoop.hive.serde2.SerDe; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.WriteBuffers; -import org.apache.hadoop.hive.serde2.ByteStream.Output; import org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe; import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; -import org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryStructObjectInspector; import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryFactory; import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryStruct; +import org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryStructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.Writable; -import java.io.IOException; -import java.io.InputStream; -import java.io.ObjectOutputStream; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.*; +import com.esotericsoftware.kryo.Kryo; /** * Hash table container that can have many partitions -- each partition has its own hashmap, @@ -67,16 +68,16 @@ public class HybridHashTableContainer implements MapJoinTableContainer { private static final Log LOG = LogFactory.getLog(HybridHashTableContainer.class); - private HashPartition[] hashPartitions; // an array of partitions holding the triplets + private final HashPartition[] hashPartitions; // an array of partitions holding the triplets private int totalInMemRowCount = 0; // total number of small table rows in memory - private long memoryThreshold; // the max memory limit allocated - private long tableRowSize; // row size of the small table + private final long memoryThreshold; // the max memory limit allocated + private final long tableRowSize; // row size of the small table private boolean isSpilled; // whether there's any spilled partition private int toSpillPartitionId; // the partition into which to spill the big table row; // This may change after every setMapJoinKey call private int numPartitionsSpilled; // number of spilled partitions private boolean lastPartitionInMem; // only one (last one) partition is left in memory - private int memoryCheckFrequency; // how often (# of rows apart) to check if memory is full + private final int memoryCheckFrequency; // how often (# of rows apart) to check if memory is full /** The OI used to deserialize values. We never deserialize keys. */ private LazyBinaryStructObjectInspector internalValueOi; @@ -182,6 +183,11 @@ private HybridHashTableContainer(float keyCountAdj, int threshold, float loadFac long noConditionalTaskThreshold, int memCheckFreq, long tableSize, long keyCount, long memUsage) throws SerDeException { + if (wbSize > noConditionalTaskThreshold) { + LOG.warn("adjusting hash table write buffer size to be smaller than noconditionaltasksize"); + wbSize = (int) noConditionalTaskThreshold; + } + int newKeyCount = HashMapWrapper.calculateTableSize( keyCountAdj, threshold, loadFactor, keyCount); @@ -398,7 +404,8 @@ private void spillPartition(int partitionId) throws IOException { */ private int calcNumPartitions(long dataSize, int wbSize) { if (memoryThreshold < wbSize) { - throw new RuntimeException("Available memory is less than hashtable writebuffer size!" + + throw new IllegalStateException("Available memory is less than hashtable writebuffer size!" + + " Try increasing hive.auto.convert.join.noconditionaltask.size."); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java index 2e15922..9034253 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java @@ -29,11 +29,11 @@ import org.apache.hadoop.hive.ql.exec.MapredContext; import org.apache.hadoop.hive.ql.exec.mr.ExecMapperContext; import org.apache.hadoop.hive.ql.exec.persistence.HashMapWrapper; +import org.apache.hadoop.hive.ql.exec.persistence.HybridHashTableContainer; import org.apache.hadoop.hive.ql.exec.persistence.MapJoinBytesTableContainer; import org.apache.hadoop.hive.ql.exec.persistence.MapJoinObjectSerDeContext; import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer; import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainerSerDe; -import org.apache.hadoop.hive.ql.exec.persistence.HybridHashTableContainer; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.MapJoinDesc; import org.apache.hadoop.hive.serde2.SerDeException; @@ -77,8 +77,7 @@ public void load(MapJoinTableContainer[] mapJoinTables, boolean useOptimizedTables = HiveConf.getBoolVar( hconf, HiveConf.ConfVars.HIVEMAPJOINUSEOPTIMIZEDTABLE); - boolean useHybridGraceHashJoin = HiveConf.getBoolVar( - hconf, HiveConf.ConfVars.HIVEUSEHYBRIDGRACEHASHJOIN); + boolean useHybridGraceHashJoin = desc.isHybridHashJoin(); boolean isFirstKey = true; // Disable hybrid grace hash join for n-way join diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java index 8423698..f18f841 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java @@ -244,7 +244,8 @@ private void convertJoinSMBJoin(JoinOperator joinOp, OptimizeTezProcContext cont joinOp.getConf().getBaseSrc(), joinOp).getSecond(), null, joinDesc.getExprs(), null, null, joinDesc.getOutputColumnNames(), mapJoinConversionPos, joinDesc.getConds(), - joinDesc.getFilters(), joinDesc.getNoOuterJoin(), null); + joinDesc.getFilters(), joinDesc.getNoOuterJoin(), null, HiveConf.getBoolVar( + context.conf, HiveConf.ConfVars.HIVEUSEHYBRIDGRACEHASHJOIN)); mapJoinDesc.setNullSafes(joinDesc.getNullSafes()); mapJoinDesc.setFilterMap(joinDesc.getFilterMap()); mapJoinDesc.resetOrder(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java index 4d84f0f..e63d27a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java @@ -420,7 +420,8 @@ public static MapJoinOperator convertSMBJoinToMapJoin(HiveConf hconf, smbJoinDesc.getValueTblDescs(), smbJoinDesc.getValueTblDescs(), smbJoinDesc.getOutputColumnNames(), bigTablePos, smbJoinDesc.getConds(), - smbJoinDesc.getFilters(), smbJoinDesc.isNoOuterJoin(), smbJoinDesc.getDumpFilePrefix()); + smbJoinDesc.getFilters(), smbJoinDesc.isNoOuterJoin(), smbJoinDesc.getDumpFilePrefix(), + HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEUSEHYBRIDGRACEHASHJOIN)); mapJoinDesc.setStatistics(smbJoinDesc.getStatistics()); @@ -1152,7 +1153,7 @@ public static MapJoinDesc getMapJoinDesc(HiveConf hconf, MapJoinDesc mapJoinDescriptor = new MapJoinDesc(keyExprMap, keyTableDesc, newValueExprs, valueTableDescs, valueFilteredTableDescs, outputColumnNames, mapJoinPos, joinCondns, filters, op - .getConf().getNoOuterJoin(), dumpFilePrefix); + .getConf().getNoOuterJoin(), dumpFilePrefix, false); mapJoinDescriptor.setStatistics(op.getConf().getStatistics()); mapJoinDescriptor.setTagOrder(tagOrder); mapJoinDescriptor.setNullSafes(desc.getNullSafes()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java index 15f0d70..e62dcd0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java @@ -18,6 +18,13 @@ package org.apache.hadoop.hive.ql.optimizer.physical; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.ColumnInfo; @@ -53,13 +60,6 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - /** * GenMRSkewJoinProcessor. * @@ -284,7 +284,8 @@ public static void processSkewJoin(JoinOperator joinOp, MapJoinDesc mapJoinDescriptor = new MapJoinDesc(newJoinKeys, keyTblDesc, newJoinValues, newJoinValueTblDesc, newJoinValueTblDesc,joinDescriptor .getOutputColumnNames(), i, joinDescriptor.getConds(), - joinDescriptor.getFilters(), joinDescriptor.getNoOuterJoin(), dumpFilePrefix); + joinDescriptor.getFilters(), + joinDescriptor.getNoOuterJoin(), dumpFilePrefix, false); mapJoinDescriptor.setTagOrder(tags); mapJoinDescriptor.setHandleSkewJoin(false); mapJoinDescriptor.setNullSafes(joinDescriptor.getNullSafes()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java index 52e6be8..8a01e31 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java @@ -18,7 +18,11 @@ package org.apache.hadoop.hive.ql.optimizer.physical; -import com.google.common.base.Preconditions; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -63,11 +67,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import com.google.common.base.Preconditions; /** * Copied from GenMRSkewJoinProcessor. It's used for spark task @@ -242,7 +242,8 @@ public static void processSkewJoin(JoinOperator joinOp, Task(); } @@ -90,13 +92,16 @@ public MapJoinDesc(MapJoinDesc clone) { this.parentToInput = clone.parentToInput; this.parentKeyCounts = clone.parentKeyCounts; this.parentDataSizes = clone.parentDataSizes; + this.isBucketMapJoin = clone.isBucketMapJoin; + this.isHybridHashJoin = clone.isHybridHashJoin; } public MapJoinDesc(final Map> keys, final TableDesc keyTblDesc, final Map> values, final List valueTblDescs,final List valueFilteredTblDescs, List outputColumnNames, final int posBigTable, final JoinCondDesc[] conds, - final Map> filters, boolean noOuterJoin, String dumpFilePrefix) { + final Map> filters, final boolean noOuterJoin, + final String dumpFilePrefix, final boolean isHybridHashJoin) { super(values, outputColumnNames, noOuterJoin, conds, filters, null); this.keys = keys; this.keyTblDesc = keyTblDesc; @@ -105,6 +110,7 @@ public MapJoinDesc(final Map> keys, this.posBigTable = posBigTable; this.bigTableBucketNumMapping = new LinkedHashMap(); this.dumpFilePrefix = dumpFilePrefix; + this.isHybridHashJoin = isHybridHashJoin; initRetainExprList(); } @@ -322,6 +328,15 @@ public void setBucketMapJoin(boolean isBucketMapJoin) { this.isBucketMapJoin = isBucketMapJoin; } + @Explain(displayName = "HybridHashJoin", displayOnlyOnTrue = true) + public boolean isHybridHashJoin() { + return isHybridHashJoin; + } + + public void setHybridHashJoin() { + this.isHybridHashJoin = true; + } + public void setHashTableMemoryUsage(float hashtableMemoryUsage) { this.hashtableMemoryUsage = hashtableMemoryUsage; } diff --git a/ql/src/test/results/clientpositive/tez/auto_join21.q.out b/ql/src/test/results/clientpositive/tez/auto_join21.q.out index ca60448..1bfc1d7 100644 --- a/ql/src/test/results/clientpositive/tez/auto_join21.q.out +++ b/ql/src/test/results/clientpositive/tez/auto_join21.q.out @@ -72,6 +72,7 @@ STAGE PLANS: 2 key (type: string) outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 diff --git a/ql/src/test/results/clientpositive/tez/auto_join29.q.out b/ql/src/test/results/clientpositive/tez/auto_join29.q.out index 10c8fcd..246c74f 100644 --- a/ql/src/test/results/clientpositive/tez/auto_join29.q.out +++ b/ql/src/test/results/clientpositive/tez/auto_join29.q.out @@ -72,6 +72,7 @@ STAGE PLANS: 2 key (type: string) outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 @@ -681,6 +682,7 @@ STAGE PLANS: 2 key (type: string) outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 @@ -1290,6 +1292,7 @@ STAGE PLANS: 2 key (type: string) outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 @@ -1908,6 +1911,7 @@ STAGE PLANS: 2 key (type: string) outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 @@ -2528,6 +2532,7 @@ STAGE PLANS: 2 key (type: string) outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 @@ -2637,6 +2642,7 @@ STAGE PLANS: 2 key (type: string) outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 diff --git a/ql/src/test/results/clientpositive/tez/auto_join30.q.out b/ql/src/test/results/clientpositive/tez/auto_join30.q.out index 27fe242..a5e6541 100644 --- a/ql/src/test/results/clientpositive/tez/auto_join30.q.out +++ b/ql/src/test/results/clientpositive/tez/auto_join30.q.out @@ -686,6 +686,7 @@ STAGE PLANS: 2 _col0 (type: string) outputColumnNames: _col2, _col3 Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: sum(hash(_col2,_col3)) mode: hash @@ -865,6 +866,7 @@ STAGE PLANS: 2 _col0 (type: string) outputColumnNames: _col2, _col3 Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: sum(hash(_col2,_col3)) mode: hash @@ -1044,6 +1046,7 @@ STAGE PLANS: 2 _col0 (type: string) outputColumnNames: _col2, _col3 Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: sum(hash(_col2,_col3)) mode: hash @@ -1223,6 +1226,7 @@ STAGE PLANS: 2 _col0 (type: string) outputColumnNames: _col2, _col3 Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: sum(hash(_col2,_col3)) mode: hash diff --git a/ql/src/test/results/clientpositive/tez/correlationoptimizer1.q.out b/ql/src/test/results/clientpositive/tez/correlationoptimizer1.q.out index ff8eafb..398604c 100644 --- a/ql/src/test/results/clientpositive/tez/correlationoptimizer1.q.out +++ b/ql/src/test/results/clientpositive/tez/correlationoptimizer1.q.out @@ -77,6 +77,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 @@ -221,6 +222,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 @@ -523,6 +525,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count(1) keys: _col0 (type: string) @@ -668,6 +671,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count(1) keys: _col0 (type: string) @@ -808,6 +812,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 @@ -946,6 +951,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 @@ -1090,6 +1096,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count(1) keys: _col0 (type: string) @@ -1224,6 +1231,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count(1) keys: _col0 (type: string) @@ -1361,6 +1369,7 @@ STAGE PLANS: 1 _col0 (type: string), _col1 (type: string) outputColumnNames: _col1, _col2 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col2 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 @@ -1489,6 +1498,7 @@ STAGE PLANS: 1 _col0 (type: string), _col1 (type: string) outputColumnNames: _col1, _col2 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col2 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 @@ -1626,6 +1636,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count(1) keys: _col0 (type: string) @@ -1760,6 +1771,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count(1) keys: _col0 (type: string) @@ -1900,6 +1912,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 @@ -2038,6 +2051,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 @@ -2184,6 +2198,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 @@ -2322,6 +2337,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 @@ -2469,6 +2485,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1, _col2 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col1 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1 @@ -2614,6 +2631,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1, _col2 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col1 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1 @@ -2760,6 +2778,7 @@ STAGE PLANS: 1 _col0 (type: string), _col1 (type: string) outputColumnNames: _col2 Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col2 (type: string) outputColumnNames: _col0 @@ -2904,6 +2923,7 @@ STAGE PLANS: 1 _col0 (type: string), _col1 (type: string) outputColumnNames: _col2 Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col2 (type: string) outputColumnNames: _col0 diff --git a/ql/src/test/results/clientpositive/tez/cross_join.q.out b/ql/src/test/results/clientpositive/tez/cross_join.q.out index 30564d7..d8066a5 100644 --- a/ql/src/test/results/clientpositive/tez/cross_join.q.out +++ b/ql/src/test/results/clientpositive/tez/cross_join.q.out @@ -49,6 +49,7 @@ STAGE PLANS: 1 outputColumnNames: _col0 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true File Output Operator compressed: false Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE @@ -114,6 +115,7 @@ STAGE PLANS: 1 outputColumnNames: _col0 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true File Output Operator compressed: false Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE @@ -189,6 +191,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true File Output Operator compressed: false Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE diff --git a/ql/src/test/results/clientpositive/tez/cross_product_check_1.q.out b/ql/src/test/results/clientpositive/tez/cross_product_check_1.q.out index 9274a59..e54f59b 100644 --- a/ql/src/test/results/clientpositive/tez/cross_product_check_1.q.out +++ b/ql/src/test/results/clientpositive/tez/cross_product_check_1.q.out @@ -76,6 +76,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true File Output Operator compressed: false Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE @@ -154,6 +155,7 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col0, _col1, _col5, _col6 Statistics: Num rows: 5 Data size: 57 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Reduce Output Operator sort order: Statistics: Num rows: 5 Data size: 57 Basic stats: COMPLETE Column stats: NONE @@ -168,6 +170,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 @@ -255,6 +258,7 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col0 Statistics: Num rows: 5 Data size: 57 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator keys: _col0 (type: string) mode: hash @@ -286,6 +290,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col5 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string) outputColumnNames: _col0, _col1, _col2 @@ -359,6 +364,7 @@ STAGE PLANS: 1 outputColumnNames: _col0 Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator keys: _col0 (type: string) mode: hash @@ -390,6 +396,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col5 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string) outputColumnNames: _col0, _col1, _col2 @@ -497,6 +504,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true File Output Operator compressed: false Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE @@ -514,6 +522,7 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col0 Statistics: Num rows: 5 Data size: 57 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator keys: _col0 (type: string) mode: hash diff --git a/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out index 30847f9..ca882a2 100644 --- a/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out +++ b/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out @@ -251,6 +251,7 @@ STAGE PLANS: 0 ds (type: string) 1 ds (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -350,6 +351,7 @@ STAGE PLANS: 0 ds (type: string) 1 ds (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -512,6 +514,7 @@ STAGE PLANS: 1 ds (type: string) outputColumnNames: _col3 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Reduce Output Operator key expressions: _col3 (type: string) sort order: + @@ -526,6 +529,7 @@ STAGE PLANS: 0 _col3 (type: string) 1 hr (type: string) Statistics: Num rows: 2420 Data size: 25709 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -648,6 +652,7 @@ STAGE PLANS: 1 ds (type: string) outputColumnNames: _col3 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Reduce Output Operator key expressions: _col3 (type: string) sort order: + @@ -662,6 +667,7 @@ STAGE PLANS: 0 _col3 (type: string) 1 hr (type: string) Statistics: Num rows: 2420 Data size: 25709 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -807,6 +813,7 @@ STAGE PLANS: 0 ds (type: string), hr (type: string) 1 ds (type: string), hr (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -906,6 +913,7 @@ STAGE PLANS: 0 ds (type: string), hr (type: string) 1 ds (type: string), hr (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -1033,6 +1041,7 @@ STAGE PLANS: 0 ds (type: string) 1 ds (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -1132,6 +1141,7 @@ STAGE PLANS: 0 ds (type: string) 1 ds (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -1260,6 +1270,7 @@ STAGE PLANS: 0 UDFToDouble(hr) (type: double) 1 UDFToDouble(UDFToInteger((hr / 2))) (type: double) Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -1377,6 +1388,7 @@ STAGE PLANS: 0 (hr * 2) (type: double) 1 hr (type: double) Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -1479,6 +1491,7 @@ STAGE PLANS: 0 UDFToDouble(hr) (type: double) 1 UDFToDouble(UDFToInteger((hr / 2))) (type: double) Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -1581,6 +1594,7 @@ STAGE PLANS: 0 (hr * 2) (type: double) 1 hr (type: double) Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -1711,6 +1725,7 @@ STAGE PLANS: 0 UDFToString((hr * 2)) (type: string) 1 UDFToString(hr) (type: string) Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -1833,6 +1848,7 @@ STAGE PLANS: 0 1 Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -1953,6 +1969,7 @@ STAGE PLANS: 1 outputColumnNames: _col2, _col3, _col7, _col9 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Filter Operator predicate: ((_col2 = _col7) or (_col3 = _col9)) (type: boolean) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE @@ -2090,6 +2107,7 @@ STAGE PLANS: 1 ds (type: string), hr (type: string) outputColumnNames: _col2, _col3, _col7, _col9 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Filter Operator predicate: ((_col2 = _col7) and (_col3 = _col9)) (type: boolean) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE @@ -2193,6 +2211,7 @@ STAGE PLANS: 1 ds (type: string) outputColumnNames: _col8 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Filter Operator predicate: (_col8 = '2008-04-08') (type: boolean) Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE @@ -2292,6 +2311,7 @@ STAGE PLANS: 0 ds (type: string) 1 ds (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -2371,6 +2391,7 @@ STAGE PLANS: 1 ds (type: string) outputColumnNames: _col8 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Filter Operator predicate: (_col8 = '2008-04-08') (type: boolean) Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE @@ -2505,6 +2526,7 @@ STAGE PLANS: 0 ds (type: string) 1 ds (type: string) Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Reduce Output Operator key expressions: '11' (type: string) sort order: + @@ -2519,6 +2541,7 @@ STAGE PLANS: 0 '11' (type: string) 1 '11' (type: string) Statistics: Num rows: 1210 Data size: 12854 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -2625,6 +2648,7 @@ STAGE PLANS: 0 ds (type: string) 1 ds (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + HybridHashJoin: true Reduce Output Operator key expressions: '13' (type: string) sort order: + @@ -2639,6 +2663,7 @@ STAGE PLANS: 0 '13' (type: string) 1 '13' (type: string) Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -2764,6 +2789,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -2948,6 +2974,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator keys: _col0 (type: string) mode: hash @@ -3210,6 +3237,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true File Output Operator compressed: false Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE @@ -4389,6 +4417,7 @@ STAGE PLANS: 1 ds (type: string) outputColumnNames: _col8 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Filter Operator predicate: (_col8 = '2008-04-08') (type: boolean) Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE diff --git a/ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out b/ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out index 06f1ca4..fa0a939 100644 --- a/ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out +++ b/ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out @@ -360,6 +360,7 @@ STAGE PLANS: outputColumnNames: _col0, _col7 Position of Big Table: 0 Statistics: Num rows: 14 Data size: 119 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Reduce Output Operator key expressions: _col7 (type: string) sort order: + @@ -380,6 +381,7 @@ STAGE PLANS: outputColumnNames: _col0, _col13 Position of Big Table: 0 Statistics: Num rows: 27 Data size: 232 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col0 (type: int), _col13 (type: string) outputColumnNames: _col0, _col1 diff --git a/ql/src/test/results/clientpositive/tez/join0.q.java1.7.out b/ql/src/test/results/clientpositive/tez/join0.q.java1.7.out index cceefb2..c5bbebb 100644 --- a/ql/src/test/results/clientpositive/tez/join0.q.java1.7.out +++ b/ql/src/test/results/clientpositive/tez/join0.q.java1.7.out @@ -75,6 +75,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) sort order: ++++ diff --git a/ql/src/test/results/clientpositive/tez/join1.q.out b/ql/src/test/results/clientpositive/tez/join1.q.out index c5bd100..12063c9 100644 --- a/ql/src/test/results/clientpositive/tez/join1.q.out +++ b/ql/src/test/results/clientpositive/tez/join1.q.out @@ -76,6 +76,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1, _col2 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: UDFToInteger(_col2) (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 diff --git a/ql/src/test/results/clientpositive/tez/join_nullsafe.q.out b/ql/src/test/results/clientpositive/tez/join_nullsafe.q.out index 787f1f5..b4b4812 100644 --- a/ql/src/test/results/clientpositive/tez/join_nullsafe.q.out +++ b/ql/src/test/results/clientpositive/tez/join_nullsafe.q.out @@ -68,6 +68,7 @@ STAGE PLANS: nullSafes: [true] outputColumnNames: _col0, _col1, _col5, _col6 Statistics: Num rows: 3 Data size: 28 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 @@ -174,6 +175,7 @@ STAGE PLANS: 2 key (type: int) outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 Statistics: Num rows: 4 Data size: 37 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 @@ -263,6 +265,7 @@ STAGE PLANS: nullSafes: [true] outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 @@ -385,6 +388,7 @@ STAGE PLANS: nullSafes: [true, false] outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 Statistics: Num rows: 4 Data size: 37 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 @@ -471,6 +475,7 @@ STAGE PLANS: nullSafes: [true, true] outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 @@ -1606,6 +1611,7 @@ STAGE PLANS: nullSafes: [true] outputColumnNames: _col1, _col5 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: null (type: void), _col1 (type: int), _col5 (type: int), null (type: void) outputColumnNames: _col0, _col1, _col2, _col3 diff --git a/ql/src/test/results/clientpositive/tez/limit_pushdown.q.out b/ql/src/test/results/clientpositive/tez/limit_pushdown.q.out index 952d7ff..97a60ba 100644 --- a/ql/src/test/results/clientpositive/tez/limit_pushdown.q.out +++ b/ql/src/test/results/clientpositive/tez/limit_pushdown.q.out @@ -883,6 +883,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col0 (type: string), _col3 (type: bigint), _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 diff --git a/ql/src/test/results/clientpositive/tez/metadataonly1.q.out b/ql/src/test/results/clientpositive/tez/metadataonly1.q.out index a595c4c..09902c9 100644 --- a/ql/src/test/results/clientpositive/tez/metadataonly1.q.out +++ b/ql/src/test/results/clientpositive/tez/metadataonly1.q.out @@ -826,6 +826,7 @@ STAGE PLANS: 1 _col0 (type: string) Position of Big Table: 0 Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash diff --git a/ql/src/test/results/clientpositive/tez/mrr.q.out b/ql/src/test/results/clientpositive/tez/mrr.q.out index e7a3b3c..e1545ea 100644 --- a/ql/src/test/results/clientpositive/tez/mrr.q.out +++ b/ql/src/test/results/clientpositive/tez/mrr.q.out @@ -451,6 +451,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count(DISTINCT _col1) keys: _col0 (type: string), _col1 (type: string) @@ -1417,6 +1418,7 @@ STAGE PLANS: 2 _col0 (type: string) outputColumnNames: _col0, _col1, _col3, _col4, _col5 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col0 (type: string), _col3 (type: bigint), _col1 (type: bigint), _col4 (type: string), _col5 (type: bigint) outputColumnNames: _col0, _col1, _col3, _col4, _col5 diff --git a/ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out b/ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out index 104654a..1b24a10 100644 --- a/ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out +++ b/ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out @@ -519,6 +519,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Position of Big Table: 0 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + HybridHashJoin: true File Output Operator compressed: false GlobalTableId: 0 @@ -1370,6 +1371,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Position of Big Table: 0 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + HybridHashJoin: true File Output Operator compressed: false GlobalTableId: 0 @@ -1667,6 +1669,7 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col0 Position of Big Table: 0 + HybridHashJoin: true File Output Operator compressed: false GlobalTableId: 0 @@ -1901,6 +1904,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col5, _col6 Position of Big Table: 0 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 diff --git a/ql/src/test/results/clientpositive/tez/ptf.q.out b/ql/src/test/results/clientpositive/tez/ptf.q.out index 2a92d77..26e5dd1 100644 --- a/ql/src/test/results/clientpositive/tez/ptf.q.out +++ b/ql/src/test/results/clientpositive/tez/ptf.q.out @@ -245,6 +245,7 @@ STAGE PLANS: 1 p_partkey (type: int) outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ @@ -1129,6 +1130,7 @@ STAGE PLANS: 1 p_partkey (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true File Output Operator compressed: false Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE @@ -1249,6 +1251,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col12 (type: int), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: string), _col17 (type: int), _col18 (type: string), _col19 (type: double), _col20 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 @@ -2388,6 +2391,7 @@ STAGE PLANS: 1 p_partkey (type: int) outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ diff --git a/ql/src/test/results/clientpositive/tez/ptf_streaming.q.out b/ql/src/test/results/clientpositive/tez/ptf_streaming.q.out index 9be0819..b69e5c6 100644 --- a/ql/src/test/results/clientpositive/tez/ptf_streaming.q.out +++ b/ql/src/test/results/clientpositive/tez/ptf_streaming.q.out @@ -245,6 +245,7 @@ STAGE PLANS: 1 p_partkey (type: int) outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ @@ -457,6 +458,7 @@ STAGE PLANS: 1 p_partkey (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true File Output Operator compressed: false Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE @@ -1679,6 +1681,7 @@ STAGE PLANS: 1 p_partkey (type: int) outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ diff --git a/ql/src/test/results/clientpositive/tez/skewjoin.q.out b/ql/src/test/results/clientpositive/tez/skewjoin.q.out index ab2ce24..bda1ebb 100644 --- a/ql/src/test/results/clientpositive/tez/skewjoin.q.out +++ b/ql/src/test/results/clientpositive/tez/skewjoin.q.out @@ -136,6 +136,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1, _col2 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: UDFToInteger(_col2) (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 @@ -279,6 +280,7 @@ STAGE PLANS: 3 key (type: string) outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11, _col15, _col16 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string), _col15 (type: string), _col16 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 @@ -411,6 +413,7 @@ STAGE PLANS: 3 key (type: string) outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11, _col15, _col16 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string), _col15 (type: string), _col16 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 @@ -504,6 +507,7 @@ STAGE PLANS: 1 (key + 1) (type: double) outputColumnNames: _col0, _col1, _col5 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: sum(hash(_col0)), sum(hash(_col1)), sum(hash(_col5)) mode: hash @@ -616,6 +620,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: hash(_col0) (type: int), hash(_col1) (type: int) outputColumnNames: _col0, _col1 @@ -740,6 +745,7 @@ STAGE PLANS: 1 _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + 1.0) (type: double) outputColumnNames: _col2, _col3 Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: hash(_col2) (type: int), hash(_col3) (type: int) outputColumnNames: _col0, _col1 @@ -890,6 +896,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1, _col3 Statistics: Num rows: 66 Data size: 706 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: hash(_col1) (type: int), hash(_col3) (type: int) outputColumnNames: _col0, _col1 @@ -927,6 +934,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1, _col2 Statistics: Num rows: 60 Data size: 642 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col2 (type: string), _col1 (type: string) outputColumnNames: _col0, _col2 @@ -1019,6 +1027,7 @@ STAGE PLANS: 1 UDFToDouble(key) (type: double) outputColumnNames: _col0, _col6 Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: sum(hash(_col0)), sum(hash(_col6)) mode: hash diff --git a/ql/src/test/results/clientpositive/tez/subquery_exists.q.out b/ql/src/test/results/clientpositive/tez/subquery_exists.q.out index f4e8df1..ebe5f91 100644 --- a/ql/src/test/results/clientpositive/tez/subquery_exists.q.out +++ b/ql/src/test/results/clientpositive/tez/subquery_exists.q.out @@ -82,6 +82,7 @@ STAGE PLANS: 1 _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true File Output Operator compressed: false Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE diff --git a/ql/src/test/results/clientpositive/tez/subquery_in.q.out b/ql/src/test/results/clientpositive/tez/subquery_in.q.out index 5937903..402d529 100644 --- a/ql/src/test/results/clientpositive/tez/subquery_in.q.out +++ b/ql/src/test/results/clientpositive/tez/subquery_in.q.out @@ -75,6 +75,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true File Output Operator compressed: false Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE @@ -192,6 +193,7 @@ STAGE PLANS: 1 _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true File Output Operator compressed: false Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE @@ -308,6 +310,7 @@ STAGE PLANS: 1 _col0 (type: double) outputColumnNames: _col0, _col1 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true File Output Operator compressed: false Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE @@ -476,6 +479,7 @@ STAGE PLANS: 1 _col0 (type: int), _col1 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 7 Data size: 931 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col1 (type: string), _col0 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2 @@ -670,6 +674,7 @@ STAGE PLANS: 1 _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true File Output Operator compressed: false Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE @@ -853,6 +858,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1, _col2 Statistics: Num rows: 27 Data size: 3298 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Reduce Output Operator key expressions: _col1 (type: int) sort order: + @@ -869,6 +875,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1, _col2 Statistics: Num rows: 29 Data size: 3627 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col1 (type: int), _col2 (type: int) outputColumnNames: _col0, _col1 diff --git a/ql/src/test/results/clientpositive/tez/tez_join_hash.q.out b/ql/src/test/results/clientpositive/tez/tez_join_hash.q.out index bf60681..ed90d13 100644 --- a/ql/src/test/results/clientpositive/tez/tez_join_hash.q.out +++ b/ql/src/test/results/clientpositive/tez/tez_join_hash.q.out @@ -71,6 +71,7 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col0, _col5 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Filter Operator predicate: (_col0 = _col5) (type: boolean) Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE @@ -237,6 +238,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col1, _col2 + HybridHashJoin: true Select Operator expressions: _col2 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 diff --git a/ql/src/test/results/clientpositive/tez/tez_join_tests.q.out b/ql/src/test/results/clientpositive/tez/tez_join_tests.q.out index 88523dc..8b443b7 100644 --- a/ql/src/test/results/clientpositive/tez/tez_join_tests.q.out +++ b/ql/src/test/results/clientpositive/tez/tez_join_tests.q.out @@ -76,6 +76,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Reduce Output Operator key expressions: _col0 (type: string) sort order: + @@ -103,6 +104,7 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Reduce Output Operator key expressions: _col0 (type: string) sort order: + diff --git a/ql/src/test/results/clientpositive/tez/tez_joins_explain.q.out b/ql/src/test/results/clientpositive/tez/tez_joins_explain.q.out index 668f75b..679d428 100644 --- a/ql/src/test/results/clientpositive/tez/tez_joins_explain.q.out +++ b/ql/src/test/results/clientpositive/tez/tez_joins_explain.q.out @@ -76,6 +76,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Reduce Output Operator key expressions: _col0 (type: string) sort order: + @@ -103,6 +104,7 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Reduce Output Operator key expressions: _col0 (type: string) sort order: + diff --git a/ql/src/test/results/clientpositive/tez/tez_smb_main.q.out b/ql/src/test/results/clientpositive/tez/tez_smb_main.q.out index f0bcea6..94130c2 100644 --- a/ql/src/test/results/clientpositive/tez/tez_smb_main.q.out +++ b/ql/src/test/results/clientpositive/tez/tez_smb_main.q.out @@ -61,6 +61,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true File Output Operator compressed: false Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE @@ -691,6 +692,7 @@ STAGE PLANS: 0 value (type: string) 1 value (type: string) Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -821,6 +823,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 key (type: int) Statistics: Num rows: 279 Data size: 2963 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash diff --git a/ql/src/test/results/clientpositive/tez/tez_union.q.out b/ql/src/test/results/clientpositive/tez/tez_union.q.out index 2472c5e..fa0b80c 100644 --- a/ql/src/test/results/clientpositive/tez/tez_union.q.out +++ b/ql/src/test/results/clientpositive/tez/tez_union.q.out @@ -222,6 +222,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash diff --git a/ql/src/test/results/clientpositive/tez/tez_union_group_by.q.out b/ql/src/test/results/clientpositive/tez/tez_union_group_by.q.out index 844ba23..99cac1d 100644 --- a/ql/src/test/results/clientpositive/tez/tez_union_group_by.q.out +++ b/ql/src/test/results/clientpositive/tez/tez_union_group_by.q.out @@ -202,6 +202,7 @@ STAGE PLANS: 1 _col0 (type: bigint) outputColumnNames: _col0, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col2 (type: bigint), _col0 (type: bigint) outputColumnNames: _col0, _col1 @@ -224,6 +225,7 @@ STAGE PLANS: nullSafes: [false, true] outputColumnNames: _col0 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + HybridHashJoin: true Group By Operator keys: _col0 (type: bigint) mode: hash diff --git a/ql/src/test/results/clientpositive/tez/unionDistinct_1.q.out b/ql/src/test/results/clientpositive/tez/unionDistinct_1.q.out index 49fbc1b..0ef2843 100644 --- a/ql/src/test/results/clientpositive/tez/unionDistinct_1.q.out +++ b/ql/src/test/results/clientpositive/tez/unionDistinct_1.q.out @@ -5188,6 +5188,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + HybridHashJoin: true File Output Operator compressed: false Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE @@ -14075,6 +14076,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1 + HybridHashJoin: true Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash diff --git a/ql/src/test/results/clientpositive/tez/vectorized_bucketmapjoin1.q.out b/ql/src/test/results/clientpositive/tez/vectorized_bucketmapjoin1.q.out index 56ef3e2..bad1c7b 100644 --- a/ql/src/test/results/clientpositive/tez/vectorized_bucketmapjoin1.q.out +++ b/ql/src/test/results/clientpositive/tez/vectorized_bucketmapjoin1.q.out @@ -151,6 +151,7 @@ STAGE PLANS: 1 key (type: int) outputColumnNames: _col0, _col1, _col5, _col6 Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 @@ -239,6 +240,7 @@ STAGE PLANS: 1 key (type: int) outputColumnNames: _col0, _col1, _col5, _col6 Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 @@ -337,6 +339,7 @@ STAGE PLANS: 1 key (type: int) outputColumnNames: _col0, _col1, _col5, _col6 Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 diff --git a/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out index 81e2855..54b13e3 100644 --- a/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out +++ b/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out @@ -253,6 +253,7 @@ STAGE PLANS: 0 ds (type: string) 1 ds (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -354,6 +355,7 @@ STAGE PLANS: 0 ds (type: string) 1 ds (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -518,6 +520,7 @@ STAGE PLANS: 1 ds (type: string) outputColumnNames: _col3 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Reduce Output Operator key expressions: _col3 (type: string) sort order: + @@ -532,6 +535,7 @@ STAGE PLANS: 0 _col3 (type: string) 1 hr (type: string) Statistics: Num rows: 2420 Data size: 25709 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -656,6 +660,7 @@ STAGE PLANS: 1 ds (type: string) outputColumnNames: _col3 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Reduce Output Operator key expressions: _col3 (type: string) sort order: + @@ -670,6 +675,7 @@ STAGE PLANS: 0 _col3 (type: string) 1 hr (type: string) Statistics: Num rows: 2420 Data size: 25709 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -816,6 +822,7 @@ STAGE PLANS: 0 ds (type: string), hr (type: string) 1 ds (type: string), hr (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -916,6 +923,7 @@ STAGE PLANS: 0 ds (type: string), hr (type: string) 1 ds (type: string), hr (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -1045,6 +1053,7 @@ STAGE PLANS: 0 ds (type: string) 1 ds (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -1146,6 +1155,7 @@ STAGE PLANS: 0 ds (type: string) 1 ds (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -1275,6 +1285,7 @@ STAGE PLANS: 0 UDFToDouble(hr) (type: double) 1 UDFToDouble(UDFToInteger((hr / 2))) (type: double) Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -1393,6 +1404,7 @@ STAGE PLANS: 0 (hr * 2) (type: double) 1 hr (type: double) Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -1496,6 +1508,7 @@ STAGE PLANS: 0 UDFToDouble(hr) (type: double) 1 UDFToDouble(UDFToInteger((hr / 2))) (type: double) Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -1599,6 +1612,7 @@ STAGE PLANS: 0 (hr * 2) (type: double) 1 hr (type: double) Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -1730,6 +1744,7 @@ STAGE PLANS: 0 UDFToString((hr * 2)) (type: string) 1 UDFToString(hr) (type: string) Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -1853,6 +1868,7 @@ STAGE PLANS: 0 1 Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -1975,6 +1991,7 @@ STAGE PLANS: 1 outputColumnNames: _col2, _col3, _col7, _col9 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Filter Operator predicate: ((_col2 = _col7) or (_col3 = _col9)) (type: boolean) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE @@ -2113,6 +2130,7 @@ STAGE PLANS: 1 ds (type: string), hr (type: string) outputColumnNames: _col2, _col3, _col7, _col9 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Filter Operator predicate: ((_col2 = _col7) and (_col3 = _col9)) (type: boolean) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE @@ -2218,6 +2236,7 @@ STAGE PLANS: 1 ds (type: string) outputColumnNames: _col8 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Filter Operator predicate: (_col8 = '2008-04-08') (type: boolean) Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE @@ -2319,6 +2338,7 @@ STAGE PLANS: 0 ds (type: string) 1 ds (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -2400,6 +2420,7 @@ STAGE PLANS: 1 ds (type: string) outputColumnNames: _col8 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Filter Operator predicate: (_col8 = '2008-04-08') (type: boolean) Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE @@ -2536,6 +2557,7 @@ STAGE PLANS: 0 ds (type: string) 1 ds (type: string) Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Reduce Output Operator key expressions: '11' (type: string) sort order: + @@ -2550,6 +2572,7 @@ STAGE PLANS: 0 '11' (type: string) 1 '11' (type: string) Statistics: Num rows: 1210 Data size: 12854 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -2658,6 +2681,7 @@ STAGE PLANS: 0 ds (type: string) 1 ds (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + HybridHashJoin: true Reduce Output Operator key expressions: '13' (type: string) sort order: + @@ -2672,6 +2696,7 @@ STAGE PLANS: 0 '13' (type: string) 1 '13' (type: string) Statistics: Num rows: 1 Data size: 189 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -2798,6 +2823,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator aggregations: count() mode: hash @@ -2983,6 +3009,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Group By Operator keys: _col0 (type: string) mode: hash @@ -3247,6 +3274,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true File Output Operator compressed: false Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE @@ -4443,6 +4471,7 @@ STAGE PLANS: 1 ds (type: string) outputColumnNames: _col8 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Filter Operator predicate: (_col8 = '2008-04-08') (type: boolean) Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE diff --git a/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out b/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out index 7b23b63..5108b2b 100644 --- a/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out +++ b/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out @@ -710,6 +710,7 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Position of Big Table: 0 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ @@ -2350,6 +2351,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Position of Big Table: 0 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true File Output Operator compressed: false GlobalTableId: 0 @@ -2638,6 +2640,7 @@ STAGE PLANS: outputColumnNames: _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20 Position of Big Table: 0 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col12 (type: int), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: string), _col17 (type: int), _col18 (type: string), _col19 (type: double), _col20 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 @@ -4837,6 +4840,7 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Position of Big Table: 0 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ diff --git a/ql/src/test/results/clientpositive/tez/vectorized_shufflejoin.q.out b/ql/src/test/results/clientpositive/tez/vectorized_shufflejoin.q.out index 408d46e..3f33eb7 100644 --- a/ql/src/test/results/clientpositive/tez/vectorized_shufflejoin.q.out +++ b/ql/src/test/results/clientpositive/tez/vectorized_shufflejoin.q.out @@ -64,6 +64,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 6758 Data size: 1453080 Basic stats: COMPLETE Column stats: NONE + HybridHashJoin: true Select Operator expressions: _col0 (type: int), _col1 (type: int), (_col0 + _col1) (type: int) outputColumnNames: _col0, _col1, _col2