diff --git ql/pom.xml ql/pom.xml
index 8c3e55eaf4..7d8f9398c6 100644
--- ql/pom.xml
+++ ql/pom.xml
@@ -770,6 +770,12 @@
${powermock.version}
test
+
+ com.google.guava
+ guava-testlib
+ ${guava.version}
+ test
+
diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
index da1dd426c9..b0b4ec8610 100644
--- ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
@@ -54,6 +54,7 @@
import org.apache.hadoop.hive.ql.exec.persistence.MatchTracker;
import org.apache.hadoop.hive.ql.exec.persistence.ObjectContainer;
import org.apache.hadoop.hive.ql.exec.persistence.UnwrapRowContainer;
+import org.apache.hadoop.hive.ql.exec.spark.SmallTableCache;
import org.apache.hadoop.hive.ql.exec.spark.SparkUtilities;
import org.apache.hadoop.hive.ql.exec.tez.LlapObjectCache;
import org.apache.hadoop.hive.ql.exec.tez.LlapObjectSubCache;
@@ -738,6 +739,21 @@ protected void generateFullOuterSmallTableNoMatches(byte smallTablePos,
@Override
public void closeOp(boolean abort) throws HiveException {
+ // Call the small table cache cache method, this way when a task finishes, we still keep the small table around
+ // for at least 30 seconds, which gives any tasks scheduled in the future a chance to re-use the small table.
+ if (HiveConf.getVar(hconf, ConfVars.HIVE_EXECUTION_ENGINE).equals("spark") &&
+ SparkUtilities.isDedicatedCluster(hconf)) {
+
+ for (byte pos = 0; pos < mapJoinTables.length; pos++) {
+ if (pos != conf.getPosBigTable()) {
+ MapJoinTableContainer container = mapJoinTables[pos];
+ if (container != null && container.getKey() != null) {
+ SmallTableCache.cache(container.getKey(), container);
+ }
+ }
+ }
+ }
+
if (isFullOuterMapJoin) {
// FULL OUTER MapJoin: After matching the Big Table row keys against the Small Table, we now
diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/AbstractMapJoinTableContainer.java ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/AbstractMapJoinTableContainer.java
index 9e65fd98d6..ee12162e4f 100644
--- ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/AbstractMapJoinTableContainer.java
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/AbstractMapJoinTableContainer.java
@@ -28,6 +28,8 @@
protected static final String THESHOLD_NAME = "threshold";
protected static final String LOAD_NAME = "load";
+ private String key;
+
/** Creates metadata for implementation classes' ctors from threshold and load factor. */
protected static Map createConstructorMetaData(int threshold, float loadFactor) {
Map metaData = new HashMap();
@@ -48,4 +50,14 @@ protected AbstractMapJoinTableContainer(Map metaData) {
protected void putMetaData(String key, String value) {
metaData.put(key, value);
}
+
+ @Override
+ public void setKey(String key) {
+ this.key = key;
+ }
+
+ @Override
+ public String getKey() {
+ return key;
+ }
}
diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java
index 54377428ea..545a729652 100644
--- ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java
@@ -121,6 +121,7 @@
private final List