diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java index 2d76848413..7a7c8a5aa5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java @@ -341,6 +341,8 @@ protected void initializeOp(Configuration hconf) throws HiveException { forwardCache = new Object[totalSz]; aliasFilterTags = new short[numAliases]; Arrays.fill(aliasFilterTags, (byte)0xff); + aliasFilterTagsNext = new short[numAliases]; + Arrays.fill(aliasFilterTagsNext, (byte) 0xff); filterTags = new short[numAliases]; skipVectors = new boolean[numAliases][]; @@ -478,6 +480,7 @@ protected long getNextSize(long sz) { * 100, 30 : N, N */ protected transient short[] aliasFilterTags; + protected transient short[] aliasFilterTagsNext; // all evaluation should be processed here for valid aliasFilterTags // @@ -491,11 +494,21 @@ protected long getNextSize(long sz) { short filterTag = JoinUtil.isFiltered(row, joinFilters[alias], joinFilterObjectInspectors[alias], filterMaps[alias]); nr.add(new ShortWritable(filterTag)); - aliasFilterTags[alias] &= filterTag; } return nr; } + protected void addToAliasFilterTags(byte alias, List object, boolean isNextGroup) { + boolean hasFilter = hasFilter(alias); + if (hasFilter) { + if (isNextGroup) { + aliasFilterTagsNext[alias] &= ((ShortWritable) (object.get(object.size() - 1))).get(); + } else { + aliasFilterTags[alias] &= ((ShortWritable) (object.get(object.size() - 1))).get(); + } + } + } + // fill forwardCache with skipvector // returns whether a record was forwarded private boolean createForwardJoinObject(boolean[] skip) throws HiveException { @@ -961,7 +974,8 @@ protected void checkAndGenObject() throws HiveException { genJoinObject(); } } - Arrays.fill(aliasFilterTags, (byte)0xff); + System.arraycopy(aliasFilterTagsNext, 0, aliasFilterTags, 0, aliasFilterTagsNext.length); + Arrays.fill(aliasFilterTagsNext, (byte) 0xff); } protected void reportProgress() { diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java index 581577e52b..0d9dc46bc1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java @@ -46,7 +46,6 @@ import org.apache.hadoop.hive.ql.plan.api.OperatorType; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; -import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.io.WritableComparator; @@ -242,6 +241,7 @@ public void process(Object row, int tag) throws HiveException { //have we reached a new key group? boolean nextKeyGroup = processKey(alias, key); + addToAliasFilterTags(alias, value, nextKeyGroup); if (nextKeyGroup) { //assert this.nextGroupStorage[alias].size() == 0; this.nextGroupStorage[alias].addRow(value); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java index 451ba1fbbe..9661bc9911 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java @@ -83,6 +83,7 @@ public void process(Object row, int tag) throws HiveException { alias = (byte) tag; List nr = getFilteredValue(alias, row); + addToAliasFilterTags(alias, nr, false); if (handleSkewJoin) { skewJoinKeyContext.handleSkew(tag); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java index 07b1fba8c6..489d09f348 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java @@ -582,6 +582,7 @@ public void process(Object row, int tag) throws HiveException { } if (joinNeeded) { List value = getFilteredValue(alias, row); + addToAliasFilterTags(alias, value, false); // Add the value to the ArrayList storage[alias].addRow(value); // generate the output records diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java index c09bf53299..55d6a6a672 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java @@ -288,6 +288,7 @@ public void process(Object row, int tag) throws HiveException { //have we reached a new key group? boolean nextKeyGroup = processKey(alias, key); + addToAliasFilterTags(alias, value, nextKeyGroup); if (nextKeyGroup) { //assert this.nextGroupStorage[alias].size() == 0; this.nextGroupStorage[alias].addRow(value); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinBaseOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinBaseOperator.java index e80a3e20e3..b39f80237c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinBaseOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinBaseOperator.java @@ -20,11 +20,9 @@ import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.Future; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java index bef1a79b53..b0a51380ef 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java @@ -37,7 +37,6 @@ import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.SMBJoinDesc; import org.apache.hadoop.hive.ql.plan.VectorDesc; -import org.apache.hadoop.hive.ql.plan.VectorMapJoinDesc; import org.apache.hadoop.hive.ql.plan.VectorSMBJoinDesc; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; diff --git ql/src/test/queries/clientpositive/vector_full_outer_join2.q ql/src/test/queries/clientpositive/vector_full_outer_join2.q new file mode 100644 index 0000000000..fefd0e0e02 --- /dev/null +++ ql/src/test/queries/clientpositive/vector_full_outer_join2.q @@ -0,0 +1,28 @@ +create table t_letters (z char(12), x int, y int); + +insert into t_letters values +('one', 1, 50), +('two', 2, 30), +('three', 3, 30), +('four', 4, 60), +('five', 5, 70), +('six', 6, 80); + +create table t_roman (z char(12), x int, y int); + +insert into t_roman values +('I', 1, 50), +('II', 2, 30), +('III', 3, 30), +('IV', 4, 60), +('V', 5, 70), +('VI', 6, 80); + + +select x1.`z`, x1.`x`, x1.`y`, + x2.`z`, x2.`x`, x2.`y` +from t_letters x1 full outer join t_roman x2 on (x1.`x` > 3) and (x2.`x` < 4) and (x1.`x` = x2.`x`); + +select x1.`z`, x1.`x`, x1.`y`, + x2.`z`, x2.`x`, x2.`y` +from t_letters x1 full outer join t_roman x2 on (x1.`x` > 2) and (x2.`x` < 4) and (x1.`x` = x2.`x`); \ No newline at end of file diff --git ql/src/test/results/clientpositive/llap/vector_full_outer_join2.q.out ql/src/test/results/clientpositive/llap/vector_full_outer_join2.q.out new file mode 100644 index 0000000000..b764fe5d36 --- /dev/null +++ ql/src/test/results/clientpositive/llap/vector_full_outer_join2.q.out @@ -0,0 +1,113 @@ +PREHOOK: query: create table t_letters (z char(12), x int, y int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t_letters +POSTHOOK: query: create table t_letters (z char(12), x int, y int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t_letters +PREHOOK: query: insert into t_letters values +('one', 1, 50), +('two', 2, 30), +('three', 3, 30), +('four', 4, 60), +('five', 5, 70), +('six', 6, 80) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@t_letters +POSTHOOK: query: insert into t_letters values +('one', 1, 50), +('two', 2, 30), +('three', 3, 30), +('four', 4, 60), +('five', 5, 70), +('six', 6, 80) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@t_letters +POSTHOOK: Lineage: t_letters.x SCRIPT [] +POSTHOOK: Lineage: t_letters.y SCRIPT [] +POSTHOOK: Lineage: t_letters.z SCRIPT [] +PREHOOK: query: create table t_roman (z char(12), x int, y int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t_roman +POSTHOOK: query: create table t_roman (z char(12), x int, y int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t_roman +PREHOOK: query: insert into t_roman values +('I', 1, 50), +('II', 2, 30), +('III', 3, 30), +('IV', 4, 60), +('V', 5, 70), +('VI', 6, 80) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@t_roman +POSTHOOK: query: insert into t_roman values +('I', 1, 50), +('II', 2, 30), +('III', 3, 30), +('IV', 4, 60), +('V', 5, 70), +('VI', 6, 80) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@t_roman +POSTHOOK: Lineage: t_roman.x SCRIPT [] +POSTHOOK: Lineage: t_roman.y SCRIPT [] +POSTHOOK: Lineage: t_roman.z SCRIPT [] +PREHOOK: query: select x1.`z`, x1.`x`, x1.`y`, + x2.`z`, x2.`x`, x2.`y` +from t_letters x1 full outer join t_roman x2 on (x1.`x` > 3) and (x2.`x` < 4) and (x1.`x` = x2.`x`) +PREHOOK: type: QUERY +PREHOOK: Input: default@t_letters +PREHOOK: Input: default@t_roman +#### A masked pattern was here #### +POSTHOOK: query: select x1.`z`, x1.`x`, x1.`y`, + x2.`z`, x2.`x`, x2.`y` +from t_letters x1 full outer join t_roman x2 on (x1.`x` > 3) and (x2.`x` < 4) and (x1.`x` = x2.`x`) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t_letters +POSTHOOK: Input: default@t_roman +#### A masked pattern was here #### +one 1 50 NULL NULL NULL +NULL NULL NULL I 1 50 +two 2 30 NULL NULL NULL +NULL NULL NULL II 2 30 +three 3 30 NULL NULL NULL +NULL NULL NULL III 3 30 +four 4 60 NULL NULL NULL +NULL NULL NULL IV 4 60 +five 5 70 NULL NULL NULL +NULL NULL NULL V 5 70 +six 6 80 NULL NULL NULL +NULL NULL NULL VI 6 80 +PREHOOK: query: select x1.`z`, x1.`x`, x1.`y`, + x2.`z`, x2.`x`, x2.`y` +from t_letters x1 full outer join t_roman x2 on (x1.`x` > 2) and (x2.`x` < 4) and (x1.`x` = x2.`x`) +PREHOOK: type: QUERY +PREHOOK: Input: default@t_letters +PREHOOK: Input: default@t_roman +#### A masked pattern was here #### +POSTHOOK: query: select x1.`z`, x1.`x`, x1.`y`, + x2.`z`, x2.`x`, x2.`y` +from t_letters x1 full outer join t_roman x2 on (x1.`x` > 2) and (x2.`x` < 4) and (x1.`x` = x2.`x`) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t_letters +POSTHOOK: Input: default@t_roman +#### A masked pattern was here #### +one 1 50 NULL NULL NULL +NULL NULL NULL I 1 50 +two 2 30 NULL NULL NULL +NULL NULL NULL II 2 30 +three 3 30 III 3 30 +four 4 60 NULL NULL NULL +NULL NULL NULL IV 4 60 +five 5 70 NULL NULL NULL +NULL NULL NULL V 5 70 +six 6 80 NULL NULL NULL +NULL NULL NULL VI 6 80