diff --git pom.xml pom.xml
index 376197e..603d98b 100644
--- pom.xml
+++ pom.xml
@@ -101,7 +101,7 @@
2.4
2.4
2.4.3
- 2.19.1
+ 2.18.1
2.4
2.8
2.9
diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
index e2363eb..0e673be 100644
--- ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
+++ ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
@@ -139,7 +139,7 @@ public static ReduceWork createReduceWork(
tezWork.add(reduceWork);
TezEdgeProperty edgeProp;
- EdgeType edgeType = determineEdgeType(context.preceedingWork, reduceWork);
+ EdgeType edgeType = determineEdgeType(context.preceedingWork, reduceWork, reduceSink);
if (reduceWork.isAutoReduceParallelism()) {
edgeProp =
new TezEdgeProperty(context.conf, edgeType, true,
@@ -491,7 +491,7 @@ public static void removeBranch(AppMasterEventOperator event) {
curr.removeChild(child);
}
- public static EdgeType determineEdgeType(BaseWork preceedingWork, BaseWork followingWork) {
+ public static EdgeType determineEdgeType(BaseWork preceedingWork, BaseWork followingWork, ReduceSinkOperator reduceSinkOperator) {
if (followingWork instanceof ReduceWork) {
// Ideally there should be a better way to determine that the followingWork contains
// a dynamic partitioned hash join, but in some cases (createReduceWork()) it looks like
@@ -505,6 +505,12 @@ public static EdgeType determineEdgeType(BaseWork preceedingWork, BaseWork follo
}
}
}
+ if(!reduceSinkOperator.getConf().isOrdering()) {
+ //if no sort keys are specified, use an edge that does not sort
+
+ //todo: Conjunction with pred on single inputs (left outer join) in join46.q; line 285 in llap/join46.q.out
+ return EdgeType.CUSTOM_SIMPLE_EDGE;
+ }
return EdgeType.SIMPLE_EDGE;
}
}
diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java
index 2b96e51..97f3300 100644
--- ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java
+++ ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java
@@ -452,7 +452,7 @@ public Object process(Node nd, Stack stack,
if (!context.connectedReduceSinks.contains(rs)) {
// add dependency between the two work items
TezEdgeProperty edgeProp;
- EdgeType edgeType = utils.determineEdgeType(work, followingWork);
+ EdgeType edgeType = GenTezUtils.determineEdgeType(work, followingWork, rs);
if (rWork.isAutoReduceParallelism()) {
edgeProp =
new TezEdgeProperty(context.conf, edgeType, true,
diff --git ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
index 9bfcc82..fb35c2e 100644
--- ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
+++ ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
@@ -737,15 +737,33 @@ public void testMergeType2SCD02() throws Exception {
Assert.assertEquals(stringifyValues(resultVals), r);
}
- @Ignore("HIVE-14707")
+// @Ignore("HIVE-14707")
@Test
public void testMergeInsertOnly() throws Exception {
+ runStatementOnDriver("CREATE TABLE dest1(key INT, ten INT, one INT, value STRING) STORED AS TEXTFILE");
+
String query = "merge into " + Table.ACIDTBL +
" as t using " + Table.NONACIDORCTBL + " s ON t.a = s.a " +
+ "WHEN MATCHED THEN DELETE " +
"WHEN NOT MATCHED THEN INSERT VALUES(s.a, s.b) ";
+
+// merge into acidTbl as t using nonAcidOrcTbl s ON t.a = s.a
+// WHEN MATCHED and t.a > 3 and t.a < 5 THEN DELETE WHEN NOT MATCHED THEN INSERT VALUES(s.a, s.b)
+
+// runStatementOnDriver("CREATE TABLE src (key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS TEXTFILE");
+// query = "select key, count(key) from src group by key";
+// query = "analyze select key from src where key < 10;";
+
+ query = "FROM src " +
+ "INSERT OVERWRITE TABLE dest1 " +
+ "MAP src.key, CAST(src.key / 10 AS INT), CAST(src.key % 10 AS INT), src.value " +
+ "USING 'cat' AS (tkey, ten, one, tvalue) " +
+ "DISTRIBUTE BY tvalue, tkey;";
+
d.destroy();
HiveConf hc = new HiveConf(hiveConf);
hc.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "tez");
+ hc.setBoolVar(HiveConf.ConfVars.HIVE_EXPLAIN_USER, false);
d = new Driver(hc);
d.setMaxRows(10000);
@@ -757,6 +775,32 @@ public void testMergeInsertOnly() throws Exception {
LOG.info("Explain1: " + sb);
}
@Test
+ public void testSortEdge() throws Exception {
+// int[][] baseValsOdd = {{2,2},{4,44},{5,5},{11,11}};
+// runStatementOnDriver("insert into " + Table.NONACIDORCTBL + " " + makeValuesClause(baseValsOdd));
+ runStatementOnDriver("CREATE TABLE test1 (key INT, value INT, col_1 STRING) ");
+ runStatementOnDriver("INSERT INTO test1 VALUES (NULL, NULL, 'None'), (98, NULL, 'None'),(99, 0, 'Alice'), (99, 2, 'Mat'), (100, 1, 'Bob'), (101, 2, 'Car')");
+ runStatementOnDriver("CREATE TABLE test2 (key INT, value INT, col_2 STRING)");
+ runStatementOnDriver("INSERT INTO test2 VALUES (102, 2, 'Del'), (103, 2, 'Ema'),(104, 3, 'Fli'), (105, NULL, 'None')");
+
+ d.destroy();
+ HiveConf hc = new HiveConf(hiveConf);
+ hc.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "tez");
+ hc.setBoolVar(HiveConf.ConfVars.HIVE_EXPLAIN_USER, false);
+ d = new Driver(hc);
+ d.setMaxRows(10000);
+ List explain = runStatementOnDriver("EXPLAIN\n" +
+ "SELECT *\n" +
+ "FROM test1 LEFT OUTER JOIN test2\n" +
+ "ON (test1.key between 100 and 102\n" +
+ " AND test2.key between 100 and 102)");
+ StringBuilder sb = new StringBuilder();
+ for(String s : explain) {
+ sb.append(s).append('\n');
+ }
+ LOG.info("Explain1: " + sb);
+ }
+ @Test
public void testMergeUpdateDelete() throws Exception {
int[][] baseValsOdd = {{2,2},{4,44},{5,5},{11,11}};
runStatementOnDriver("insert into " + Table.NONACIDORCTBL + " " + makeValuesClause(baseValsOdd));
diff --git ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out
index 21d82d2..dd1e275 100644
--- ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out
+++ ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out
@@ -1060,7 +1060,7 @@ STAGE PLANS:
#### A masked pattern was here ####
Edges:
Map 1 <- Map 3 (BROADCAST_EDGE)
- Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out
index ba56486..e6d691f 100644
--- ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out
+++ ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out
@@ -812,7 +812,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1349,7 +1349,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git ql/src/test/results/clientpositive/llap/insert_into1.q.out ql/src/test/results/clientpositive/llap/insert_into1.q.out
index c39711b..704649b 100644
--- ql/src/test/results/clientpositive/llap/insert_into1.q.out
+++ ql/src/test/results/clientpositive/llap/insert_into1.q.out
@@ -586,7 +586,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git ql/src/test/results/clientpositive/llap/insert_into2.q.out ql/src/test/results/clientpositive/llap/insert_into2.q.out
index cc44706..a42c651 100644
--- ql/src/test/results/clientpositive/llap/insert_into2.q.out
+++ ql/src/test/results/clientpositive/llap/insert_into2.q.out
@@ -462,7 +462,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git ql/src/test/results/clientpositive/llap/join46.q.out ql/src/test/results/clientpositive/llap/join46.q.out
index 1cd856b..2f5c68c 100644
--- ql/src/test/results/clientpositive/llap/join46.q.out
+++ ql/src/test/results/clientpositive/llap/join46.q.out
@@ -282,7 +282,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -494,7 +494,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -608,7 +608,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -723,7 +723,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -836,7 +836,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1057,7 +1057,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1171,7 +1171,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1286,7 +1286,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1396,7 +1396,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1619,7 +1619,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1734,7 +1734,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1847,7 +1847,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2101,7 +2101,7 @@ STAGE PLANS:
#### A masked pattern was here ####
Edges:
Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE)
- Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE)
Reducer 6 <- Map 5 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
diff --git ql/src/test/results/clientpositive/llap/llap_nullscan.q.out ql/src/test/results/clientpositive/llap/llap_nullscan.q.out
index c755268..c848093 100644
--- ql/src/test/results/clientpositive/llap/llap_nullscan.q.out
+++ ql/src/test/results/clientpositive/llap/llap_nullscan.q.out
@@ -156,7 +156,7 @@ STAGE PLANS:
#### A masked pattern was here ####
Edges:
Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE)
- Reducer 4 <- Map 3 (SIMPLE_EDGE)
+ Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -259,8 +259,8 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS)
- Reducer 5 <- Map 4 (SIMPLE_EDGE), Union 3 (CONTAINS)
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS)
+ Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git ql/src/test/results/clientpositive/llap/llapdecider.q.out ql/src/test/results/clientpositive/llap/llapdecider.q.out
index c7339c9..0056eb3 100644
--- ql/src/test/results/clientpositive/llap/llapdecider.q.out
+++ ql/src/test/results/clientpositive/llap/llapdecider.q.out
@@ -1057,7 +1057,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1118,7 +1118,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1176,7 +1176,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1234,7 +1234,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git ql/src/test/results/clientpositive/llap/mapreduce2.q.out ql/src/test/results/clientpositive/llap/mapreduce2.q.out
index 1904768..71bbb7e 100644
--- ql/src/test/results/clientpositive/llap/mapreduce2.q.out
+++ ql/src/test/results/clientpositive/llap/mapreduce2.q.out
@@ -31,7 +31,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out
index 48a86cf..ef24cb8 100644
--- ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out
+++ ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out
@@ -134,7 +134,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -192,7 +192,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -330,7 +330,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -397,7 +397,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1