[HIVE-21980] Parsing time can be high in case of deeply nested subqueries

• Type: Improvement
• Status: Closed
• Priority: Major
• Resolution: Fixed
• Fix Version/s: 2.3.7, 2.4.0, 3.1.2, 3.2.0, 4.0.0-alpha-1
• Component/s: Parser
• Assignee: Zoltan Haindrich
• Reporter: Zoltan Haindrich
• Created: 10/Jul/19 07:58
• Updated: 24/Aug/23 18:44
• Resolved: 16/Jul/19 09:23

Description

for queries which are recursively doing:

SELECT ...
FROM (SELECT ...
             FROM ( [...]
             ) JOIN
            (SELECT ...
            FROM ( [...] )
JOIN ( [...]
)

when the issue is present hs2 stops right after starting compiling and the following stack is there

	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.subQuerySource(HiveParser_FromClauseParser.java:4745)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.synpred1_FromClauseParser_fragment(HiveParser_FromClauseParser.java:6513)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.synpred1_FromClauseParser(HiveParser_FromClauseParser.java:6660)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser$DFA10.specialStateTransition(HiveParser_FromClauseParser.java:6942)
	at org.antlr.runtime.DFA.predict(DFA.java:80)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.atomjoinSource(HiveParser_FromClauseParser.java:1596)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.joinSource(HiveParser_FromClauseParser.java:1859)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromSource(HiveParser_FromClauseParser.java:1532)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromClause(HiveParser_FromClauseParser.java:1375)
	at org.apache.hadoop.hive.ql.parse.HiveParser.fromClause(HiveParser.java:45755)
	at org.apache.hadoop.hive.ql.parse.HiveParser.atomSelectStatement(HiveParser.java:40337)
	at org.apache.hadoop.hive.ql.parse.HiveParser.selectStatement(HiveParser.java:40589)
	at org.apache.hadoop.hive.ql.parse.HiveParser.regularBody(HiveParser.java:40235)
	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpressionBody(HiveParser.java:39445)
	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpression(HiveParser.java:39333)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.subQuerySource(HiveParser_FromClauseParser.java:4745)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.atomjoinSource(HiveParser_FromClauseParser.java:1699)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.joinSource(HiveParser_FromClauseParser.java:1859)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromSource(HiveParser_FromClauseParser.java:1532)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromClause(HiveParser_FromClauseParser.java:1375)
	at org.apache.hadoop.hive.ql.parse.HiveParser.fromClause(HiveParser.java:45755)
	at org.apache.hadoop.hive.ql.parse.HiveParser.atomSelectStatement(HiveParser.java:40337)
	at org.apache.hadoop.hive.ql.parse.HiveParser.selectStatement(HiveParser.java:40589)
	at org.apache.hadoop.hive.ql.parse.HiveParser.regularBody(HiveParser.java:40235)
	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpressionBody(HiveParser.java:39445)
	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpression(HiveParser.java:39333)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.subQuerySource(HiveParser_FromClauseParser.java:4745)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.synpred1_FromClauseParser_fragment(HiveParser_FromClauseParser.java:6513)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.synpred1_FromClauseParser(HiveParser_FromClauseParser.java:6660)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser$DFA10.specialStateTransition(HiveParser_FromClauseParser.java:6942)
	at org.antlr.runtime.DFA.predict(DFA.java:80)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.atomjoinSource(HiveParser_FromClauseParser.java:1596)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.joinSource(HiveParser_FromClauseParser.java:1859)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromSource(HiveParser_FromClauseParser.java:1532)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromClause(HiveParser_FromClauseParser.java:1375)
	at org.apache.hadoop.hive.ql.parse.HiveParser.fromClause(HiveParser.java:45755)
	at org.apache.hadoop.hive.ql.parse.HiveParser.atomSelectStatement(HiveParser.java:40337)
	at org.apache.hadoop.hive.ql.parse.HiveParser.selectStatement(HiveParser.java:40589)
	at org.apache.hadoop.hive.ql.parse.HiveParser.regularBody(HiveParser.java:40235)
	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpressionBody(HiveParser.java:39445)
	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpression(HiveParser.java:39333)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.subQuerySource(HiveParser_FromClauseParser.java:4745)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.atomjoinSource(HiveParser_FromClauseParser.java:1699)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.joinSource(HiveParser_FromClauseParser.java:1859)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromSource(HiveParser_FromClauseParser.java:1532)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromClause(HiveParser_FromClauseParser.java:1375)
	at org.apache.hadoop.hive.ql.parse.HiveParser.fromClause(HiveParser.java:45755)
	at org.apache.hadoop.hive.ql.parse.HiveParser.atomSelectStatement(HiveParser.java:40337)
	at org.apache.hadoop.hive.ql.parse.HiveParser.selectStatement(HiveParser.java:40589)
	at org.apache.hadoop.hive.ql.parse.HiveParser.regularBody(HiveParser.java:40235)
	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpressionBody(HiveParser.java:39445)
	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpression(HiveParser.java:39333)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.subQuerySource(HiveParser_FromClauseParser.java:4745)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.atomjoinSource(HiveParser_FromClauseParser.java:1699)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.joinSource(HiveParser_FromClauseParser.java:1859)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromSource(HiveParser_FromClauseParser.java:1532)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromClause(HiveParser_FromClauseParser.java:1375)
	at org.apache.hadoop.hive.ql.parse.HiveParser.fromClause(HiveParser.java:45755)
	at org.apache.hadoop.hive.ql.parse.HiveParser.atomSelectStatement(HiveParser.java:40337)
	at org.apache.hadoop.hive.ql.parse.HiveParser.selectStatement(HiveParser.java:40589)
	at org.apache.hadoop.hive.ql.parse.HiveParser.regularBody(HiveParser.java:40235)
	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpressionBody(HiveParser.java:39445)
	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpression(HiveParser.java:39333)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.subQuerySource(HiveParser_FromClauseParser.java:4745)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.synpred1_FromClauseParser_fragment(HiveParser_FromClauseParser.java:6513)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.synpred1_FromClauseParser(HiveParser_FromClauseParser.java:6660)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser$DFA10.specialStateTransition(HiveParser_FromClauseParser.java:6942)
	at org.antlr.runtime.DFA.predict(DFA.java:80)

Issue Links

• causes: HIVE-27649 Support ORDER BY clause in subqueries with set operators