Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

solve stack overflow on RecursionLimitExceeded during debug building #1171

Merged
merged 7 commits into from
Apr 7, 2024
Merged

solve stack overflow on RecursionLimitExceeded during debug building #1171

Changes from 3 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
1362e46
Use additional call to box values(to reserve less stack mem)
Nikita-str Mar 11, 2024
cfc1525
Make stack size less by moving out big structs into closure
Nikita-str Mar 12, 2024
63baa83
moving out use of big structs into case-related fns
Nikita-str Mar 12, 2024
f9ab24a
Make functions non pub
alamb Apr 7, 2024
c6496f9
Merge remote-tracking branch 'origin/main' into solve-stack-overflow-…
alamb Apr 7, 2024
8101f47
Remove inner module
alamb Apr 7, 2024
780a037
fix docs
alamb Apr 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
141 changes: 88 additions & 53 deletions src/parser/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -470,7 +470,7 @@ impl<'a> Parser<'a> {
Keyword::ANALYZE => Ok(self.parse_analyze()?),
Keyword::SELECT | Keyword::WITH | Keyword::VALUES => {
self.prev_token();
Ok(Statement::Query(Box::new(self.parse_query()?)))
Ok(Statement::Query(self.parse_boxed_query()?))
}
Keyword::TRUNCATE => Ok(self.parse_truncate()?),
Keyword::ATTACH => Ok(self.parse_attach_database()?),
Expand Down Expand Up @@ -530,7 +530,7 @@ impl<'a> Parser<'a> {
},
Token::LParen => {
self.prev_token();
Ok(Statement::Query(Box::new(self.parse_query()?)))
Ok(Statement::Query(self.parse_boxed_query()?))
}
_ => self.expected("an SQL statement", next_token),
}
Expand Down Expand Up @@ -1084,7 +1084,7 @@ impl<'a> Parser<'a> {
let expr =
if self.parse_keyword(Keyword::SELECT) || self.parse_keyword(Keyword::WITH) {
self.prev_token();
Expr::Subquery(Box::new(self.parse_query()?))
Expr::Subquery(self.parse_boxed_query()?)
} else {
let exprs = self.parse_comma_separated(Parser::parse_expr)?;
match exprs.len() {
Expand Down Expand Up @@ -1461,7 +1461,7 @@ impl<'a> Parser<'a> {
self.expect_token(&Token::LParen)?;
let exists_node = Expr::Exists {
negated,
subquery: Box::new(self.parse_query()?),
subquery: self.parse_boxed_query()?,
};
self.expect_token(&Token::RParen)?;
Ok(exists_node)
Expand Down Expand Up @@ -1670,9 +1670,9 @@ impl<'a> Parser<'a> {

// Parses an array constructed from a subquery
pub fn parse_array_subquery(&mut self) -> Result<Expr, ParserError> {
let query = self.parse_query()?;
let query = self.parse_boxed_query()?;
self.expect_token(&Token::RParen)?;
Ok(Expr::ArraySubquery(Box::new(query)))
Ok(Expr::ArraySubquery(query))
}

/// Parse a SQL LISTAGG expression, e.g. `LISTAGG(...) WITHIN GROUP (ORDER BY ...)`.
Expand Down Expand Up @@ -2531,7 +2531,7 @@ impl<'a> Parser<'a> {
self.prev_token();
Expr::InSubquery {
expr: Box::new(expr),
subquery: Box::new(self.parse_query()?),
subquery: self.parse_boxed_query()?,
negated,
}
} else {
Expand Down Expand Up @@ -3574,7 +3574,7 @@ impl<'a> Parser<'a> {
}

self.expect_keyword(Keyword::AS)?;
let query = Box::new(self.parse_query()?);
let query = self.parse_boxed_query()?;
// Optional `WITH [ CASCADED | LOCAL ] CHECK OPTION` is widely supported here.

let with_no_schema_binding = dialect_of!(self is RedshiftSqlDialect | GenericDialect)
Expand Down Expand Up @@ -3969,7 +3969,7 @@ impl<'a> Parser<'a> {

self.expect_keyword(Keyword::FOR)?;

let query = Some(Box::new(self.parse_query()?));
let query = Some(self.parse_boxed_query()?);

Ok(Statement::Declare {
stmts: vec![Declare {
Expand Down Expand Up @@ -4063,7 +4063,7 @@ impl<'a> Parser<'a> {
match self.peek_token().token {
Token::Word(w) if w.keyword == Keyword::SELECT => (
Some(DeclareType::Cursor),
Some(Box::new(self.parse_query()?)),
Some(self.parse_boxed_query()?),
None,
None,
),
Expand Down Expand Up @@ -4587,7 +4587,7 @@ impl<'a> Parser<'a> {

// Parse optional `AS ( query )`
let query = if self.parse_keyword(Keyword::AS) {
Some(Box::new(self.parse_query()?))
Some(self.parse_boxed_query()?)
} else {
None
};
Expand Down Expand Up @@ -5583,7 +5583,7 @@ impl<'a> Parser<'a> {
let with_options = self.parse_options(Keyword::WITH)?;

self.expect_keyword(Keyword::AS)?;
let query = Box::new(self.parse_query()?);
let query = self.parse_boxed_query()?;

Ok(Statement::AlterView {
name,
Expand Down Expand Up @@ -5623,7 +5623,7 @@ impl<'a> Parser<'a> {
pub fn parse_copy(&mut self) -> Result<Statement, ParserError> {
let source;
if self.consume_token(&Token::LParen) {
source = CopySource::Query(Box::new(self.parse_query()?));
source = CopySource::Query(self.parse_boxed_query()?);
self.expect_token(&Token::RParen)?;
} else {
let table_name = self.parse_object_name(false)?;
Expand Down Expand Up @@ -6846,11 +6846,60 @@ impl<'a> Parser<'a> {
}
}

/// Do the same as `parse_query` but return `Box`'ed `Ok(..)` result
/// and therefore caller fn reserves only `poiter size` memory on stack
/// instead of `sizeof(Query)` (if the call isn't optimized by compiler)
pub fn parse_boxed_query(&mut self) -> Result<Box<Query>, ParserError> {
self.parse_query().map(Box::new)
}

/// Parse a query expression, i.e. a `SELECT` statement optionally
/// preceded with some `WITH` CTE declarations and optionally followed
/// by `ORDER BY`. Unlike some other parse_... methods, this one doesn't
/// expect the initial keyword to be already consumed
///
/// If you need `Box<Query>` then maybe there is sense to use `parse_boxed_query`
/// due to prevent stack overflow in debug building(to reserve less memory on stack).
pub fn parse_query(&mut self) -> Result<Query, ParserError> {
mod parse_query {
use super::*;
type ReturnTy = Result<Query, ParserError>;
pub fn insert_case(parser: &mut Parser<'_>, with: Option<With>) -> ReturnTy {
let body = parser
.parse_insert()
.map(|insert| Box::new(SetExpr::Insert(insert)))?;

Ok(Query {
with,
body,
limit: None,
limit_by: vec![],
order_by: vec![],
offset: None,
fetch: None,
locks: vec![],
for_clause: None,
})
}
pub fn update_case(parser: &mut Parser<'_>, with: Option<With>) -> ReturnTy {
let body = parser
.parse_update()
.map(|update| Box::new(SetExpr::Update(update)))?;

Ok(Query {
with,
body,
limit: None,
limit_by: vec![],
order_by: vec![],
offset: None,
fetch: None,
locks: vec![],
for_clause: None,
})
}
}

let _guard = self.recursion_counter.try_decrease()?;
let with = if self.parse_keyword(Keyword::WITH) {
Some(With {
Expand All @@ -6860,36 +6909,12 @@ impl<'a> Parser<'a> {
} else {
None
};

if self.parse_keyword(Keyword::INSERT) {
let insert = self.parse_insert()?;

Ok(Query {
with,
body: Box::new(SetExpr::Insert(insert)),
limit: None,
limit_by: vec![],
order_by: vec![],
offset: None,
fetch: None,
locks: vec![],
for_clause: None,
})
parse_query::insert_case(self, with)
} else if self.parse_keyword(Keyword::UPDATE) {
let update = self.parse_update()?;
Ok(Query {
with,
body: Box::new(SetExpr::Update(update)),
limit: None,
limit_by: vec![],
order_by: vec![],
offset: None,
fetch: None,
locks: vec![],
for_clause: None,
})
parse_query::update_case(self, with)
} else {
let body = Box::new(self.parse_query_body(0)?);
let body = self.parse_boxed_query_body(0)?;

let order_by = if self.parse_keywords(&[Keyword::ORDER, Keyword::BY]) {
self.parse_comma_separated(Parser::parse_order_by_expr)?
Expand Down Expand Up @@ -7079,7 +7104,7 @@ impl<'a> Parser<'a> {
}
}
self.expect_token(&Token::LParen)?;
let query = Box::new(self.parse_query()?);
let query = self.parse_boxed_query()?;
self.expect_token(&Token::RParen)?;
let alias = TableAlias {
name,
Expand All @@ -7103,7 +7128,7 @@ impl<'a> Parser<'a> {
}
}
self.expect_token(&Token::LParen)?;
let query = Box::new(self.parse_query()?);
let query = self.parse_boxed_query()?;
self.expect_token(&Token::RParen)?;
let alias = TableAlias { name, columns };
Cte {
Expand All @@ -7119,6 +7144,13 @@ impl<'a> Parser<'a> {
Ok(cte)
}

/// Do the same as `parse_query_body` but return `Box`'ed `Ok(..)` result
/// and therefore caller fn reserves only `poiter size` memory on stack
/// instead of `sizeof(SetExpr)` (if the call isn't optimized by compiler)
pub fn parse_boxed_query_body(&mut self, precedence: u8) -> Result<Box<SetExpr>, ParserError> {
self.parse_query_body(precedence).map(Box::new)
}

/// Parse a "query body", which is an expression with roughly the
/// following grammar:
/// ```sql
Expand All @@ -7127,16 +7159,19 @@ impl<'a> Parser<'a> {
/// subquery ::= query_body [ order_by_limit ]
/// set_operation ::= query_body { 'UNION' | 'EXCEPT' | 'INTERSECT' } [ 'ALL' ] query_body
/// ```
///
/// If you need `Box<SetExpr>` then maybe there is sense to use `parse_boxed_query_body`
/// due to prevent stack overflow in debug building(to reserve less memory on stack).
pub fn parse_query_body(&mut self, precedence: u8) -> Result<SetExpr, ParserError> {
// We parse the expression using a Pratt parser, as in `parse_expr()`.
// Start by parsing a restricted SELECT or a `(subquery)`:
let mut expr = if self.parse_keyword(Keyword::SELECT) {
SetExpr::Select(Box::new(self.parse_select()?))
SetExpr::Select(self.parse_select().map(Box::new)?)
} else if self.consume_token(&Token::LParen) {
// CTEs are not allowed here, but the parser currently accepts them
let subquery = self.parse_query()?;
let subquery = self.parse_boxed_query()?;
self.expect_token(&Token::RParen)?;
SetExpr::Query(Box::new(subquery))
SetExpr::Query(subquery)
} else if self.parse_keyword(Keyword::VALUES) {
let is_mysql = dialect_of!(self is MySqlDialect);
SetExpr::Values(self.parse_values(is_mysql)?)
Expand Down Expand Up @@ -7169,7 +7204,7 @@ impl<'a> Parser<'a> {
left: Box::new(expr),
op: op.unwrap(),
set_quantifier,
right: Box::new(self.parse_query_body(next_precedence)?),
right: self.parse_boxed_query_body(next_precedence)?,
};
}

Expand Down Expand Up @@ -8083,7 +8118,7 @@ impl<'a> Parser<'a> {
&mut self,
lateral: IsLateral,
) -> Result<TableFactor, ParserError> {
let subquery = Box::new(self.parse_query()?);
let subquery = self.parse_boxed_query()?;
self.expect_token(&Token::RParen)?;
let alias = self.parse_optional_table_alias(keywords::RESERVED_FOR_TABLE_ALIAS)?;
Ok(TableFactor::Derived {
Expand Down Expand Up @@ -8381,7 +8416,7 @@ impl<'a> Parser<'a> {
} else {
None
};
let source = Box::new(self.parse_query()?);
let source = self.parse_boxed_query()?;
Ok(Statement::Directory {
local,
path,
Expand Down Expand Up @@ -8414,7 +8449,7 @@ impl<'a> Parser<'a> {
// Hive allows you to specify columns after partitions as well if you want.
let after_columns = self.parse_parenthesized_column_list(Optional, false)?;

let source = Some(Box::new(self.parse_query()?));
let source = Some(self.parse_boxed_query()?);

(columns, partitioned, after_columns, source)
};
Expand Down Expand Up @@ -8607,11 +8642,11 @@ impl<'a> Parser<'a> {
.is_some()
{
self.prev_token();
let subquery = self.parse_query()?;
let subquery = self.parse_boxed_query()?;
self.expect_token(&Token::RParen)?;
return Ok((
vec![FunctionArg::Unnamed(FunctionArgExpr::from(Expr::Subquery(
Box::new(subquery),
subquery,
)))],
vec![],
));
Expand Down Expand Up @@ -9125,7 +9160,7 @@ impl<'a> Parser<'a> {

pub fn parse_unload(&mut self) -> Result<Statement, ParserError> {
self.expect_token(&Token::LParen)?;
let query = self.parse_query()?;
let query = self.parse_boxed_query()?;
self.expect_token(&Token::RParen)?;

self.expect_keyword(Keyword::TO)?;
Expand All @@ -9134,7 +9169,7 @@ impl<'a> Parser<'a> {
let with_options = self.parse_options(Keyword::WITH)?;

Ok(Statement::Unload {
query: Box::new(query),
query,
to,
with: with_options,
})
Expand Down
Loading