Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[syntax-errors] Tuple unpacking in return and yield before Python 3.8 #16485

Merged
merged 9 commits into from
Mar 6, 2025
+1,217 −8
Merged

[syntax-errors] Tuple unpacking in return and yield before Python 3.8 #16485

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
914d0a6
[syntax-errors] Tuple unpacking in `return` and `yield` before 3.8
ntBre Mar 3, 2025
f2aa25a
separate return and yield tests
ntBre Mar 5, 2025
99a866e
move yield test to expression, add suggested tests
ntBre Mar 5, 2025
0d93a56
Merge branch 'main' into brent/syn-unpack-return-yield
ntBre Mar 5, 2025
67dd41c
document variant
ntBre Mar 5, 2025
c5485dc
yield statements -> yield expressions in message
ntBre Mar 5, 2025
7987697
Merge branch 'main' into brent/syn-unpack-return-yield
ntBre Mar 5, 2025
d2b68f7
add blank lines above variant docs
ntBre Mar 6, 2025
bc240ef
delete todos moved into their own issues
ntBre Mar 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions crates/ruff_python_parser/resources/inline/err/iter_unpack_return_py37.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# parse_options: {"target-version": "3.7"}
rest = (4, 5, 6)
def f(): return 1, 2, 3, *rest
4 changes: 4 additions & 0 deletions crates/ruff_python_parser/resources/inline/err/iter_unpack_yield_py37.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# parse_options: {"target-version": "3.7"}
rest = (4, 5, 6)
def g(): yield 1, 2, 3, *rest
def h(): yield 1, (yield 2, *rest), 3
3 changes: 3 additions & 0 deletions crates/ruff_python_parser/resources/inline/ok/iter_unpack_return_py37.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# parse_options: {"target-version": "3.7"}
rest = (4, 5, 6)
def f(): return (1, 2, 3, *rest)
3 changes: 3 additions & 0 deletions crates/ruff_python_parser/resources/inline/ok/iter_unpack_return_py38.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# parse_options: {"target-version": "3.8"}
rest = (4, 5, 6)
def f(): return 1, 2, 3, *rest
3 changes: 3 additions & 0 deletions crates/ruff_python_parser/resources/inline/ok/iter_unpack_yield_py37.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# parse_options: {"target-version": "3.7"}
rest = (4, 5, 6)
def g(): yield (1, 2, 3, *rest)
4 changes: 4 additions & 0 deletions crates/ruff_python_parser/resources/inline/ok/iter_unpack_yield_py38.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# parse_options: {"target-version": "3.8"}
rest = (4, 5, 6)
def g(): yield 1, 2, 3, *rest
def h(): yield 1, (yield 2, *rest), 3
66 changes: 66 additions & 0 deletions crates/ruff_python_parser/src/error.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -444,11 +444,68 @@ pub struct UnsupportedSyntaxError {
pub target_version: PythonVersion,
}

/// The type of tuple unpacking for [`UnsupportedSyntaxErrorKind::StarTuple`].
#[derive(Debug, PartialEq, Clone, Copy)]
pub enum StarTupleKind {
Return,
Yield,
}

#[derive(Debug, PartialEq, Clone, Copy)]
pub enum UnsupportedSyntaxErrorKind {
Match,
Walrus,
ExceptStar,

/// Represents the use of unparenthesized tuple unpacking in a `return` statement or `yield`
/// expression before Python 3.8.
///
/// ## Examples
///
/// Before Python 3.8, this syntax was allowed:
///
/// ```python
/// rest = (4, 5, 6)
///
/// def f():
/// t = 1, 2, 3, *rest
/// return t
///
/// def g():
/// t = 1, 2, 3, *rest
/// yield t
/// ```
///
/// But this was not:
///
/// ```python
/// rest = (4, 5, 6)
///
/// def f():
/// return 1, 2, 3, *rest
///
/// def g():
/// yield 1, 2, 3, *rest
/// ```
///
/// Instead, parentheses were required in the `return` and `yield` cases:
///
/// ```python
/// rest = (4, 5, 6)
///
/// def f():
/// return (1, 2, 3, *rest)
///
/// def g():
/// yield (1, 2, 3, *rest)
/// ```
///
/// This was reported in [BPO 32117] and updated in Python 3.8 to allow the unparenthesized
/// form.
///
/// [BPO 32117]: https://github.com/python/cpython/issues/76298
StarTuple(StarTupleKind),

/// Represents the use of a "relaxed" [PEP 614] decorator before Python 3.9.
///
/// ## Examples
Expand Down Expand Up @@ -480,6 +537,7 @@ pub enum UnsupportedSyntaxErrorKind {
/// [`dotted_name`]: https://docs.python.org/3.8/reference/compound_stmts.html#grammar-token-dotted-name
/// [decorator grammar]: https://docs.python.org/3/reference/compound_stmts.html#grammar-token-python-grammar-decorator
RelaxedDecorator,

/// Represents the use of a [PEP 570] positional-only parameter before Python 3.8.
///
/// ## Examples
Expand All @@ -506,6 +564,7 @@ pub enum UnsupportedSyntaxErrorKind {
///
/// [PEP 570]: https://peps.python.org/pep-0570/
PositionalOnlyParameter,

/// Represents the use of a [type parameter list] before Python 3.12.
///
/// ## Examples
Expand Down Expand Up @@ -544,6 +603,12 @@ impl Display for UnsupportedSyntaxError {
UnsupportedSyntaxErrorKind::Match => "Cannot use `match` statement",
UnsupportedSyntaxErrorKind::Walrus => "Cannot use named assignment expression (`:=`)",
UnsupportedSyntaxErrorKind::ExceptStar => "Cannot use `except*`",
UnsupportedSyntaxErrorKind::StarTuple(StarTupleKind::Return) => {
"Cannot use iterable unpacking in return statements"
}
UnsupportedSyntaxErrorKind::StarTuple(StarTupleKind::Yield) => {
"Cannot use iterable unpacking in yield expressions"
}
UnsupportedSyntaxErrorKind::RelaxedDecorator => "Unsupported expression in decorators",
UnsupportedSyntaxErrorKind::PositionalOnlyParameter => {
"Cannot use positional-only parameter separator"
Expand All @@ -570,6 +635,7 @@ impl UnsupportedSyntaxErrorKind {
UnsupportedSyntaxErrorKind::Match => PythonVersion::PY310,
UnsupportedSyntaxErrorKind::Walrus => PythonVersion::PY38,
UnsupportedSyntaxErrorKind::ExceptStar => PythonVersion::PY311,
UnsupportedSyntaxErrorKind::StarTuple(_) => PythonVersion::PY38,
UnsupportedSyntaxErrorKind::RelaxedDecorator => PythonVersion::PY39,
UnsupportedSyntaxErrorKind::PositionalOnlyParameter => PythonVersion::PY38,
UnsupportedSyntaxErrorKind::TypeParameterList => PythonVersion::PY312,
Expand Down
26 changes: 22 additions & 4 deletions crates/ruff_python_parser/src/parser/expression.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ use ruff_python_ast::{
};
use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};

use crate::error::StarTupleKind;
use crate::parser::progress::ParserProgress;
use crate::parser::{helpers, FunctionKind, Parser};
use crate::string::{parse_fstring_literal_element, parse_string_literal, StringType};
Expand Down Expand Up @@ -2089,10 +2090,27 @@ impl<'src> Parser<'src> {
}

let value = self.at_expr().then(|| {
Box::new(
self.parse_expression_list(ExpressionContext::starred_bitwise_or())
.expr,
)
let parsed_expr = self.parse_expression_list(ExpressionContext::starred_bitwise_or());

// test_ok iter_unpack_yield_py37
// # parse_options: {"target-version": "3.7"}
// rest = (4, 5, 6)
// def g(): yield (1, 2, 3, *rest)

// test_ok iter_unpack_yield_py38
// # parse_options: {"target-version": "3.8"}
// rest = (4, 5, 6)
// def g(): yield 1, 2, 3, *rest
// def h(): yield 1, (yield 2, *rest), 3

// test_err iter_unpack_yield_py37
// # parse_options: {"target-version": "3.7"}
// rest = (4, 5, 6)
// def g(): yield 1, 2, 3, *rest
// def h(): yield 1, (yield 2, *rest), 3
self.check_tuple_unpacking(&parsed_expr, StarTupleKind::Yield);

Box::new(parsed_expr.expr)
});

Expr::Yield(ast::ExprYield {
Expand Down
51 changes: 47 additions & 4 deletions crates/ruff_python_parser/src/parser/statement.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ use ruff_python_ast::{
};
use ruff_text_size::{Ranged, TextRange, TextSize};

use crate::error::StarTupleKind;
use crate::parser::expression::{ParsedExpr, EXPR_SET};
use crate::parser::progress::ParserProgress;
use crate::parser::{
Expand Down Expand Up @@ -389,10 +390,25 @@ impl<'src> Parser<'src> {
// return x := 1
// return *x and y
let value = self.at_expr().then(|| {
Box::new(
self.parse_expression_list(ExpressionContext::starred_bitwise_or())
.expr,
)
let parsed_expr = self.parse_expression_list(ExpressionContext::starred_bitwise_or());

// test_ok iter_unpack_return_py37
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we add test cases using single unpack element like return *rest or yield *rest?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What about yield 1, (*rest), yield 1, (yield 2, *rest), 3 (yield can be used as an expression)? I'm just throwing out random examples :)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for these! The single cases are actually still invalid on my system Python (3.13):

>>> def foo(): return *rest
  File "<python-input-0>", line 1
    def foo(): return *rest
                      ^^^^^
SyntaxError: can't use starred expression here
>>> def foo(): yield *rest
  File "<python-input-1>", line 1
    def foo(): yield *rest
                     ^^^^^
SyntaxError: can't use starred expression here

But these aren't flagged as ParseErrors by ruff.

The second case is very interesting. It's actually accepted on both 3.7 and 3.8, but not on my 3.13 system Python (SyntaxError: cannot use starred expression here). That might be another change not currently tracked in #6591.

I added your third case as an example of a yield expression and moved the check into parse_yield_expression as @MichaReiser suggested.

Copy link
Member

@dhruvmanila dhruvmanila Mar 6, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But these aren't flagged as ParseErrors by ruff.

That's because the grammar says it's allowed.

return_stmt:
    | 'return' [star_expressions] 

star_expressions:
    | star_expression (',' star_expression )+ [','] 
    | star_expression ',' 
    | star_expression

star_expression:
    | '*' bitwise_or 
    | expression

I think it's caught by the CPython compiler and not the parser. You can see that python -m ast test.py wouldn't raise a syntax error here.

$ python -m ast _.py       
Module(
   body=[
      FunctionDef(
         name='foo',
         args=arguments(
            posonlyargs=[],
            args=[
               arg(arg='args')],
            kwonlyargs=[],
            kw_defaults=[],
            defaults=[]),
         body=[
            Return(
               value=Starred(
                  value=Name(id='args', ctx=Load()),
                  ctx=Load()))],
         decorator_list=[],
         type_params=[])],
   type_ignores=[])

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oops, missed the issue #16520

// # parse_options: {"target-version": "3.7"}
// rest = (4, 5, 6)
// def f(): return (1, 2, 3, *rest)

// test_ok iter_unpack_return_py38
// # parse_options: {"target-version": "3.8"}
// rest = (4, 5, 6)
// def f(): return 1, 2, 3, *rest

// test_err iter_unpack_return_py37
// # parse_options: {"target-version": "3.7"}
// rest = (4, 5, 6)
// def f(): return 1, 2, 3, *rest
self.check_tuple_unpacking(&parsed_expr, StarTupleKind::Return);

Box::new(parsed_expr.expr)
});

ast::StmtReturn {
Expand All @@ -401,6 +417,33 @@ impl<'src> Parser<'src> {
}
}

/// Report [`UnsupportedSyntaxError`]s for each starred element in `expr` if it is an
/// unparenthesized tuple.
///
/// This method can be used to check for tuple unpacking in return and yield statements, which
/// are only allowed in Python 3.8 and later: <https://github.com/python/cpython/issues/76298>.
pub(crate) fn check_tuple_unpacking(&mut self, expr: &Expr, kind: StarTupleKind) {
let kind = UnsupportedSyntaxErrorKind::StarTuple(kind);
if self.options.target_version >= kind.minimum_version() {
return;
}

let Expr::Tuple(ast::ExprTuple {
elts,
parenthesized: false,
..
}) = expr
else {
return;
};

for elt in elts {
if elt.is_starred_expr() {
self.add_unsupported_syntax_error(kind, elt.range());
}
}
}

/// Parses a `raise` statement.
///
/// # Panics
Expand Down
146 changes: 146 additions & 0 deletions crates/ruff_python_parser/tests/snapshots/invalid_syntax@iter_unpack_return_py37.py.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
---
source: crates/ruff_python_parser/tests/fixtures.rs
input_file: crates/ruff_python_parser/resources/inline/err/iter_unpack_return_py37.py
---
## AST

```
Module(
ModModule {
range: 0..91,
body: [
Assign(
StmtAssign {
range: 43..59,
targets: [
Name(
ExprName {
range: 43..47,
id: Name("rest"),
ctx: Store,
},
),
],
value: Tuple(
ExprTuple {
range: 50..59,
elts: [
NumberLiteral(
ExprNumberLiteral {
range: 51..52,
value: Int(
4,
),
},
),
NumberLiteral(
ExprNumberLiteral {
range: 54..55,
value: Int(
5,
),
},
),
NumberLiteral(
ExprNumberLiteral {
range: 57..58,
value: Int(
6,
),
},
),
],
ctx: Load,
parenthesized: true,
},
),
},
),
FunctionDef(
StmtFunctionDef {
range: 60..90,
is_async: false,
decorator_list: [],
name: Identifier {
id: Name("f"),
range: 64..65,
},
type_params: None,
parameters: Parameters {
range: 65..67,
posonlyargs: [],
args: [],
vararg: None,
kwonlyargs: [],
kwarg: None,
},
returns: None,
body: [
Return(
StmtReturn {
range: 69..90,
value: Some(
Tuple(
ExprTuple {
range: 76..90,
elts: [
NumberLiteral(
ExprNumberLiteral {
range: 76..77,
value: Int(
1,
),
},
),
NumberLiteral(
ExprNumberLiteral {
range: 79..80,
value: Int(
2,
),
},
),
NumberLiteral(
ExprNumberLiteral {
range: 82..83,
value: Int(
3,
),
},
),
Starred(
ExprStarred {
range: 85..90,
value: Name(
ExprName {
range: 86..90,
id: Name("rest"),
ctx: Load,
},
),
ctx: Load,
},
),
],
ctx: Load,
parenthesized: false,
},
),
),
},
),
],
},
),
],
},
)
```
## Unsupported Syntax Errors

|
1 | # parse_options: {"target-version": "3.7"}
2 | rest = (4, 5, 6)
3 | def f(): return 1, 2, 3, *rest
| ^^^^^ Syntax Error: Cannot use iterable unpacking in return statements on Python 3.7 (syntax was added in Python 3.8)
|
Loading
Loading