diff --git a/Cargo.lock b/Cargo.lock index 4bcd9e09f791b..6507451ed993b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3090,6 +3090,8 @@ dependencies = [ "ruff_source_file", "ruff_text_size", "rustc-hash 2.1.1", + "serde", + "serde_json", "static_assertions", "unicode-ident", "unicode-normalization", diff --git a/crates/ruff_python_parser/Cargo.toml b/crates/ruff_python_parser/Cargo.toml index d6b8425bc35f1..e6e96335bc9e6 100644 --- a/crates/ruff_python_parser/Cargo.toml +++ b/crates/ruff_python_parser/Cargo.toml @@ -29,10 +29,13 @@ unicode-normalization = { workspace = true } [dev-dependencies] ruff_annotate_snippets = { workspace = true } +ruff_python_ast = { workspace = true, features = ["serde"] } ruff_source_file = { workspace = true } anyhow = { workspace = true } insta = { workspace = true, features = ["glob"] } +serde = { workspace = true } +serde_json = { workspace = true } walkdir = { workspace = true } [lints] diff --git a/crates/ruff_python_parser/resources/inline/err/match_before_py310.py b/crates/ruff_python_parser/resources/inline/err/match_before_py310.py new file mode 100644 index 0000000000000..7cf41cdeced6c --- /dev/null +++ b/crates/ruff_python_parser/resources/inline/err/match_before_py310.py @@ -0,0 +1,4 @@ +# parse_options: { "target-version": "3.9" } +match 2: + case 1: + pass diff --git a/crates/ruff_python_parser/resources/inline/ok/match_after_py310.py b/crates/ruff_python_parser/resources/inline/ok/match_after_py310.py new file mode 100644 index 0000000000000..3c6690f1ae214 --- /dev/null +++ b/crates/ruff_python_parser/resources/inline/ok/match_after_py310.py @@ -0,0 +1,4 @@ +# parse_options: { "target-version": "3.10" } +match 2: + case 1: + pass diff --git a/crates/ruff_python_parser/src/parser/statement.rs b/crates/ruff_python_parser/src/parser/statement.rs index 5e5bee4ce252e..24bed309a3c79 100644 --- a/crates/ruff_python_parser/src/parser/statement.rs +++ b/crates/ruff_python_parser/src/parser/statement.rs @@ -2265,6 +2265,18 @@ impl<'src> Parser<'src> { let cases = self.parse_match_body(); + // test_err match_before_py310 + // # parse_options: { "target-version": "3.9" } + // match 2: + // case 1: + // pass + + // test_ok match_after_py310 + // # parse_options: { "target-version": "3.10" } + // match 2: + // case 1: + // pass + if self.options.target_version < PythonVersion::PY310 { self.unsupported_syntax_errors.push(UnsupportedSyntaxError { kind: UnsupportedSyntaxErrorKind::MatchBeforePy310, diff --git a/crates/ruff_python_parser/tests/fixtures.rs b/crates/ruff_python_parser/tests/fixtures.rs index 32ff8ce60f71e..219729a196ef8 100644 --- a/crates/ruff_python_parser/tests/fixtures.rs +++ b/crates/ruff_python_parser/tests/fixtures.rs @@ -5,7 +5,7 @@ use std::path::Path; use ruff_annotate_snippets::{Level, Renderer, Snippet}; use ruff_python_ast::visitor::source_order::{walk_module, SourceOrderVisitor, TraversalSignal}; -use ruff_python_ast::{AnyNodeRef, Mod}; +use ruff_python_ast::{AnyNodeRef, Mod, PythonVersion}; use ruff_python_parser::{parse_unchecked, Mode, ParseErrorType, ParseOptions, Token}; use ruff_source_file::{LineIndex, OneIndexed, SourceCode}; use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; @@ -34,9 +34,14 @@ fn inline_err() { /// Snapshots the AST. fn test_valid_syntax(input_path: &Path) { let source = fs::read_to_string(input_path).expect("Expected test file to exist"); - let parsed = parse_unchecked(&source, ParseOptions::from(Mode::Module)); + let options = extract_options(&source).unwrap_or_else(|| { + ParseOptions::from(Mode::Module).with_target_version(PythonVersion::latest()) + }); + let parsed = parse_unchecked(&source, options); - if !parsed.is_valid() { + let is_valid = parsed.is_valid() && parsed.unsupported_syntax_errors().is_empty(); + + if !is_valid { let line_index = LineIndex::from_source_text(&source); let source_code = SourceCode::new(&source, &line_index); @@ -55,6 +60,19 @@ fn test_valid_syntax(input_path: &Path) { .unwrap(); } + for error in parsed.unsupported_syntax_errors() { + writeln!( + &mut message, + "{}\n", + CodeFrame { + range: error.range, + error: &ParseErrorType::OtherError(error.to_string()), + source_code: &source_code, + } + ) + .unwrap(); + } + panic!("{input_path:?}: {message}"); } @@ -78,10 +96,15 @@ fn test_valid_syntax(input_path: &Path) { /// Snapshots the AST and the error messages. fn test_invalid_syntax(input_path: &Path) { let source = fs::read_to_string(input_path).expect("Expected test file to exist"); - let parsed = parse_unchecked(&source, ParseOptions::from(Mode::Module)); + let options = extract_options(&source).unwrap_or_else(|| { + ParseOptions::from(Mode::Module).with_target_version(PythonVersion::latest()) + }); + let parsed = parse_unchecked(&source, options); + + let is_valid = parsed.is_valid() && parsed.unsupported_syntax_errors().is_empty(); assert!( - !parsed.is_valid(), + !is_valid, "{input_path:?}: Expected parser to generate at least one syntax error for a program containing syntax errors." ); @@ -92,11 +115,13 @@ fn test_invalid_syntax(input_path: &Path) { writeln!(&mut output, "## AST").unwrap(); writeln!(&mut output, "\n```\n{:#?}\n```", parsed.syntax()).unwrap(); - writeln!(&mut output, "## Errors\n").unwrap(); - let line_index = LineIndex::from_source_text(&source); let source_code = SourceCode::new(&source, &line_index); + if !parsed.errors().is_empty() { + writeln!(&mut output, "## Errors\n").unwrap(); + } + for error in parsed.errors() { writeln!( &mut output, @@ -110,6 +135,23 @@ fn test_invalid_syntax(input_path: &Path) { .unwrap(); } + if !parsed.unsupported_syntax_errors().is_empty() { + writeln!(&mut output, "## Unsupported Syntax Errors\n").unwrap(); + } + + for error in parsed.unsupported_syntax_errors() { + writeln!( + &mut output, + "{}\n", + CodeFrame { + range: error.range, + error: &ParseErrorType::OtherError(error.to_string()), + source_code: &source_code, + } + ) + .unwrap(); + } + insta::with_settings!({ omit_expression => true, input_file => input_path, @@ -119,6 +161,53 @@ fn test_invalid_syntax(input_path: &Path) { }); } +/// Copy of [`ParseOptions`] for deriving [`Deserialize`] with serde as a dev-dependency. +#[derive(serde::Deserialize)] +#[serde(rename_all = "kebab-case")] +struct JsonParseOptions { + #[serde(default)] + mode: JsonMode, + #[serde(default)] + target_version: PythonVersion, +} + +/// Copy of [`Mode`] for deserialization. +#[derive(Default, serde::Deserialize)] +#[serde(rename_all = "kebab-case")] +enum JsonMode { + #[default] + Module, + Expression, + ParenthesizedExpression, + Ipython, +} + +impl From for ParseOptions { + fn from(value: JsonParseOptions) -> Self { + let mode = match value.mode { + JsonMode::Module => Mode::Module, + JsonMode::Expression => Mode::Expression, + JsonMode::ParenthesizedExpression => Mode::ParenthesizedExpression, + JsonMode::Ipython => Mode::Ipython, + }; + Self::from(mode).with_target_version(value.target_version) + } +} + +/// Extract [`ParseOptions`] from an initial pragma line, if present. +/// +/// For example, +/// +/// ```python +/// # parse_options: { "target-version": "3.10" } +/// def f(): ... +fn extract_options(source: &str) -> Option { + let header = source.lines().next()?; + let (_label, options) = header.split_once("# parse_options: ")?; + let options: Option = serde_json::from_str(options.trim()).ok(); + options.map(ParseOptions::from) +} + // Test that is intentionally ignored by default. // Use it for quickly debugging a parser issue. #[test] diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@match_before_py310.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@match_before_py310.py.snap new file mode 100644 index 0000000000000..a951295d8a809 --- /dev/null +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@match_before_py310.py.snap @@ -0,0 +1,63 @@ +--- +source: crates/ruff_python_parser/tests/fixtures.rs +input_file: crates/ruff_python_parser/resources/inline/err/match_before_py310.py +--- +## AST + +``` +Module( + ModModule { + range: 0..79, + body: [ + Match( + StmtMatch { + range: 45..78, + subject: NumberLiteral( + ExprNumberLiteral { + range: 51..52, + value: Int( + 2, + ), + }, + ), + cases: [ + MatchCase { + range: 58..78, + pattern: MatchValue( + PatternMatchValue { + range: 63..64, + value: NumberLiteral( + ExprNumberLiteral { + range: 63..64, + value: Int( + 1, + ), + }, + ), + }, + ), + guard: None, + body: [ + Pass( + StmtPass { + range: 74..78, + }, + ), + ], + }, + ], + }, + ), + ], + }, +) +``` +## Unsupported Syntax Errors + + | +1 | # parse_options: { "target-version": "3.9" } +2 | match 2: + | ^^^^^ Syntax Error: Cannot use `match` statement on Python 3.9 (syntax was added in Python 3.10) +3 | case 1: +4 | pass + | diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@match_after_py310.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@match_after_py310.py.snap new file mode 100644 index 0000000000000..7c12a5fce7488 --- /dev/null +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@match_after_py310.py.snap @@ -0,0 +1,54 @@ +--- +source: crates/ruff_python_parser/tests/fixtures.rs +input_file: crates/ruff_python_parser/resources/inline/ok/match_after_py310.py +--- +## AST + +``` +Module( + ModModule { + range: 0..80, + body: [ + Match( + StmtMatch { + range: 46..79, + subject: NumberLiteral( + ExprNumberLiteral { + range: 52..53, + value: Int( + 2, + ), + }, + ), + cases: [ + MatchCase { + range: 59..79, + pattern: MatchValue( + PatternMatchValue { + range: 64..65, + value: NumberLiteral( + ExprNumberLiteral { + range: 64..65, + value: Int( + 1, + ), + }, + ), + }, + ), + guard: None, + body: [ + Pass( + StmtPass { + range: 75..79, + }, + ), + ], + }, + ], + }, + ), + ], + }, +) +```