diff --git a/Cargo.lock b/Cargo.lock index cc0f080fd1..53b4f96922 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2282,29 +2282,6 @@ dependencies = [ "thiserror 2.0.17", ] -[[package]] -name = "ruff_python_codegen" -version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?tag=0.14.1#2bffef59665ce7d2630dfd72ee99846663660db8" -dependencies = [ - "ruff_python_ast", - "ruff_python_literal", - "ruff_python_parser", - "ruff_source_file", - "ruff_text_size", -] - -[[package]] -name = "ruff_python_literal" -version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?tag=0.14.1#2bffef59665ce7d2630dfd72ee99846663660db8" -dependencies = [ - "bitflags 2.9.4", - "itertools 0.14.0", - "ruff_python_ast", - "unic-ucd-category", -] - [[package]] name = "ruff_python_parser" version = "0.0.0" @@ -2410,9 +2387,7 @@ dependencies = [ "num-complex", "num-traits", "ruff_python_ast", - "ruff_python_codegen", "ruff_python_parser", - "ruff_source_file", "ruff_text_size", "rustpython-compiler-core", "rustpython-literal", diff --git a/Cargo.toml b/Cargo.toml index 77bba61a3b..3cdc471dc3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -159,7 +159,6 @@ rustpython-wtf8 = { path = "wtf8", version = "0.4.0" } rustpython-doc = { git = "https://github.com/RustPython/__doc__", tag = "0.3.0", version = "0.3.0" } ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" } -ruff_python_codegen = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" } ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" } ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" } ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" } diff --git a/Lib/test/test_future_stmt/badsyntax_future.py b/Lib/test/test_future_stmt/badsyntax_future10.py similarity index 100% rename from Lib/test/test_future_stmt/badsyntax_future.py rename to Lib/test/test_future_stmt/badsyntax_future10.py diff --git a/Lib/test/test_future_stmt/badsyntax_future3.py b/Lib/test/test_future_stmt/badsyntax_future3.py new file mode 100644 index 0000000000..f1c8417eda --- /dev/null +++ b/Lib/test/test_future_stmt/badsyntax_future3.py @@ -0,0 +1,10 @@ +"""This is a test""" +from __future__ import nested_scopes +from __future__ import rested_snopes + +def f(x): + def g(y): + return x + y + return g + +result = f(2)(4) diff --git a/Lib/test/test_future_stmt/badsyntax_future4.py b/Lib/test/test_future_stmt/badsyntax_future4.py new file mode 100644 index 0000000000..b5f4c98e92 --- /dev/null +++ b/Lib/test/test_future_stmt/badsyntax_future4.py @@ -0,0 +1,10 @@ +"""This is a test""" +import __future__ +from __future__ import nested_scopes + +def f(x): + def g(y): + return x + y + return g + +result = f(2)(4) diff --git a/Lib/test/test_future_stmt/badsyntax_future5.py b/Lib/test/test_future_stmt/badsyntax_future5.py new file mode 100644 index 0000000000..8a7e5fcb70 --- /dev/null +++ b/Lib/test/test_future_stmt/badsyntax_future5.py @@ -0,0 +1,12 @@ +"""This is a test""" +from __future__ import nested_scopes +import foo +from __future__ import nested_scopes + + +def f(x): + def g(y): + return x + y + return g + +result = f(2)(4) diff --git a/Lib/test/test_future_stmt/badsyntax_future6.py b/Lib/test/test_future_stmt/badsyntax_future6.py new file mode 100644 index 0000000000..5a8b55a02c --- /dev/null +++ b/Lib/test/test_future_stmt/badsyntax_future6.py @@ -0,0 +1,10 @@ +"""This is a test""" +"this isn't a doc string" +from __future__ import nested_scopes + +def f(x): + def g(y): + return x + y + return g + +result = f(2)(4) diff --git a/Lib/test/test_future_stmt/badsyntax_future7.py b/Lib/test/test_future_stmt/badsyntax_future7.py new file mode 100644 index 0000000000..131db2c216 --- /dev/null +++ b/Lib/test/test_future_stmt/badsyntax_future7.py @@ -0,0 +1,11 @@ +"""This is a test""" + +from __future__ import nested_scopes; import string; from __future__ import \ + nested_scopes + +def f(x): + def g(y): + return x + y + return g + +result = f(2)(4) diff --git a/Lib/test/test_future_stmt/badsyntax_future8.py b/Lib/test/test_future_stmt/badsyntax_future8.py new file mode 100644 index 0000000000..ca45289e2e --- /dev/null +++ b/Lib/test/test_future_stmt/badsyntax_future8.py @@ -0,0 +1,10 @@ +"""This is a test""" + +from __future__ import * + +def f(x): + def g(y): + return x + y + return g + +print(f(2)(4)) diff --git a/Lib/test/test_future_stmt/badsyntax_future9.py b/Lib/test/test_future_stmt/badsyntax_future9.py new file mode 100644 index 0000000000..916de06ab7 --- /dev/null +++ b/Lib/test/test_future_stmt/badsyntax_future9.py @@ -0,0 +1,10 @@ +"""This is a test""" + +from __future__ import nested_scopes, braces + +def f(x): + def g(y): + return x + y + return g + +print(f(2)(4)) diff --git a/Lib/test/test_future_stmt/import_nested_scope_twice.py b/Lib/test/test_future_stmt/future_test1.py similarity index 100% rename from Lib/test/test_future_stmt/import_nested_scope_twice.py rename to Lib/test/test_future_stmt/future_test1.py diff --git a/Lib/test/test_future_stmt/nested_scope.py b/Lib/test/test_future_stmt/future_test2.py similarity index 100% rename from Lib/test/test_future_stmt/nested_scope.py rename to Lib/test/test_future_stmt/future_test2.py diff --git a/Lib/test/test_future_stmt/test_future.py b/Lib/test/test_future_stmt/test_future.py index e6e6201a76..9c30054963 100644 --- a/Lib/test/test_future_stmt/test_future.py +++ b/Lib/test/test_future_stmt/test_future.py @@ -10,8 +10,6 @@ import re import sys -TOP_LEVEL_MSG = 'from __future__ imports must occur at the beginning of the file' - rx = re.compile(r'\((\S+).py, line (\d+)') def get_error_location(msg): @@ -20,48 +18,21 @@ def get_error_location(msg): class FutureTest(unittest.TestCase): - def check_syntax_error(self, err, basename, - *, - lineno, - message=TOP_LEVEL_MSG, offset=1): - if basename != '': - basename += '.py' - - self.assertEqual(f'{message} ({basename}, line {lineno})', str(err)) - self.assertEqual(os.path.basename(err.filename), basename) + def check_syntax_error(self, err, basename, lineno, offset=1): + self.assertIn('%s.py, line %d' % (basename, lineno), str(err)) + self.assertEqual(os.path.basename(err.filename), basename + '.py') self.assertEqual(err.lineno, lineno) self.assertEqual(err.offset, offset) - def assertSyntaxError(self, code, - *, - lineno=1, - message=TOP_LEVEL_MSG, offset=1, - parametrize_docstring=True): - code = dedent(code.lstrip('\n')) - for add_docstring in ([False, True] if parametrize_docstring else [False]): - with self.subTest(code=code, add_docstring=add_docstring): - if add_docstring: - code = '"""Docstring"""\n' + code - lineno += 1 - with self.assertRaises(SyntaxError) as cm: - exec(code) - self.check_syntax_error(cm.exception, "", - lineno=lineno, - message=message, - offset=offset) - - def test_import_nested_scope_twice(self): - # Import the name nested_scopes twice to trigger SF bug #407394 - with import_helper.CleanImport( - 'test.test_future_stmt.import_nested_scope_twice', - ): - from test.test_future_stmt import import_nested_scope_twice - self.assertEqual(import_nested_scope_twice.result, 6) + def test_future1(self): + with import_helper.CleanImport('test.test_future_stmt.future_test1'): + from test.test_future_stmt import future_test1 + self.assertEqual(future_test1.result, 6) - def test_nested_scope(self): - with import_helper.CleanImport('test.test_future_stmt.nested_scope'): - from test.test_future_stmt import nested_scope - self.assertEqual(nested_scope.result, 6) + def test_future2(self): + with import_helper.CleanImport('test.test_future_stmt.future_test2'): + from test.test_future_stmt import future_test2 + self.assertEqual(future_test2.result, 6) def test_future_single_import(self): with import_helper.CleanImport( @@ -81,87 +52,47 @@ def test_future_multiple_features(self): ): from test.test_future_stmt import test_future_multiple_features - @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message - def test_unknown_future_flag(self): - code = """ - from __future__ import nested_scopes - from __future__ import rested_snopes # typo error here: nested => rested - """ - self.assertSyntaxError( - code, lineno=2, - message='future feature rested_snopes is not defined', offset=24, - ) - - @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message - def test_future_import_not_on_top(self): - code = """ - import some_module - from __future__ import annotations - """ - self.assertSyntaxError(code, lineno=2) + def test_badfuture3(self): + with self.assertRaises(SyntaxError) as cm: + from test.test_future_stmt import badsyntax_future3 + self.check_syntax_error(cm.exception, "badsyntax_future3", 3) - code = """ - import __future__ - from __future__ import annotations - """ - self.assertSyntaxError(code, lineno=2) + def test_badfuture4(self): + with self.assertRaises(SyntaxError) as cm: + from test.test_future_stmt import badsyntax_future4 + self.check_syntax_error(cm.exception, "badsyntax_future4", 3) - code = """ - from __future__ import absolute_import - "spam, bar, blah" - from __future__ import print_function - """ - self.assertSyntaxError(code, lineno=3) - - @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message - def test_future_import_with_extra_string(self): - code = """ - '''Docstring''' - "this isn't a doc string" - from __future__ import nested_scopes - """ - self.assertSyntaxError(code, lineno=3, parametrize_docstring=False) - - @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message - def test_multiple_import_statements_on_same_line(self): - # With `\`: - code = """ - from __future__ import nested_scopes; import string; from __future__ import \ - nested_scopes - """ - self.assertSyntaxError(code, offset=54) + def test_badfuture5(self): + with self.assertRaises(SyntaxError) as cm: + from test.test_future_stmt import badsyntax_future5 + self.check_syntax_error(cm.exception, "badsyntax_future5", 4) - # Without `\`: - code = """ - from __future__ import nested_scopes; import string; from __future__ import nested_scopes - """ - self.assertSyntaxError(code, offset=54) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_badfuture6(self): + with self.assertRaises(SyntaxError) as cm: + from test.test_future_stmt import badsyntax_future6 + self.check_syntax_error(cm.exception, "badsyntax_future6", 3) - @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message - def test_future_import_star(self): - code = """ - from __future__ import * - """ - self.assertSyntaxError(code, message='future feature * is not defined', offset=24) + def test_badfuture7(self): + with self.assertRaises(SyntaxError) as cm: + from test.test_future_stmt import badsyntax_future7 + self.check_syntax_error(cm.exception, "badsyntax_future7", 3, 54) - @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message - def test_future_import_braces(self): - code = """ - from __future__ import braces - """ - # Congrats, you found an easter egg! - self.assertSyntaxError(code, message='not a chance', offset=24) + def test_badfuture8(self): + with self.assertRaises(SyntaxError) as cm: + from test.test_future_stmt import badsyntax_future8 + self.check_syntax_error(cm.exception, "badsyntax_future8", 3) - code = """ - from __future__ import nested_scopes, braces - """ - self.assertSyntaxError(code, message='not a chance', offset=39) + def test_badfuture9(self): + with self.assertRaises(SyntaxError) as cm: + from test.test_future_stmt import badsyntax_future9 + self.check_syntax_error(cm.exception, "badsyntax_future9", 3) - @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message - def test_module_with_future_import_not_on_top(self): + def test_badfuture10(self): with self.assertRaises(SyntaxError) as cm: - from test.test_future_stmt import badsyntax_future - self.check_syntax_error(cm.exception, "badsyntax_future", lineno=3) + from test.test_future_stmt import badsyntax_future10 + self.check_syntax_error(cm.exception, "badsyntax_future10", 3) def test_ensure_flags_dont_clash(self): # bpo-39562: test that future flags and compiler flags doesn't clash @@ -178,6 +109,26 @@ def test_ensure_flags_dont_clash(self): } self.assertCountEqual(set(flags.values()), flags.values()) + def test_parserhack(self): + # test that the parser.c::future_hack function works as expected + # Note: although this test must pass, it's not testing the original + # bug as of 2.6 since the with statement is not optional and + # the parser hack disabled. If a new keyword is introduced in + # 2.6, change this to refer to the new future import. + try: + exec("from __future__ import print_function; print 0") + except SyntaxError: + pass + else: + self.fail("syntax error didn't occur") + + try: + exec("from __future__ import (print_function); print 0") + except SyntaxError: + pass + else: + self.fail("syntax error didn't occur") + def test_unicode_literals_exec(self): scope = {} exec("from __future__ import unicode_literals; x = ''", {}, scope) @@ -190,26 +141,6 @@ def test_syntactical_future_repl(self): out = kill_python(p) self.assertNotIn(b'SyntaxError: invalid syntax', out) - @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError: future feature spam is not defined - def test_future_dotted_import(self): - with self.assertRaises(ImportError): - exec("from .__future__ import spam") - - code = dedent( - """ - from __future__ import print_function - from ...__future__ import ham - """ - ) - with self.assertRaises(ImportError): - exec(code) - - code = """ - from .__future__ import nested_scopes - from __future__ import barry_as_FLUFL - """ - self.assertSyntaxError(code, lineno=2) - class AnnotationsFutureTestCase(unittest.TestCase): template = dedent( """ @@ -267,7 +198,6 @@ def _exec_future(self, code): ) return scope - @unittest.expectedFailure # TODO: RUSTPYTHON; 'a,' != '(a,)' def test_annotations(self): eq = self.assertAnnotationEqual eq('...') @@ -432,7 +362,6 @@ def test_annotations(self): eq('(((a, b)))', '(a, b)') eq("1 + 2 + 3") - @unittest.expectedFailure # TODO: RUSTPYTHON; "f'{x=!r}'" != "f'x={x!r}'" def test_fstring_debug_annotations(self): # f-strings with '=' don't round trip very well, so set the expected # result explicitly. @@ -443,7 +372,6 @@ def test_fstring_debug_annotations(self): self.assertAnnotationEqual("f'{x=!a}'", expected="f'x={x!a}'") self.assertAnnotationEqual("f'{x=!s:*^20}'", expected="f'x={x!s:*^20}'") - @unittest.expectedFailure # TODO: RUSTPYTHON; '1e309, 1e309j' != '(1e309, 1e309j)' def test_infinity_numbers(self): inf = "1e" + repr(sys.float_info.max_10_exp + 1) infj = f"{inf}j" @@ -456,7 +384,8 @@ def test_infinity_numbers(self): self.assertAnnotationEqual("('inf', 1e1000, 'infxxx', 1e1000j)", expected=f"('inf', {inf}, 'infxxx', {infj})") self.assertAnnotationEqual("(1e1000, (1e1000j,))", expected=f"({inf}, ({infj},))") - @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_annotation_with_complex_target(self): with self.assertRaises(SyntaxError): exec( @@ -480,7 +409,8 @@ def bar(): self.assertEqual(foo.__code__.co_cellvars, ()) self.assertEqual(foo().__code__.co_freevars, ()) - @unittest.expectedFailure # TODO: RUSTPYTHON; "f'{x=!r}'" != "f'x={x!r}'" + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_annotations_forbidden(self): with self.assertRaises(SyntaxError): self._exec_future("test: (yield)") diff --git a/compiler/codegen/Cargo.toml b/compiler/codegen/Cargo.toml index e2bfec6c3c..ce7e8d74f5 100644 --- a/compiler/codegen/Cargo.toml +++ b/compiler/codegen/Cargo.toml @@ -13,9 +13,6 @@ rustpython-compiler-core = { workspace = true } rustpython-literal = {workspace = true } rustpython-wtf8 = { workspace = true } ruff_python_ast = { workspace = true } -ruff_python_parser = { workspace = true } -ruff_python_codegen = { workspace = true } -ruff_source_file = { workspace = true } ruff_text_size = { workspace = true } ahash = { workspace = true } diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index 132998fe83..5f8caebf27 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -14,6 +14,7 @@ use crate::{ error::{CodegenError, CodegenErrorType, InternalError, PatternUnreachableReason}, ir::{self, BlockIdx}, symboltable::{self, CompilerScope, SymbolFlags, SymbolScope, SymbolTable}, + unparse::UnparseExpr, }; use itertools::Itertools; use malachite_bigint::BigInt; @@ -30,7 +31,6 @@ use ruff_python_ast::{ PatternMatchStar, PatternMatchValue, Singleton, Stmt, StmtExpr, TypeParam, TypeParamParamSpec, TypeParamTypeVar, TypeParamTypeVarTuple, TypeParams, UnaryOp, WithItem, }; -use ruff_source_file::LineEnding; use ruff_text_size::{Ranged, TextRange}; use rustpython_compiler_core::{ Mode, OneIndexed, PositionEncoding, SourceFile, SourceLocation, @@ -147,15 +147,6 @@ enum ComprehensionType { Dict, } -fn unparse_expr(expr: &Expr) -> String { - use ruff_python_ast::str::Quote; - use ruff_python_codegen::{Generator, Indentation}; - - Generator::new(&Indentation::default(), LineEnding::default()) - .with_preferred_quote(Some(Quote::Single)) - .expr(expr) -} - fn validate_duplicate_params(params: &Parameters) -> Result<(), CodegenErrorType> { let mut seen_params = HashSet::new(); for param in params { @@ -3618,7 +3609,7 @@ impl Compiler { | Expr::NoneLiteral(_) ); let key_repr = if is_literal { - unparse_expr(key) + UnparseExpr::new(key, &self.source_file).to_string() } else if is_attribute { String::new() } else { @@ -4172,7 +4163,9 @@ impl Compiler { fn compile_annotation(&mut self, annotation: &Expr) -> CompileResult<()> { if self.future_annotations { self.emit_load_const(ConstantData::Str { - value: unparse_expr(annotation).into(), + value: UnparseExpr::new(annotation, &self.source_file) + .to_string() + .into(), }); } else { let was_in_annotation = self.in_annotation; diff --git a/compiler/codegen/src/lib.rs b/compiler/codegen/src/lib.rs index e794407998..291b57d7f6 100644 --- a/compiler/codegen/src/lib.rs +++ b/compiler/codegen/src/lib.rs @@ -13,6 +13,7 @@ pub mod error; pub mod ir; mod string_parser; pub mod symboltable; +mod unparse; pub use compile::CompileOpts; use ruff_python_ast::Expr;