From 38a73167bc279944fbf33f54c4cc18118f980e1d Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Wed, 3 Sep 2025 20:16:28 +0300 Subject: [PATCH 1/8] Use ruff for unparse backend --- Cargo.lock | 25 ++ Cargo.toml | 1 + compiler/codegen/Cargo.toml | 3 + compiler/codegen/src/compile.rs | 21 +- compiler/codegen/src/lib.rs | 1 - compiler/codegen/src/unparse.rs | 633 -------------------------------- 6 files changed, 45 insertions(+), 639 deletions(-) delete mode 100644 compiler/codegen/src/unparse.rs diff --git a/Cargo.lock b/Cargo.lock index 94805fd5b6..deb766185a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2176,6 +2176,29 @@ dependencies = [ "rustc-hash", ] +[[package]] +name = "ruff_python_codegen" +version = "0.0.0" +source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" +dependencies = [ + "ruff_python_ast", + "ruff_python_literal", + "ruff_python_parser", + "ruff_source_file", + "ruff_text_size", +] + +[[package]] +name = "ruff_python_literal" +version = "0.0.0" +source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" +dependencies = [ + "bitflags 2.9.3", + "itertools 0.14.0", + "ruff_python_ast", + "unic-ucd-category", +] + [[package]] name = "ruff_python_parser" version = "0.0.0" @@ -2277,7 +2300,9 @@ dependencies = [ "num-complex", "num-traits", "ruff_python_ast", + "ruff_python_codegen", "ruff_python_parser", + "ruff_source_file", "ruff_text_size", "rustpython-compiler-core", "rustpython-literal", diff --git a/Cargo.toml b/Cargo.toml index c373860324..70ceebf7a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -162,6 +162,7 @@ ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "0.1 ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } +ruff_python_codegen = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } ahash = "0.8.11" ascii = "1.1" diff --git a/compiler/codegen/Cargo.toml b/compiler/codegen/Cargo.toml index ce7e8d74f5..e2bfec6c3c 100644 --- a/compiler/codegen/Cargo.toml +++ b/compiler/codegen/Cargo.toml @@ -13,6 +13,9 @@ rustpython-compiler-core = { workspace = true } rustpython-literal = {workspace = true } rustpython-wtf8 = { workspace = true } ruff_python_ast = { workspace = true } +ruff_python_parser = { workspace = true } +ruff_python_codegen = { workspace = true } +ruff_source_file = { workspace = true } ruff_text_size = { workspace = true } ahash = { workspace = true } diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index c2bdf0a6e0..a0c3a3b92e 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -14,7 +14,6 @@ use crate::{ error::{CodegenError, CodegenErrorType, InternalError, PatternUnreachableReason}, ir::{self, BlockIdx}, symboltable::{self, CompilerScope, SymbolFlags, SymbolScope, SymbolTable}, - unparse::UnparseExpr, }; use itertools::Itertools; use malachite_bigint::BigInt; @@ -31,6 +30,7 @@ use ruff_python_ast::{ TypeParam, TypeParamParamSpec, TypeParamTypeVar, TypeParamTypeVarTuple, TypeParams, UnaryOp, WithItem, }; +use ruff_source_file::LineEnding; use ruff_text_size::{Ranged, TextRange}; use rustpython_compiler_core::{ Mode, OneIndexed, SourceFile, SourceLocation, @@ -147,6 +147,19 @@ enum ComprehensionType { Dict, } +fn unparse_expr(expr: &Expr) -> String { + // Hack, because we can't do `ruff_python_codegen::Indentation::default()` + // https://github.com/astral-sh/ruff/pull/20216 + let indentation = { + let contents = r"x = 1"; + let module = ruff_python_parser::parse_module(contents).unwrap(); + let stylist = ruff_python_codegen::Stylist::from_tokens(module.tokens(), contents); + stylist.indentation().clone() + }; + + ruff_python_codegen::Generator::new(&indentation, LineEnding::default()).expr(expr) +} + /// Compile an Mod produced from ruff parser pub fn compile_top( ast: ruff_python_ast::Mod, @@ -3592,7 +3605,7 @@ impl Compiler { | Expr::NoneLiteral(_) ); let key_repr = if is_literal { - UnparseExpr::new(key, &self.source_file).to_string() + unparse_expr(key) } else if is_attribute { String::new() } else { @@ -4146,9 +4159,7 @@ impl Compiler { fn compile_annotation(&mut self, annotation: &Expr) -> CompileResult<()> { if self.future_annotations { self.emit_load_const(ConstantData::Str { - value: UnparseExpr::new(annotation, &self.source_file) - .to_string() - .into(), + value: unparse_expr(annotation).into(), }); } else { let was_in_annotation = self.in_annotation; diff --git a/compiler/codegen/src/lib.rs b/compiler/codegen/src/lib.rs index 9b444de994..b936ffbd1a 100644 --- a/compiler/codegen/src/lib.rs +++ b/compiler/codegen/src/lib.rs @@ -13,7 +13,6 @@ pub mod error; pub mod ir; mod string_parser; pub mod symboltable; -mod unparse; pub use compile::CompileOpts; use ruff_python_ast::Expr; diff --git a/compiler/codegen/src/unparse.rs b/compiler/codegen/src/unparse.rs deleted file mode 100644 index f2fb86ed4e..0000000000 --- a/compiler/codegen/src/unparse.rs +++ /dev/null @@ -1,633 +0,0 @@ -use ruff_python_ast::{ - self as ruff, Arguments, BoolOp, Comprehension, ConversionFlag, Expr, Identifier, Operator, - Parameter, ParameterWithDefault, Parameters, -}; -use ruff_text_size::Ranged; -use rustpython_compiler_core::SourceFile; -use rustpython_literal::escape::{AsciiEscape, UnicodeEscape}; -use std::fmt::{self, Display as _}; - -mod precedence { - macro_rules! precedence { - ($($op:ident,)*) => { - precedence!(@0, $($op,)*); - }; - (@$i:expr, $op1:ident, $($op:ident,)*) => { - pub const $op1: u8 = $i; - precedence!(@$i + 1, $($op,)*); - }; - (@$i:expr,) => {}; - } - precedence!( - TUPLE, TEST, OR, AND, NOT, CMP, // "EXPR" = - BOR, BXOR, BAND, SHIFT, ARITH, TERM, FACTOR, POWER, AWAIT, ATOM, - ); - pub const EXPR: u8 = BOR; -} - -struct Unparser<'a, 'b, 'c> { - f: &'b mut fmt::Formatter<'a>, - source: &'c SourceFile, -} - -impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { - const fn new(f: &'b mut fmt::Formatter<'a>, source: &'c SourceFile) -> Self { - Self { f, source } - } - - fn p(&mut self, s: &str) -> fmt::Result { - self.f.write_str(s) - } - - fn p_id(&mut self, s: &Identifier) -> fmt::Result { - self.f.write_str(s.as_str()) - } - - fn p_if(&mut self, cond: bool, s: &str) -> fmt::Result { - if cond { - self.f.write_str(s)?; - } - Ok(()) - } - - fn p_delim(&mut self, first: &mut bool, s: &str) -> fmt::Result { - self.p_if(!std::mem::take(first), s) - } - - fn write_fmt(&mut self, f: fmt::Arguments<'_>) -> fmt::Result { - self.f.write_fmt(f) - } - - fn unparse_expr(&mut self, ast: &Expr, level: u8) -> fmt::Result { - macro_rules! op_prec { - ($op_ty:ident, $x:expr, $enu:path, $($var:ident($op:literal, $prec:ident)),*$(,)?) => { - match $x { - $(<$enu>::$var => (op_prec!(@space $op_ty, $op), precedence::$prec),)* - } - }; - (@space bin, $op:literal) => { - concat!(" ", $op, " ") - }; - (@space un, $op:literal) => { - $op - }; - } - macro_rules! group_if { - ($lvl:expr, $body:block) => {{ - let group = level > $lvl; - self.p_if(group, "(")?; - let ret = $body; - self.p_if(group, ")")?; - ret - }}; - } - match &ast { - Expr::BoolOp(ruff::ExprBoolOp { - op, - values, - range: _range, - }) => { - let (op, prec) = op_prec!(bin, op, BoolOp, And("and", AND), Or("or", OR)); - group_if!(prec, { - let mut first = true; - for val in values { - self.p_delim(&mut first, op)?; - self.unparse_expr(val, prec + 1)?; - } - }) - } - Expr::Named(ruff::ExprNamed { - target, - value, - range: _range, - }) => { - group_if!(precedence::TUPLE, { - self.unparse_expr(target, precedence::ATOM)?; - self.p(" := ")?; - self.unparse_expr(value, precedence::ATOM)?; - }) - } - Expr::BinOp(ruff::ExprBinOp { - left, - op, - right, - range: _range, - }) => { - let right_associative = matches!(op, Operator::Pow); - let (op, prec) = op_prec!( - bin, - op, - Operator, - Add("+", ARITH), - Sub("-", ARITH), - Mult("*", TERM), - MatMult("@", TERM), - Div("/", TERM), - Mod("%", TERM), - Pow("**", POWER), - LShift("<<", SHIFT), - RShift(">>", SHIFT), - BitOr("|", BOR), - BitXor("^", BXOR), - BitAnd("&", BAND), - FloorDiv("//", TERM), - ); - group_if!(prec, { - self.unparse_expr(left, prec + right_associative as u8)?; - self.p(op)?; - self.unparse_expr(right, prec + !right_associative as u8)?; - }) - } - Expr::UnaryOp(ruff::ExprUnaryOp { - op, - operand, - range: _range, - }) => { - let (op, prec) = op_prec!( - un, - op, - ruff::UnaryOp, - Invert("~", FACTOR), - Not("not ", NOT), - UAdd("+", FACTOR), - USub("-", FACTOR) - ); - group_if!(prec, { - self.p(op)?; - self.unparse_expr(operand, prec)?; - }) - } - Expr::Lambda(ruff::ExprLambda { - parameters, - body, - range: _range, - }) => { - group_if!(precedence::TEST, { - if let Some(parameters) = parameters { - self.p("lambda ")?; - self.unparse_arguments(parameters)?; - } else { - self.p("lambda")?; - } - write!(self, ": {}", UnparseExpr::new(body, self.source))?; - }) - } - Expr::If(ruff::ExprIf { - test, - body, - orelse, - range: _range, - }) => { - group_if!(precedence::TEST, { - self.unparse_expr(body, precedence::TEST + 1)?; - self.p(" if ")?; - self.unparse_expr(test, precedence::TEST + 1)?; - self.p(" else ")?; - self.unparse_expr(orelse, precedence::TEST)?; - }) - } - Expr::Dict(ruff::ExprDict { - items, - range: _range, - }) => { - self.p("{")?; - let mut first = true; - for item in items { - self.p_delim(&mut first, ", ")?; - if let Some(k) = &item.key { - write!(self, "{}: ", UnparseExpr::new(k, self.source))?; - } else { - self.p("**")?; - } - self.unparse_expr(&item.value, level)?; - } - self.p("}")?; - } - Expr::Set(ruff::ExprSet { - elts, - range: _range, - }) => { - self.p("{")?; - let mut first = true; - for v in elts { - self.p_delim(&mut first, ", ")?; - self.unparse_expr(v, precedence::TEST)?; - } - self.p("}")?; - } - Expr::ListComp(ruff::ExprListComp { - elt, - generators, - range: _range, - }) => { - self.p("[")?; - self.unparse_expr(elt, precedence::TEST)?; - self.unparse_comp(generators)?; - self.p("]")?; - } - Expr::SetComp(ruff::ExprSetComp { - elt, - generators, - range: _range, - }) => { - self.p("{")?; - self.unparse_expr(elt, precedence::TEST)?; - self.unparse_comp(generators)?; - self.p("}")?; - } - Expr::DictComp(ruff::ExprDictComp { - key, - value, - generators, - range: _range, - }) => { - self.p("{")?; - self.unparse_expr(key, precedence::TEST)?; - self.p(": ")?; - self.unparse_expr(value, precedence::TEST)?; - self.unparse_comp(generators)?; - self.p("}")?; - } - Expr::Generator(ruff::ExprGenerator { - parenthesized: _, - elt, - generators, - range: _range, - }) => { - self.p("(")?; - self.unparse_expr(elt, precedence::TEST)?; - self.unparse_comp(generators)?; - self.p(")")?; - } - Expr::Await(ruff::ExprAwait { - value, - range: _range, - }) => { - group_if!(precedence::AWAIT, { - self.p("await ")?; - self.unparse_expr(value, precedence::ATOM)?; - }) - } - Expr::Yield(ruff::ExprYield { - value, - range: _range, - }) => { - if let Some(value) = value { - write!(self, "(yield {})", UnparseExpr::new(value, self.source))?; - } else { - self.p("(yield)")?; - } - } - Expr::YieldFrom(ruff::ExprYieldFrom { - value, - range: _range, - }) => { - write!( - self, - "(yield from {})", - UnparseExpr::new(value, self.source) - )?; - } - Expr::Compare(ruff::ExprCompare { - left, - ops, - comparators, - range: _range, - }) => { - group_if!(precedence::CMP, { - let new_lvl = precedence::CMP + 1; - self.unparse_expr(left, new_lvl)?; - for (op, cmp) in ops.iter().zip(comparators) { - self.p(" ")?; - self.p(op.as_str())?; - self.p(" ")?; - self.unparse_expr(cmp, new_lvl)?; - } - }) - } - Expr::Call(ruff::ExprCall { - func, - arguments: Arguments { args, keywords, .. }, - range: _range, - }) => { - self.unparse_expr(func, precedence::ATOM)?; - self.p("(")?; - if let ( - [ - Expr::Generator(ruff::ExprGenerator { - elt, - generators, - range: _range, - .. - }), - ], - [], - ) = (&**args, &**keywords) - { - // make sure a single genexpr doesn't get double parens - self.unparse_expr(elt, precedence::TEST)?; - self.unparse_comp(generators)?; - } else { - let mut first = true; - for arg in args { - self.p_delim(&mut first, ", ")?; - self.unparse_expr(arg, precedence::TEST)?; - } - for kw in keywords { - self.p_delim(&mut first, ", ")?; - if let Some(arg) = &kw.arg { - self.p_id(arg)?; - self.p("=")?; - } else { - self.p("**")?; - } - self.unparse_expr(&kw.value, precedence::TEST)?; - } - } - self.p(")")?; - } - Expr::FString(ruff::ExprFString { value, .. }) => self.unparse_fstring(value)?, - Expr::StringLiteral(ruff::ExprStringLiteral { value, .. }) => { - if value.is_unicode() { - self.p("u")? - } - UnicodeEscape::new_repr(value.to_str().as_ref()) - .str_repr() - .fmt(self.f)? - } - Expr::BytesLiteral(ruff::ExprBytesLiteral { value, .. }) => { - AsciiEscape::new_repr(&value.bytes().collect::>()) - .bytes_repr() - .fmt(self.f)? - } - Expr::NumberLiteral(ruff::ExprNumberLiteral { value, .. }) => { - const { assert!(f64::MAX_10_EXP == 308) }; - let inf_str = "1e309"; - match value { - ruff::Number::Int(int) => int.fmt(self.f)?, - &ruff::Number::Float(fp) => { - if fp.is_infinite() { - self.p(inf_str)? - } else { - self.p(&rustpython_literal::float::to_string(fp))? - } - } - &ruff::Number::Complex { real, imag } => self - .p(&rustpython_literal::complex::to_string(real, imag) - .replace("inf", inf_str))?, - } - } - Expr::BooleanLiteral(ruff::ExprBooleanLiteral { value, .. }) => { - self.p(if *value { "True" } else { "False" })? - } - Expr::NoneLiteral(ruff::ExprNoneLiteral { .. }) => self.p("None")?, - Expr::EllipsisLiteral(ruff::ExprEllipsisLiteral { .. }) => self.p("...")?, - Expr::Attribute(ruff::ExprAttribute { value, attr, .. }) => { - self.unparse_expr(value, precedence::ATOM)?; - let period = if let Expr::NumberLiteral(ruff::ExprNumberLiteral { - value: ruff::Number::Int(_), - .. - }) = value.as_ref() - { - " ." - } else { - "." - }; - self.p(period)?; - self.p_id(attr)?; - } - Expr::Subscript(ruff::ExprSubscript { value, slice, .. }) => { - self.unparse_expr(value, precedence::ATOM)?; - let lvl = precedence::TUPLE; - self.p("[")?; - self.unparse_expr(slice, lvl)?; - self.p("]")?; - } - Expr::Starred(ruff::ExprStarred { value, .. }) => { - self.p("*")?; - self.unparse_expr(value, precedence::EXPR)?; - } - Expr::Name(ruff::ExprName { id, .. }) => self.p(id.as_str())?, - Expr::List(ruff::ExprList { elts, .. }) => { - self.p("[")?; - let mut first = true; - for elt in elts { - self.p_delim(&mut first, ", ")?; - self.unparse_expr(elt, precedence::TEST)?; - } - self.p("]")?; - } - Expr::Tuple(ruff::ExprTuple { elts, .. }) => { - if elts.is_empty() { - self.p("()")?; - } else { - group_if!(precedence::TUPLE, { - let mut first = true; - for elt in elts { - self.p_delim(&mut first, ", ")?; - self.unparse_expr(elt, precedence::TEST)?; - } - self.p_if(elts.len() == 1, ",")?; - }) - } - } - Expr::Slice(ruff::ExprSlice { - lower, - upper, - step, - range: _range, - }) => { - if let Some(lower) = lower { - self.unparse_expr(lower, precedence::TEST)?; - } - self.p(":")?; - if let Some(upper) = upper { - self.unparse_expr(upper, precedence::TEST)?; - } - if let Some(step) = step { - self.p(":")?; - self.unparse_expr(step, precedence::TEST)?; - } - } - Expr::IpyEscapeCommand(_) => {} - } - Ok(()) - } - - fn unparse_arguments(&mut self, args: &Parameters) -> fmt::Result { - let mut first = true; - for (i, arg) in args.posonlyargs.iter().chain(&args.args).enumerate() { - self.p_delim(&mut first, ", ")?; - self.unparse_function_arg(arg)?; - self.p_if(i + 1 == args.posonlyargs.len(), ", /")?; - } - if args.vararg.is_some() || !args.kwonlyargs.is_empty() { - self.p_delim(&mut first, ", ")?; - self.p("*")?; - } - if let Some(vararg) = &args.vararg { - self.unparse_arg(vararg)?; - } - for kwarg in &args.kwonlyargs { - self.p_delim(&mut first, ", ")?; - self.unparse_function_arg(kwarg)?; - } - if let Some(kwarg) = &args.kwarg { - self.p_delim(&mut first, ", ")?; - self.p("**")?; - self.unparse_arg(kwarg)?; - } - Ok(()) - } - fn unparse_function_arg(&mut self, arg: &ParameterWithDefault) -> fmt::Result { - self.unparse_arg(&arg.parameter)?; - if let Some(default) = &arg.default { - write!(self, "={}", UnparseExpr::new(default, self.source))?; - } - Ok(()) - } - - fn unparse_arg(&mut self, arg: &Parameter) -> fmt::Result { - self.p_id(&arg.name)?; - if let Some(ann) = &arg.annotation { - write!(self, ": {}", UnparseExpr::new(ann, self.source))?; - } - Ok(()) - } - - fn unparse_comp(&mut self, generators: &[Comprehension]) -> fmt::Result { - for comp in generators { - self.p(if comp.is_async { - " async for " - } else { - " for " - })?; - self.unparse_expr(&comp.target, precedence::TUPLE)?; - self.p(" in ")?; - self.unparse_expr(&comp.iter, precedence::TEST + 1)?; - for cond in &comp.ifs { - self.p(" if ")?; - self.unparse_expr(cond, precedence::TEST + 1)?; - } - } - Ok(()) - } - - fn unparse_fstring_body(&mut self, elements: &[ruff::FStringElement]) -> fmt::Result { - for elem in elements { - self.unparse_fstring_elem(elem)?; - } - Ok(()) - } - - fn unparse_formatted( - &mut self, - val: &Expr, - debug_text: Option<&ruff::DebugText>, - conversion: ConversionFlag, - spec: Option<&ruff::FStringFormatSpec>, - ) -> fmt::Result { - let buffered = to_string_fmt(|f| { - Unparser::new(f, self.source).unparse_expr(val, precedence::TEST + 1) - }); - if let Some(ruff::DebugText { leading, trailing }) = debug_text { - self.p(leading)?; - self.p(self.source.slice(val.range()))?; - self.p(trailing)?; - } - let brace = if buffered.starts_with('{') { - // put a space to avoid escaping the bracket - "{ " - } else { - "{" - }; - self.p(brace)?; - self.p(&buffered)?; - drop(buffered); - - if conversion != ConversionFlag::None { - self.p("!")?; - let buf = &[conversion as u8]; - let c = std::str::from_utf8(buf).unwrap(); - self.p(c)?; - } - - if let Some(spec) = spec { - self.p(":")?; - self.unparse_fstring_body(&spec.elements)?; - } - - self.p("}")?; - - Ok(()) - } - - fn unparse_fstring_elem(&mut self, elem: &ruff::FStringElement) -> fmt::Result { - match elem { - ruff::FStringElement::Expression(ruff::FStringExpressionElement { - expression, - debug_text, - conversion, - format_spec, - .. - }) => self.unparse_formatted( - expression, - debug_text.as_ref(), - *conversion, - format_spec.as_deref(), - ), - ruff::FStringElement::Literal(ruff::FStringLiteralElement { value, .. }) => { - self.unparse_fstring_str(value) - } - } - } - - fn unparse_fstring_str(&mut self, s: &str) -> fmt::Result { - let s = s.replace('{', "{{").replace('}', "}}"); - self.p(&s) - } - - fn unparse_fstring(&mut self, value: &ruff::FStringValue) -> fmt::Result { - self.p("f")?; - let body = to_string_fmt(|f| { - value.iter().try_for_each(|part| match part { - ruff::FStringPart::Literal(lit) => f.write_str(lit), - ruff::FStringPart::FString(ruff::FString { elements, .. }) => { - Unparser::new(f, self.source).unparse_fstring_body(elements) - } - }) - }); - // .unparse_fstring_body(elements)); - UnicodeEscape::new_repr(body.as_str().as_ref()) - .str_repr() - .write(self.f) - } -} - -pub struct UnparseExpr<'a> { - expr: &'a Expr, - source: &'a SourceFile, -} - -impl<'a> UnparseExpr<'a> { - pub const fn new(expr: &'a Expr, source: &'a SourceFile) -> Self { - Self { expr, source } - } -} - -impl fmt::Display for UnparseExpr<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - Unparser::new(f, self.source).unparse_expr(self.expr, precedence::TEST) - } -} - -fn to_string_fmt(f: impl FnOnce(&mut fmt::Formatter<'_>) -> fmt::Result) -> String { - use std::cell::Cell; - struct Fmt(Cell>); - impl) -> fmt::Result> fmt::Display for Fmt { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.take().unwrap()(f) - } - } - Fmt(Cell::new(Some(f))).to_string() -} From 1ec2b91593c6567044373bec6320495c1d07101c Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Wed, 3 Sep 2025 20:39:58 +0300 Subject: [PATCH 2/8] Update `test_future_stmt/*.py` from 3.13.7 --- ...syntax_future10.py => badsyntax_future.py} | 0 .../test_future_stmt/badsyntax_future3.py | 10 - .../test_future_stmt/badsyntax_future4.py | 10 - .../test_future_stmt/badsyntax_future5.py | 12 -- .../test_future_stmt/badsyntax_future6.py | 10 - .../test_future_stmt/badsyntax_future7.py | 11 - .../test_future_stmt/badsyntax_future8.py | 10 - .../test_future_stmt/badsyntax_future9.py | 10 - ..._test1.py => import_nested_scope_twice.py} | 0 .../{future_test2.py => nested_scope.py} | 0 Lib/test/test_future_stmt/test_future.py | 193 ++++++++++++------ 11 files changed, 125 insertions(+), 141 deletions(-) rename Lib/test/test_future_stmt/{badsyntax_future10.py => badsyntax_future.py} (100%) delete mode 100644 Lib/test/test_future_stmt/badsyntax_future3.py delete mode 100644 Lib/test/test_future_stmt/badsyntax_future4.py delete mode 100644 Lib/test/test_future_stmt/badsyntax_future5.py delete mode 100644 Lib/test/test_future_stmt/badsyntax_future6.py delete mode 100644 Lib/test/test_future_stmt/badsyntax_future7.py delete mode 100644 Lib/test/test_future_stmt/badsyntax_future8.py delete mode 100644 Lib/test/test_future_stmt/badsyntax_future9.py rename Lib/test/test_future_stmt/{future_test1.py => import_nested_scope_twice.py} (100%) rename Lib/test/test_future_stmt/{future_test2.py => nested_scope.py} (100%) diff --git a/Lib/test/test_future_stmt/badsyntax_future10.py b/Lib/test/test_future_stmt/badsyntax_future.py similarity index 100% rename from Lib/test/test_future_stmt/badsyntax_future10.py rename to Lib/test/test_future_stmt/badsyntax_future.py diff --git a/Lib/test/test_future_stmt/badsyntax_future3.py b/Lib/test/test_future_stmt/badsyntax_future3.py deleted file mode 100644 index f1c8417eda..0000000000 --- a/Lib/test/test_future_stmt/badsyntax_future3.py +++ /dev/null @@ -1,10 +0,0 @@ -"""This is a test""" -from __future__ import nested_scopes -from __future__ import rested_snopes - -def f(x): - def g(y): - return x + y - return g - -result = f(2)(4) diff --git a/Lib/test/test_future_stmt/badsyntax_future4.py b/Lib/test/test_future_stmt/badsyntax_future4.py deleted file mode 100644 index b5f4c98e92..0000000000 --- a/Lib/test/test_future_stmt/badsyntax_future4.py +++ /dev/null @@ -1,10 +0,0 @@ -"""This is a test""" -import __future__ -from __future__ import nested_scopes - -def f(x): - def g(y): - return x + y - return g - -result = f(2)(4) diff --git a/Lib/test/test_future_stmt/badsyntax_future5.py b/Lib/test/test_future_stmt/badsyntax_future5.py deleted file mode 100644 index 8a7e5fcb70..0000000000 --- a/Lib/test/test_future_stmt/badsyntax_future5.py +++ /dev/null @@ -1,12 +0,0 @@ -"""This is a test""" -from __future__ import nested_scopes -import foo -from __future__ import nested_scopes - - -def f(x): - def g(y): - return x + y - return g - -result = f(2)(4) diff --git a/Lib/test/test_future_stmt/badsyntax_future6.py b/Lib/test/test_future_stmt/badsyntax_future6.py deleted file mode 100644 index 5a8b55a02c..0000000000 --- a/Lib/test/test_future_stmt/badsyntax_future6.py +++ /dev/null @@ -1,10 +0,0 @@ -"""This is a test""" -"this isn't a doc string" -from __future__ import nested_scopes - -def f(x): - def g(y): - return x + y - return g - -result = f(2)(4) diff --git a/Lib/test/test_future_stmt/badsyntax_future7.py b/Lib/test/test_future_stmt/badsyntax_future7.py deleted file mode 100644 index 131db2c216..0000000000 --- a/Lib/test/test_future_stmt/badsyntax_future7.py +++ /dev/null @@ -1,11 +0,0 @@ -"""This is a test""" - -from __future__ import nested_scopes; import string; from __future__ import \ - nested_scopes - -def f(x): - def g(y): - return x + y - return g - -result = f(2)(4) diff --git a/Lib/test/test_future_stmt/badsyntax_future8.py b/Lib/test/test_future_stmt/badsyntax_future8.py deleted file mode 100644 index ca45289e2e..0000000000 --- a/Lib/test/test_future_stmt/badsyntax_future8.py +++ /dev/null @@ -1,10 +0,0 @@ -"""This is a test""" - -from __future__ import * - -def f(x): - def g(y): - return x + y - return g - -print(f(2)(4)) diff --git a/Lib/test/test_future_stmt/badsyntax_future9.py b/Lib/test/test_future_stmt/badsyntax_future9.py deleted file mode 100644 index 916de06ab7..0000000000 --- a/Lib/test/test_future_stmt/badsyntax_future9.py +++ /dev/null @@ -1,10 +0,0 @@ -"""This is a test""" - -from __future__ import nested_scopes, braces - -def f(x): - def g(y): - return x + y - return g - -print(f(2)(4)) diff --git a/Lib/test/test_future_stmt/future_test1.py b/Lib/test/test_future_stmt/import_nested_scope_twice.py similarity index 100% rename from Lib/test/test_future_stmt/future_test1.py rename to Lib/test/test_future_stmt/import_nested_scope_twice.py diff --git a/Lib/test/test_future_stmt/future_test2.py b/Lib/test/test_future_stmt/nested_scope.py similarity index 100% rename from Lib/test/test_future_stmt/future_test2.py rename to Lib/test/test_future_stmt/nested_scope.py diff --git a/Lib/test/test_future_stmt/test_future.py b/Lib/test/test_future_stmt/test_future.py index 9c30054963..94a6f46d0d 100644 --- a/Lib/test/test_future_stmt/test_future.py +++ b/Lib/test/test_future_stmt/test_future.py @@ -10,6 +10,8 @@ import re import sys +TOP_LEVEL_MSG = 'from __future__ imports must occur at the beginning of the file' + rx = re.compile(r'\((\S+).py, line (\d+)') def get_error_location(msg): @@ -18,21 +20,48 @@ def get_error_location(msg): class FutureTest(unittest.TestCase): - def check_syntax_error(self, err, basename, lineno, offset=1): - self.assertIn('%s.py, line %d' % (basename, lineno), str(err)) - self.assertEqual(os.path.basename(err.filename), basename + '.py') + def check_syntax_error(self, err, basename, + *, + lineno, + message=TOP_LEVEL_MSG, offset=1): + if basename != '': + basename += '.py' + + self.assertEqual(f'{message} ({basename}, line {lineno})', str(err)) + self.assertEqual(os.path.basename(err.filename), basename) self.assertEqual(err.lineno, lineno) self.assertEqual(err.offset, offset) - def test_future1(self): - with import_helper.CleanImport('test.test_future_stmt.future_test1'): - from test.test_future_stmt import future_test1 - self.assertEqual(future_test1.result, 6) + def assertSyntaxError(self, code, + *, + lineno=1, + message=TOP_LEVEL_MSG, offset=1, + parametrize_docstring=True): + code = dedent(code.lstrip('\n')) + for add_docstring in ([False, True] if parametrize_docstring else [False]): + with self.subTest(code=code, add_docstring=add_docstring): + if add_docstring: + code = '"""Docstring"""\n' + code + lineno += 1 + with self.assertRaises(SyntaxError) as cm: + exec(code) + self.check_syntax_error(cm.exception, "", + lineno=lineno, + message=message, + offset=offset) + + def test_import_nested_scope_twice(self): + # Import the name nested_scopes twice to trigger SF bug #407394 + with import_helper.CleanImport( + 'test.test_future_stmt.import_nested_scope_twice', + ): + from test.test_future_stmt import import_nested_scope_twice + self.assertEqual(import_nested_scope_twice.result, 6) - def test_future2(self): - with import_helper.CleanImport('test.test_future_stmt.future_test2'): - from test.test_future_stmt import future_test2 - self.assertEqual(future_test2.result, 6) + def test_nested_scope(self): + with import_helper.CleanImport('test.test_future_stmt.nested_scope'): + from test.test_future_stmt import nested_scope + self.assertEqual(nested_scope.result, 6) def test_future_single_import(self): with import_helper.CleanImport( @@ -52,47 +81,80 @@ def test_future_multiple_features(self): ): from test.test_future_stmt import test_future_multiple_features - def test_badfuture3(self): - with self.assertRaises(SyntaxError) as cm: - from test.test_future_stmt import badsyntax_future3 - self.check_syntax_error(cm.exception, "badsyntax_future3", 3) + def test_unknown_future_flag(self): + code = """ + from __future__ import nested_scopes + from __future__ import rested_snopes # typo error here: nested => rested + """ + self.assertSyntaxError( + code, lineno=2, + message='future feature rested_snopes is not defined', offset=24, + ) - def test_badfuture4(self): - with self.assertRaises(SyntaxError) as cm: - from test.test_future_stmt import badsyntax_future4 - self.check_syntax_error(cm.exception, "badsyntax_future4", 3) + def test_future_import_not_on_top(self): + code = """ + import some_module + from __future__ import annotations + """ + self.assertSyntaxError(code, lineno=2) - def test_badfuture5(self): - with self.assertRaises(SyntaxError) as cm: - from test.test_future_stmt import badsyntax_future5 - self.check_syntax_error(cm.exception, "badsyntax_future5", 4) + code = """ + import __future__ + from __future__ import annotations + """ + self.assertSyntaxError(code, lineno=2) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_badfuture6(self): - with self.assertRaises(SyntaxError) as cm: - from test.test_future_stmt import badsyntax_future6 - self.check_syntax_error(cm.exception, "badsyntax_future6", 3) + code = """ + from __future__ import absolute_import + "spam, bar, blah" + from __future__ import print_function + """ + self.assertSyntaxError(code, lineno=3) - def test_badfuture7(self): - with self.assertRaises(SyntaxError) as cm: - from test.test_future_stmt import badsyntax_future7 - self.check_syntax_error(cm.exception, "badsyntax_future7", 3, 54) + def test_future_import_with_extra_string(self): + code = """ + '''Docstring''' + "this isn't a doc string" + from __future__ import nested_scopes + """ + self.assertSyntaxError(code, lineno=3, parametrize_docstring=False) - def test_badfuture8(self): - with self.assertRaises(SyntaxError) as cm: - from test.test_future_stmt import badsyntax_future8 - self.check_syntax_error(cm.exception, "badsyntax_future8", 3) + def test_multiple_import_statements_on_same_line(self): + # With `\`: + code = """ + from __future__ import nested_scopes; import string; from __future__ import \ + nested_scopes + """ + self.assertSyntaxError(code, offset=54) - def test_badfuture9(self): - with self.assertRaises(SyntaxError) as cm: - from test.test_future_stmt import badsyntax_future9 - self.check_syntax_error(cm.exception, "badsyntax_future9", 3) + # Without `\`: + code = """ + from __future__ import nested_scopes; import string; from __future__ import nested_scopes + """ + self.assertSyntaxError(code, offset=54) + + def test_future_import_star(self): + code = """ + from __future__ import * + """ + self.assertSyntaxError(code, message='future feature * is not defined', offset=24) + + def test_future_import_braces(self): + code = """ + from __future__ import braces + """ + # Congrats, you found an easter egg! + self.assertSyntaxError(code, message='not a chance', offset=24) - def test_badfuture10(self): + code = """ + from __future__ import nested_scopes, braces + """ + self.assertSyntaxError(code, message='not a chance', offset=39) + + def test_module_with_future_import_not_on_top(self): with self.assertRaises(SyntaxError) as cm: - from test.test_future_stmt import badsyntax_future10 - self.check_syntax_error(cm.exception, "badsyntax_future10", 3) + from test.test_future_stmt import badsyntax_future + self.check_syntax_error(cm.exception, "badsyntax_future", lineno=3) def test_ensure_flags_dont_clash(self): # bpo-39562: test that future flags and compiler flags doesn't clash @@ -109,26 +171,6 @@ def test_ensure_flags_dont_clash(self): } self.assertCountEqual(set(flags.values()), flags.values()) - def test_parserhack(self): - # test that the parser.c::future_hack function works as expected - # Note: although this test must pass, it's not testing the original - # bug as of 2.6 since the with statement is not optional and - # the parser hack disabled. If a new keyword is introduced in - # 2.6, change this to refer to the new future import. - try: - exec("from __future__ import print_function; print 0") - except SyntaxError: - pass - else: - self.fail("syntax error didn't occur") - - try: - exec("from __future__ import (print_function); print 0") - except SyntaxError: - pass - else: - self.fail("syntax error didn't occur") - def test_unicode_literals_exec(self): scope = {} exec("from __future__ import unicode_literals; x = ''", {}, scope) @@ -141,6 +183,25 @@ def test_syntactical_future_repl(self): out = kill_python(p) self.assertNotIn(b'SyntaxError: invalid syntax', out) + def test_future_dotted_import(self): + with self.assertRaises(ImportError): + exec("from .__future__ import spam") + + code = dedent( + """ + from __future__ import print_function + from ...__future__ import ham + """ + ) + with self.assertRaises(ImportError): + exec(code) + + code = """ + from .__future__ import nested_scopes + from __future__ import barry_as_FLUFL + """ + self.assertSyntaxError(code, lineno=2) + class AnnotationsFutureTestCase(unittest.TestCase): template = dedent( """ @@ -384,8 +445,6 @@ def test_infinity_numbers(self): self.assertAnnotationEqual("('inf', 1e1000, 'infxxx', 1e1000j)", expected=f"('inf', {inf}, 'infxxx', {infj})") self.assertAnnotationEqual("(1e1000, (1e1000j,))", expected=f"({inf}, ({infj},))") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_annotation_with_complex_target(self): with self.assertRaises(SyntaxError): exec( @@ -409,8 +468,6 @@ def bar(): self.assertEqual(foo.__code__.co_cellvars, ()) self.assertEqual(foo().__code__.co_freevars, ()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_annotations_forbidden(self): with self.assertRaises(SyntaxError): self._exec_future("test: (yield)") From 87dd525a7b68ae5fc0d4b4add1f532e05bef5ed2 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Wed, 3 Sep 2025 20:49:24 +0300 Subject: [PATCH 3/8] Mark failing tests --- Lib/test/test_future_stmt/test_future.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Lib/test/test_future_stmt/test_future.py b/Lib/test/test_future_stmt/test_future.py index 94a6f46d0d..e6e6201a76 100644 --- a/Lib/test/test_future_stmt/test_future.py +++ b/Lib/test/test_future_stmt/test_future.py @@ -81,6 +81,7 @@ def test_future_multiple_features(self): ): from test.test_future_stmt import test_future_multiple_features + @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message def test_unknown_future_flag(self): code = """ from __future__ import nested_scopes @@ -91,6 +92,7 @@ def test_unknown_future_flag(self): message='future feature rested_snopes is not defined', offset=24, ) + @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message def test_future_import_not_on_top(self): code = """ import some_module @@ -111,6 +113,7 @@ def test_future_import_not_on_top(self): """ self.assertSyntaxError(code, lineno=3) + @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message def test_future_import_with_extra_string(self): code = """ '''Docstring''' @@ -119,6 +122,7 @@ def test_future_import_with_extra_string(self): """ self.assertSyntaxError(code, lineno=3, parametrize_docstring=False) + @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message def test_multiple_import_statements_on_same_line(self): # With `\`: code = """ @@ -133,12 +137,14 @@ def test_multiple_import_statements_on_same_line(self): """ self.assertSyntaxError(code, offset=54) + @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message def test_future_import_star(self): code = """ from __future__ import * """ self.assertSyntaxError(code, message='future feature * is not defined', offset=24) + @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message def test_future_import_braces(self): code = """ from __future__ import braces @@ -151,6 +157,7 @@ def test_future_import_braces(self): """ self.assertSyntaxError(code, message='not a chance', offset=39) + @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message def test_module_with_future_import_not_on_top(self): with self.assertRaises(SyntaxError) as cm: from test.test_future_stmt import badsyntax_future @@ -183,6 +190,7 @@ def test_syntactical_future_repl(self): out = kill_python(p) self.assertNotIn(b'SyntaxError: invalid syntax', out) + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError: future feature spam is not defined def test_future_dotted_import(self): with self.assertRaises(ImportError): exec("from .__future__ import spam") @@ -259,6 +267,7 @@ def _exec_future(self, code): ) return scope + @unittest.expectedFailure # TODO: RUSTPYTHON; 'a,' != '(a,)' def test_annotations(self): eq = self.assertAnnotationEqual eq('...') @@ -423,6 +432,7 @@ def test_annotations(self): eq('(((a, b)))', '(a, b)') eq("1 + 2 + 3") + @unittest.expectedFailure # TODO: RUSTPYTHON; "f'{x=!r}'" != "f'x={x!r}'" def test_fstring_debug_annotations(self): # f-strings with '=' don't round trip very well, so set the expected # result explicitly. @@ -433,6 +443,7 @@ def test_fstring_debug_annotations(self): self.assertAnnotationEqual("f'{x=!a}'", expected="f'x={x!a}'") self.assertAnnotationEqual("f'{x=!s:*^20}'", expected="f'x={x!s:*^20}'") + @unittest.expectedFailure # TODO: RUSTPYTHON; '1e309, 1e309j' != '(1e309, 1e309j)' def test_infinity_numbers(self): inf = "1e" + repr(sys.float_info.max_10_exp + 1) infj = f"{inf}j" @@ -445,6 +456,7 @@ def test_infinity_numbers(self): self.assertAnnotationEqual("('inf', 1e1000, 'infxxx', 1e1000j)", expected=f"('inf', {inf}, 'infxxx', {infj})") self.assertAnnotationEqual("(1e1000, (1e1000j,))", expected=f"({inf}, ({infj},))") + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised def test_annotation_with_complex_target(self): with self.assertRaises(SyntaxError): exec( @@ -468,6 +480,7 @@ def bar(): self.assertEqual(foo.__code__.co_cellvars, ()) self.assertEqual(foo().__code__.co_freevars, ()) + @unittest.expectedFailure # TODO: RUSTPYTHON; "f'{x=!r}'" != "f'x={x!r}'" def test_annotations_forbidden(self): with self.assertRaises(SyntaxError): self._exec_future("test: (yield)") From 105981444c4ae15c6340cbd16995cadcc65e18d6 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Wed, 3 Sep 2025 21:09:47 +0300 Subject: [PATCH 4/8] Mark failing test --- Lib/test/test_typing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 318c088fbc..13d0d72e5b 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -4952,6 +4952,7 @@ def barfoo(x: AT): ... def barfoo2(x: CT): ... self.assertIs(get_type_hints(barfoo2, globals(), locals())['x'], CT) + @unittest.expectedFailure # TODO: RUSTPYTHON; 'List[list["C2"]]' != "List[list['C2']]" def test_generic_pep585_forward_ref(self): # See https://bugs.python.org/issue41370 From 7baf20feaed3bce6e425bffb5b1d566f82c763e6 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Wed, 22 Oct 2025 11:46:55 +0300 Subject: [PATCH 5/8] Merge remote-tracking branch 'upstream/main' into ruff-unparse --- .cargo/config.toml | 3 + .cspell.dict/cpython.txt | 3 +- .github/copilot-instructions.md | 6 +- .github/dependabot.yml | 16 +- .github/workflows/ci.yaml | 24 +- .github/workflows/cron-ci.yaml | 8 + .github/workflows/release.yml | 10 +- Cargo.lock | 897 ++++---- Cargo.toml | 45 +- Lib/_opcode_metadata.py | 343 +++ Lib/_strptime.py | 373 +++- Lib/codecs.py | 13 +- Lib/encodings/aliases.py | 1 + Lib/encodings/idna.py | 166 +- Lib/encodings/palmos.py | 2 +- Lib/encodings/punycode.py | 42 +- Lib/encodings/undefined.py | 2 +- Lib/encodings/utf_16.py | 4 +- Lib/encodings/utf_32.py | 6 +- Lib/fnmatch.py | 37 +- Lib/glob.py | 301 ++- Lib/html/parser.py | 218 +- Lib/io.py | 15 +- Lib/numbers.py | 11 +- Lib/opcode.py | 449 +--- Lib/runpy.py | 28 +- Lib/site.py | 229 +- Lib/{sysconfig.py => sysconfig/__init__.py} | 554 ++--- Lib/sysconfig/__main__.py | 248 +++ Lib/test/support/__init__.py | 2 +- Lib/test/support/bytecode_helper.py | 109 +- Lib/test/test__opcode.py | 143 ++ Lib/test/test_ast/test_ast.py | 329 +-- Lib/test/test_call.py | 884 ++++++-- Lib/test/test_code.py | 12 - Lib/test/test_codecs.py | 717 +++--- Lib/test/test_collections.py | 38 +- Lib/test/test_compile.py | 2 - Lib/test/test_csv.py | 84 +- Lib/test/test_dtrace.py | 32 + Lib/test/test_faulthandler.py | 1 + Lib/test/test_file.py | 5 +- Lib/test/test_fnmatch.py | 14 +- Lib/test/test_fstring.py | 1924 ++++++++--------- Lib/test/test_genericpath.py | 8 + Lib/test/test_glob.py | 194 +- Lib/test/test_htmlparser.py | 386 +++- .../test_importlib/source/test_file_loader.py | 171 +- Lib/test/test_io.py | 577 ++--- Lib/test/test_itertools.py | 23 +- Lib/test/test_list.py | 11 +- Lib/test/test_logging.py | 121 +- Lib/test/test_ntpath.py | 8 + Lib/test/test_os.py | 193 +- Lib/test/test_posix.py | 4 +- Lib/test/test_py_compile.py | 1 - Lib/test/test_raise.py | 18 +- Lib/test/test_runpy.py | 68 +- Lib/test/test_scope.py | 80 +- Lib/test/test_shutil.py | 8 +- Lib/test/test_site.py | 60 +- Lib/test/test_slice.py | 3 +- Lib/test/test_sqlite3/test_regression.py | 4 - Lib/test/test_stat.py | 11 - Lib/test/test_str.py | 67 +- Lib/test/test_string_literals.py | 15 +- Lib/test/test_strtod.py | 17 +- Lib/test/test_sundry.py | 29 +- Lib/test/test_syntax.py | 710 +++++- Lib/test/test_sysconfig.py | 256 ++- Lib/test/test_tarfile.py | 3 - Lib/test/test_uuid.py | 31 +- Lib/test/test_xml_etree.py | 390 ++-- Lib/test/test_yield_from.py | 552 ++++- Lib/uuid.py | 20 +- Lib/zoneinfo/_common.py | 11 +- Lib/zoneinfo/_zoneinfo.py | 6 +- benches/execution.rs | 12 +- benches/microbenchmarks.rs | 2 +- common/Cargo.toml | 3 +- common/src/crt_fd.rs | 4 + common/src/fileutils.rs | 11 +- compiler/Cargo.toml | 1 - compiler/codegen/src/compile.rs | 262 ++- compiler/codegen/src/ir.rs | 141 +- compiler/codegen/src/lib.rs | 1 + compiler/codegen/src/symboltable.rs | 128 +- compiler/core/Cargo.toml | 3 +- compiler/core/src/bytecode.rs | 95 +- compiler/core/src/lib.rs | 4 +- compiler/core/src/marshal.rs | 24 +- compiler/literal/Cargo.toml | 2 +- compiler/literal/src/float.rs | 2 +- compiler/src/lib.rs | 26 +- derive-impl/src/compile_bytecode.rs | 24 +- derive/Cargo.toml | 3 +- jit/src/instructions.rs | 17 +- pylib/build.rs | 18 +- scripts/lib_updater.py | 352 +++ src/shell.rs | 20 +- stdlib/Cargo.toml | 3 +- stdlib/src/contextvars.rs | 11 +- stdlib/src/faulthandler.rs | 2 +- stdlib/src/lib.rs | 2 + stdlib/src/math.rs | 20 +- stdlib/src/mmap.rs | 36 +- stdlib/src/opcode.rs | 282 +++ stdlib/src/posixsubprocess.rs | 178 +- stdlib/src/scproxy.rs | 27 +- stdlib/src/select.rs | 8 +- stdlib/src/socket.rs | 17 +- stdlib/src/sqlite.rs | 135 +- stdlib/src/syslog.rs | 20 +- stdlib/src/unicodedata.rs | 19 +- stdlib/src/uuid.rs | 3 + stdlib/src/zlib.rs | 20 +- vm/Cargo.toml | 9 +- vm/src/builtins/code.rs | 482 ++++- vm/src/builtins/descriptor.rs | 14 +- vm/src/builtins/enumerate.rs | 8 +- vm/src/builtins/frame.rs | 2 +- vm/src/builtins/function.rs | 10 +- vm/src/builtins/genericalias.rs | 94 +- vm/src/builtins/int.rs | 47 +- vm/src/builtins/iter.rs | 8 +- vm/src/builtins/mappingproxy.rs | 20 +- vm/src/builtins/memory.rs | 24 +- vm/src/builtins/object.rs | 63 +- vm/src/builtins/property.rs | 32 +- vm/src/builtins/str.rs | 12 +- vm/src/builtins/super.rs | 17 +- vm/src/builtins/type.rs | 79 +- vm/src/byte.rs | 8 +- vm/src/cformat.rs | 8 +- vm/src/exceptions.rs | 45 +- vm/src/frame.rs | 17 +- vm/src/function/argument.rs | 4 + vm/src/object/core.rs | 10 +- vm/src/protocol/mapping.rs | 14 +- vm/src/protocol/object.rs | 37 +- vm/src/readline.rs | 12 +- vm/src/signal.rs | 11 +- vm/src/stdlib/ast.rs | 22 +- vm/src/stdlib/ast/argument.rs | 4 + vm/src/stdlib/ast/constant.rs | 65 +- vm/src/stdlib/ast/elif_else_clause.rs | 11 +- vm/src/stdlib/ast/exception.rs | 2 + vm/src/stdlib/ast/expression.rs | 106 +- vm/src/stdlib/ast/module.rs | 9 +- vm/src/stdlib/ast/other.rs | 10 +- vm/src/stdlib/ast/parameter.rs | 9 + vm/src/stdlib/ast/pattern.rs | 19 + vm/src/stdlib/ast/python.rs | 16 +- vm/src/stdlib/ast/statement.rs | 60 +- vm/src/stdlib/ast/string.rs | 117 +- vm/src/stdlib/ast/type_parameters.rs | 12 +- vm/src/stdlib/builtins.rs | 30 +- vm/src/stdlib/io.rs | 10 +- vm/src/stdlib/itertools.rs | 57 +- vm/src/stdlib/mod.rs | 2 + vm/src/stdlib/nt.rs | 28 +- vm/src/stdlib/os.rs | 3 +- vm/src/stdlib/posix.rs | 217 +- vm/src/stdlib/signal.rs | 7 +- vm/src/stdlib/sys.rs | 68 +- vm/src/stdlib/sysconfig.rs | 14 + vm/src/stdlib/time.rs | 11 +- vm/src/stdlib/typing.rs | 18 +- vm/src/stdlib/warnings.rs | 8 +- vm/src/vm/context.rs | 20 +- vm/src/vm/method.rs | 19 +- vm/src/vm/mod.rs | 43 +- vm/src/vm/vm_new.rs | 13 +- vm/src/vm/vm_ops.rs | 43 +- vm/src/windows.rs | 10 +- wasm/demo/package-lock.json | 73 +- wasm/demo/package.json | 2 +- wasm/lib/.cargo/config.toml | 3 - wasm/lib/Cargo.toml | 2 +- wasm/lib/src/convert.rs | 104 +- wasm/tests/conftest.py | 4 +- wasm/tests/requirements.txt | 2 +- 182 files changed, 10845 insertions(+), 5859 deletions(-) create mode 100644 Lib/_opcode_metadata.py rename Lib/{sysconfig.py => sysconfig/__init__.py} (53%) create mode 100644 Lib/sysconfig/__main__.py create mode 100644 Lib/test/test__opcode.py create mode 100755 scripts/lib_updater.py create mode 100644 stdlib/src/opcode.rs create mode 100644 vm/src/stdlib/sysconfig.rs diff --git a/.cargo/config.toml b/.cargo/config.toml index a590c044d8..635229119f 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -3,3 +3,6 @@ rustflags = "-C link-arg=/STACK:8000000" [target.'cfg(all(target_os = "windows", not(target_env = "msvc")))'] rustflags = "-C link-args=-Wl,--stack,8000000" + +[target.wasm32-unknown-unknown] +rustflags = ["--cfg=getrandom_backend=\"wasm_js\""] diff --git a/.cspell.dict/cpython.txt b/.cspell.dict/cpython.txt index 48059cf4e4..8c733e343d 100644 --- a/.cspell.dict/cpython.txt +++ b/.cspell.dict/cpython.txt @@ -23,6 +23,7 @@ freevars fromlist heaptype HIGHRES +Itertool IMMUTABLETYPE kwonlyarg kwonlyargs @@ -58,4 +59,4 @@ weakreflist withitem withs xstat -XXPRIME \ No newline at end of file +XXPRIME diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index e175cd5184..46fc16c5ed 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -206,7 +206,7 @@ cargo run --features ssl ## Documentation -- Check the [architecture document](architecture/architecture.md) for a high-level overview -- Read the [development guide](DEVELOPMENT.md) for detailed setup instructions +- Check the [architecture document](/architecture/architecture.md) for a high-level overview +- Read the [development guide](/DEVELOPMENT.md) for detailed setup instructions - Generate documentation with `cargo doc --no-deps --all` -- Online documentation is available at [docs.rs/rustpython](https://docs.rs/rustpython/) \ No newline at end of file +- Online documentation is available at [docs.rs/rustpython](https://docs.rs/rustpython/) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index be006de9a1..b3b7b446e4 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,13 +1,15 @@ -# Keep GitHub Actions up to date with GitHub's Dependabot... -# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot -# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem version: 2 updates: + - package-ecosystem: cargo + directory: / + schedule: + interval: weekly + ignore: + # TODO: Remove when we use ruff from crates.io + # for some reason dependabot only updates the Cargo.lock file when dealing + # with git dependencies. i.e. not updating the version in Cargo.toml + - dependency-name: "ruff_*" - package-ecosystem: github-actions directory: / - groups: - github-actions: - patterns: - - "*" # Group all Actions updates into a single larger pull request schedule: interval: weekly diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b0ff575f1e..977f27f376 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -119,7 +119,7 @@ jobs: os: [macos-latest, ubuntu-latest, windows-latest] fail-fast: false steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: dtolnay/rust-toolchain@stable with: components: clippy @@ -178,7 +178,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: dtolnay/rust-toolchain@stable with: target: i686-unknown-linux-gnu @@ -245,10 +245,10 @@ jobs: os: [macos-latest, ubuntu-latest, windows-latest] fail-fast: false steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: ${{ env.PYTHON_VERSION }} - name: Set up the Windows environment @@ -267,7 +267,7 @@ jobs: - name: build rustpython run: cargo build --release --verbose --features=threading ${{ env.CARGO_ARGS }},jit if: runner.os != 'macOS' - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: ${{ env.PYTHON_VERSION }} - name: run snippets @@ -310,7 +310,7 @@ jobs: name: Check Rust code with rustfmt and clippy runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: dtolnay/rust-toolchain@stable with: components: rustfmt, clippy @@ -318,7 +318,7 @@ jobs: run: cargo fmt --check - name: run clippy on wasm run: cargo clippy --manifest-path=wasm/lib/Cargo.toml -- -Dwarnings - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: ${{ env.PYTHON_VERSION }} - name: install ruff @@ -351,7 +351,7 @@ jobs: env: NIGHTLY_CHANNEL: nightly steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: dtolnay/rust-toolchain@master with: @@ -373,7 +373,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 @@ -384,12 +384,12 @@ jobs: wget https://github.com/mozilla/geckodriver/releases/download/v0.36.0/geckodriver-v0.36.0-linux64.tar.gz mkdir geckodriver tar -xzf geckodriver-v0.36.0-linux64.tar.gz -C geckodriver - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: ${{ env.PYTHON_VERSION }} - run: python -m pip install -r requirements.txt working-directory: ./wasm/tests - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v6 with: cache: "npm" cache-dependency-path: "wasm/demo/package-lock.json" @@ -434,7 +434,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: dtolnay/rust-toolchain@stable with: target: wasm32-wasip1 diff --git a/.github/workflows/cron-ci.yaml b/.github/workflows/cron-ci.yaml index 6389fee1cb..868675ca1c 100644 --- a/.github/workflows/cron-ci.yaml +++ b/.github/workflows/cron-ci.yaml @@ -18,6 +18,8 @@ jobs: codecov: name: Collect code coverage data runs-on: ubuntu-latest + # Disable this scheduled job when running on a fork. + if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable @@ -44,6 +46,8 @@ jobs: testdata: name: Collect regression test data runs-on: ubuntu-latest + # Disable this scheduled job when running on a fork. + if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable @@ -73,6 +77,8 @@ jobs: whatsleft: name: Collect what is left data runs-on: ubuntu-latest + # Disable this scheduled job when running on a fork. + if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable @@ -111,6 +117,8 @@ jobs: benchmark: name: Collect benchmark data runs-on: ubuntu-latest + # Disable this scheduled job when running on a fork. + if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f6a1ad3209..c85614369b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,6 +21,8 @@ env: jobs: build: runs-on: ${{ matrix.platform.runner }} + # Disable this scheduled job when running on a fork. + if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} strategy: matrix: platform: @@ -88,6 +90,8 @@ jobs: build-wasm: runs-on: ubuntu-latest + # Disable this scheduled job when running on a fork. + if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable @@ -108,7 +112,7 @@ jobs: - name: install wasm-pack run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v6 - uses: mwilliamson/setup-wabt-action@v3 with: { wabt-version: "1.0.30" } - name: build demo @@ -136,10 +140,12 @@ jobs: release: runs-on: ubuntu-latest + # Disable this scheduled job when running on a fork. + if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} needs: [build, build-wasm] steps: - name: Download Binary Artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v5 with: path: bin pattern: rustpython-* diff --git a/Cargo.lock b/Cargo.lock index deb766185a..53b4f96922 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21,7 +21,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", - "getrandom 0.3.3", + "getrandom 0.3.4", "once_cell", "version_check", "zerocopy", @@ -42,12 +42,6 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - [[package]] name = "android_system_properties" version = "0.1.5" @@ -65,9 +59,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.20" +version = "0.6.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ae563653d1938f79b1ab1b5e668c87c76a9930414574a6583a7b7e11a8e6192" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" dependencies = [ "anstyle", "anstyle-parse", @@ -80,9 +74,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.11" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" [[package]] name = "anstyle-parse" @@ -115,9 +109,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.99" +version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" [[package]] name = "approx" @@ -149,6 +143,36 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "attribute-derive" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05832cdddc8f2650cc2cc187cc2e952b8c133a48eb055f35211f61ee81502d77" +dependencies = [ + "attribute-derive-macro", + "derive-where", + "manyhow", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "attribute-derive-macro" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a7cdbbd4bd005c5d3e2e9c885e6fa575db4f4a3572335b974d8db853b6beb61" +dependencies = [ + "collection_literals", + "interpolator", + "manyhow", + "proc-macro-utils", + "proc-macro2", + "quote", + "quote-use", + "syn", +] + [[package]] name = "autocfg" version = "1.5.0" @@ -167,7 +191,7 @@ version = "0.71.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" dependencies = [ - "bitflags 2.9.3", + "bitflags 2.9.4", "cexpr", "clang-sys", "itertools 0.13.0", @@ -178,7 +202,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.106", + "syn", ] [[package]] @@ -189,9 +213,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.9.3" +version = "2.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34efbcccd345379ca2868b2b2c9d3782e9cc58ba87bc7d79d5b53d9c9ae6f25d" +checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" [[package]] name = "blake2" @@ -233,15 +257,15 @@ dependencies = [ [[package]] name = "bytemuck" -version = "1.23.2" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3995eaeebcdf32f91f980d360f78732ddc061097ab4e39991ae7a6ace9194677" +checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" [[package]] name = "bzip2" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bea8dcd42434048e4f7a304411d9273a411f647446c1234a65ce0554923f4cff" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" dependencies = [ "libbz2-rs-sys", ] @@ -272,10 +296,11 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.34" +version = "1.2.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42bc4aea80032b7bf409b0bc7ccad88853858911b7713a8062fdc0623867bedc" +checksum = "ac9fe6cdbb24b6ade63616c0a0688e45bb56732262c158df3c0c4bea4ca47cb7" dependencies = [ + "find-msvc-tools", "shlex", ] @@ -290,9 +315,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" @@ -302,11 +327,10 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.41" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" +checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" dependencies = [ - "android-tzdata", "iana-time-zone", "js-sys", "num-traits", @@ -354,18 +378,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.46" +version = "4.5.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c5e4fcf9c21d2e544ca1ee9d8552de13019a42aa7dbf32747fa7aaf1df76e57" +checksum = "f4512b90fa68d3a9932cea5184017c5d200f5921df706d45e853537dea51508f" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.5.46" +version = "4.5.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fecb53a0e6fcfb055f686001bc2e2592fa527efaf38dbe81a6a9563562e57d41" +checksum = "0025e98baa12e766c67ba13ff4695a887a1eba19569aad00a472546795bd6730" dependencies = [ "anstyle", "clap_lex", @@ -373,9 +397,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" +checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" [[package]] name = "clipboard-win" @@ -386,6 +410,12 @@ dependencies = [ "error-code", ] +[[package]] +name = "collection_literals" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2550f75b8cfac212855f6b1885455df8eaee8fe8e246b647d69146142e016084" + [[package]] name = "colorchoice" version = "1.0.4" @@ -394,9 +424,9 @@ checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "compact_str" -version = "0.8.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b79c4069c6cad78e2e0cdfcbd26275770669fb39fd308a752dc110e83b9af32" +checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a" dependencies = [ "castaway", "cfg-if", @@ -519,7 +549,7 @@ dependencies = [ "cranelift-entity", "cranelift-isle", "gimli", - "hashbrown", + "hashbrown 0.15.5", "log", "regalloc2", "rustc-hash", @@ -640,25 +670,22 @@ dependencies = [ [[package]] name = "criterion" -version = "0.5.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +checksum = "e1c047a62b0cc3e145fa84415a3191f628e980b194c2755aa12300a4e6cbd928" dependencies = [ "anes", "cast", "ciborium", "clap", "criterion-plot", - "is-terminal", - "itertools 0.10.5", + "itertools 0.13.0", "num-traits", - "once_cell", "oorandom", "plotters", "rayon", "regex", "serde", - "serde_derive", "serde_json", "tinytemplate", "walkdir", @@ -666,12 +693,12 @@ dependencies = [ [[package]] name = "criterion-plot" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +checksum = "9b1bcc0dc7dfae599d84ad0b1a55f80cde8af3725da8313b528da95ef783e338" dependencies = [ "cast", - "itertools 0.10.5", + "itertools 0.13.0", ] [[package]] @@ -717,13 +744,24 @@ dependencies = [ [[package]] name = "csv-core" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" dependencies = [ "memchr", ] +[[package]] +name = "derive-where" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef941ded77d15ca19b40374869ac6000af1c9f2a4c0f3d4c70926287e6364a8f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "digest" version = "0.10.7" @@ -794,9 +832,9 @@ checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" [[package]] name = "env_filter" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" +checksum = "1bf3c259d255ca70051b30e2e95b5446cdb8949ac4cd22c0d7fd634d89f568e2" dependencies = [ "log", "regex", @@ -829,12 +867,12 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -866,6 +904,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "find-msvc-tools" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" + [[package]] name = "flame" version = "0.2.2" @@ -881,13 +925,13 @@ dependencies = [ [[package]] name = "flamer" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36b732da54fd4ea34452f2431cf464ac7be94ca4b339c9cd3d3d12eb06fe7aab" +checksum = "7693d9dd1ec1c54f52195dfe255b627f7cec7da33b679cd56de949e662b3db10" dependencies = [ "flame", "quote", - "syn 1.0.109", + "syn", ] [[package]] @@ -904,21 +948,15 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.2" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" +checksum = "dc5a4e564e38c699f2880d3fda590bedc2e69f3f84cd48b457bd892ce61d0aa9" dependencies = [ "crc32fast", "libz-rs-sys", "miniz_oxide", ] -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - [[package]] name = "foldhash" version = "0.1.5" @@ -942,22 +980,45 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "generic-array" -version = "0.14.7" +version = "0.14.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" dependencies = [ "typenum", "version_check", ] +[[package]] +name = "get-size-derive2" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3814abc7da8ab18d2fd820f5b540b5e39b6af0a32de1bdd7c47576693074843" +dependencies = [ + "attribute-derive", + "quote", + "syn", +] + +[[package]] +name = "get-size2" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfe2cec5b5ce8fb94dcdb16a1708baa4d0609cc3ce305ca5d3f6f2ffb59baed" +dependencies = [ + "compact_str", + "get-size-derive2", + "hashbrown 0.16.0", + "smallvec", +] + [[package]] name = "gethostname" -version = "1.0.2" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc257fdb4038301ce4b9cd1b3b51704509692bb3ff716a410cbd07925d9dae55" +checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" dependencies = [ "rustix", - "windows-targets 0.52.6", + "windows-link", ] [[package]] @@ -977,20 +1038,20 @@ checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", "libc", - "wasi 0.11.1+wasi-snapshot-preview1", + "wasi", ] [[package]] name = "getrandom" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", "js-sys", "libc", "r-efi", - "wasi 0.14.3+wasi-0.2.4", + "wasip2", "wasm-bindgen", ] @@ -1013,12 +1074,13 @@ checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] name = "half" -version = "2.6.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ "cfg-if", "crunchy", + "zerocopy", ] [[package]] @@ -1030,6 +1092,12 @@ dependencies = [ "foldhash", ] +[[package]] +name = "hashbrown" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" + [[package]] name = "heck" version = "0.5.0" @@ -1065,9 +1133,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.63" +version = "0.1.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" +checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1075,7 +1143,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.61.2", + "windows-core 0.62.2", ] [[package]] @@ -1089,12 +1157,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.11.0" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2481980430f9f78649238835720ddccc57e52df14ffce1c6f37391d61b563e9" +checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.16.0", ] [[package]] @@ -1105,15 +1173,21 @@ checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" [[package]] name = "insta" -version = "1.43.1" +version = "1.43.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "154934ea70c58054b556dd430b99a98c2a7ff5309ac9891597e339b5c28f4371" +checksum = "46fdb647ebde000f43b5b53f773c30cf9b0cb4300453208713fa38b2c70935a0" dependencies = [ "console", "once_cell", "similar", ] +[[package]] +name = "interpolator" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71dd52191aae121e8611f1e8dc3e324dd0dd1dee1e6dd91d10ee07a3cfb4d9d8" + [[package]] name = "is-macro" version = "0.3.7" @@ -1123,18 +1197,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.106", -] - -[[package]] -name = "is-terminal" -version = "0.4.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys 0.59.0", + "syn", ] [[package]] @@ -1143,15 +1206,6 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.13.0" @@ -1197,14 +1251,14 @@ checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] name = "js-sys" -version = "0.3.77" +version = "0.3.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +checksum = "ec48937a97411dcb524a265206ccd4c90bb711fca92b2792c407f268825b9305" dependencies = [ "once_cell", "wasm-bindgen", @@ -1212,12 +1266,12 @@ dependencies = [ [[package]] name = "junction" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72bbdfd737a243da3dfc1f99ee8d6e166480f17ab4ac84d7c34aacd73fc7bd16" +checksum = "c52f6e1bf39a7894f618c9d378904a11dbd7e10fe3ec20d1173600e79b1408d8" dependencies = [ "scopeguard", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -1237,33 +1291,28 @@ checksum = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73" [[package]] name = "lexical-parse-float" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de6f9cb01fb0b08060209a057c048fcbab8717b4c1ecd2eac66ebfe39a65b0f2" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" dependencies = [ "lexical-parse-integer", "lexical-util", - "static_assertions", ] [[package]] name = "lexical-parse-integer" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72207aae22fc0a121ba7b6d479e42cbfea549af1479c3f3a4f12c70dd66df12e" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" dependencies = [ "lexical-util", - "static_assertions", ] [[package]] name = "lexical-util" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a82e24bf537fd24c177ffbbdc6ebcc8d54732c35b50a3f28cc3f4e4c949a0b3" -dependencies = [ - "static_assertions", -] +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" [[package]] name = "lexopt" @@ -1279,9 +1328,9 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" [[package]] name = "libc" -version = "0.2.175" +version = "0.2.177" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" +checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" [[package]] name = "libffi" @@ -1304,12 +1353,12 @@ dependencies = [ [[package]] name = "libloading" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" dependencies = [ "cfg-if", - "windows-targets 0.53.3", + "windows-link", ] [[package]] @@ -1320,11 +1369,11 @@ checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" [[package]] name = "libredox" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "391290121bad3d37fbddad76d8f5d1c1c314cfc646d143d7e07a3086ddff0ce3" +checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb" dependencies = [ - "bitflags 2.9.3", + "bitflags 2.9.4", "libc", ] @@ -1341,34 +1390,33 @@ dependencies = [ [[package]] name = "libz-rs-sys" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "172a788537a2221661b480fee8dc5f96c580eb34fa88764d3205dc356c7e4221" +checksum = "840db8cf39d9ec4dd794376f38acc40d0fc65eec2a8f484f7fd375b84602becd" dependencies = [ "zlib-rs", ] [[package]] name = "linux-raw-sys" -version = "0.9.4" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "lock_api" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" dependencies = [ - "autocfg", "scopeguard", ] [[package]] name = "log" -version = "0.4.27" +version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" [[package]] name = "lz4_flex" @@ -1415,7 +1463,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c738d3789301e957a8f7519318fcbb1b92bb95863b28f6938ae5a05be6259f34" dependencies = [ - "hashbrown", + "hashbrown 0.15.5", "itertools 0.14.0", "libm", "ryu", @@ -1457,6 +1505,29 @@ dependencies = [ "malachite-nz", ] +[[package]] +name = "manyhow" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b33efb3ca6d3b07393750d4030418d594ab1139cee518f0dc88db70fec873587" +dependencies = [ + "manyhow-macros", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "manyhow-macros" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46fce34d199b78b6e6073abf984c9cf5fd3e9330145a93ee0738a7443e371495" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", +] + [[package]] name = "maplit" version = "1.0.2" @@ -1481,9 +1552,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.5" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" [[package]] name = "memmap2" @@ -1516,6 +1587,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", + "simd-adler32", ] [[package]] @@ -1542,7 +1614,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.9.3", + "bitflags 2.9.4", "cfg-if", "cfg_aliases", "libc", @@ -1555,10 +1627,11 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 2.9.3", + "bitflags 2.9.4", "cfg-if", "cfg_aliases", "libc", + "memoffset", ] [[package]] @@ -1626,7 +1699,7 @@ checksum = "77e878c846a8abae00dd069496dbe8751b16ac1c3d6bd2a7283a938e8228f90d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -1649,11 +1722,11 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "openssl" -version = "0.10.73" +version = "0.10.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" +checksum = "24ad14dd45412269e1a30f52ad8f0664f0f4f4a89ee8fe28c3b3527021ebb654" dependencies = [ - "bitflags 2.9.3", + "bitflags 2.9.4", "cfg-if", "foreign-types", "libc", @@ -1670,7 +1743,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -1681,18 +1754,18 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "openssl-src" -version = "300.5.2+3.5.2" +version = "300.5.3+3.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d270b79e2926f5150189d475bc7e9d2c69f9c4697b185fa917d5a32b792d21b4" +checksum = "dc6bad8cd0233b63971e232cc9c5e83039375b8586d2312f31fda85db8f888c2" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.109" +version = "0.9.110" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90096e2e47630d78b7d1c20952dc621f957103f8bc2c8359ec81290d75238571" +checksum = "0a9f0075ba3c21b09f8e8b2026584b1d18d49388648f2fbbf3c97ea8deced8e2" dependencies = [ "cc", "libc", @@ -1719,9 +1792,9 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" dependencies = [ "lock_api", "parking_lot_core", @@ -1729,15 +1802,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.11" +version = "0.9.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.17", + "redox_syscall 0.5.18", "smallvec", - "windows-targets 0.52.6", + "windows-link", ] [[package]] @@ -1826,7 +1899,7 @@ checksum = "52a40bc70c2c58040d2d8b167ba9a5ff59fc9dab7ad44771cfde3dcfde7a09c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -1860,7 +1933,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.106", + "syn", +] + +[[package]] +name = "proc-macro-utils" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeaf08a13de400bc215877b5bdc088f241b12eb42f0a548d3390dc1c56bb7071" +dependencies = [ + "proc-macro2", + "quote", + "smallvec", ] [[package]] @@ -1883,11 +1967,10 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.24.2" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5203598f366b11a02b13aa20cab591229ff0a89fd121a308a5df751d5fc9219" +checksum = "7ba0117f4212101ee6544044dae45abe1083d30ce7b29c4b5cbdfa2354e07383" dependencies = [ - "cfg-if", "indoc", "libc", "memoffset", @@ -1901,19 +1984,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.24.2" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99636d423fa2ca130fa5acde3059308006d46f98caac629418e53f7ebb1e9999" +checksum = "4fc6ddaf24947d12a9aa31ac65431fb1b851b8f4365426e182901eabfb87df5f" dependencies = [ - "once_cell", "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.24.2" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78f9cf92ba9c409279bc3305b5409d90db2d2c22392d443a87df3a1adad59e33" +checksum = "025474d3928738efb38ac36d4744a74a400c901c7596199e20e45d98eb194105" dependencies = [ "libc", "pyo3-build-config", @@ -1921,38 +2003,60 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.24.2" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b999cb1a6ce21f9a6b147dcf1be9ffedf02e0043aec74dc390f3007047cecd9" +checksum = "2e64eb489f22fe1c95911b77c44cc41e7c19f3082fc81cce90f657cdc42ffded" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.106", + "syn", ] [[package]] name = "pyo3-macros-backend" -version = "0.24.2" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "822ece1c7e1012745607d5cf0bcb2874769f0f7cb34c4cde03b9358eb9ef911a" +checksum = "100246c0ecf400b475341b8455a9213344569af29a3c841d29270e53102e0fcf" dependencies = [ "heck", "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.106", + "syn", ] [[package]] name = "quote" -version = "1.0.40" +version = "1.0.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" dependencies = [ "proc-macro2", ] +[[package]] +name = "quote-use" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9619db1197b497a36178cfc736dc96b271fe918875fbf1344c436a7e93d0321e" +dependencies = [ + "quote", + "quote-use-macros", +] + +[[package]] +name = "quote-use-macros" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82ebfb7faafadc06a7ab141a6f67bcfb24cb8beb158c6fe933f2f035afa99f35" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "r-efi" version = "5.3.0" @@ -2034,7 +2138,7 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "getrandom 0.3.3", + "getrandom 0.3.4", ] [[package]] @@ -2065,11 +2169,11 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] name = "redox_syscall" -version = "0.5.17" +version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.9.3", + "bitflags 2.9.4", ] [[package]] @@ -2091,7 +2195,7 @@ checksum = "dc06e6b318142614e4a48bc725abbf08ff166694835c43c9dae5a9009704639a" dependencies = [ "allocator-api2", "bumpalo", - "hashbrown", + "hashbrown 0.15.5", "log", "rustc-hash", "smallvec", @@ -2099,9 +2203,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.11.2" +version = "1.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" dependencies = [ "aho-corasick", "memchr", @@ -2111,9 +2215,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.10" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" dependencies = [ "aho-corasick", "memchr", @@ -2122,9 +2226,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" [[package]] name = "region" @@ -2140,33 +2244,34 @@ dependencies = [ [[package]] name = "result-like" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abf7172fef6a7d056b5c26bf6c826570267562d51697f4982ff3ba4aec68a9df" +checksum = "bffa194499266bd8a1ac7da6ac7355aa0f81ffa1a5db2baaf20dd13854fd6f4e" dependencies = [ "result-like-derive", ] [[package]] name = "result-like-derive" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8d6574c02e894d66370cfc681e5d68fedbc9a548fb55b30a96b3f0ae22d0fe5" +checksum = "01d3b03471c9700a3a6bd166550daaa6124cb4a146ea139fb028e4edaa8f4277" dependencies = [ "pmutil", "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] name = "ruff_python_ast" version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" +source = "git+https://github.com/astral-sh/ruff.git?tag=0.14.1#2bffef59665ce7d2630dfd72ee99846663660db8" dependencies = [ "aho-corasick", - "bitflags 2.9.3", + "bitflags 2.9.4", "compact_str", + "get-size2", "is-macro", "itertools 0.14.0", "memchr", @@ -2174,39 +2279,18 @@ dependencies = [ "ruff_source_file", "ruff_text_size", "rustc-hash", -] - -[[package]] -name = "ruff_python_codegen" -version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" -dependencies = [ - "ruff_python_ast", - "ruff_python_literal", - "ruff_python_parser", - "ruff_source_file", - "ruff_text_size", -] - -[[package]] -name = "ruff_python_literal" -version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" -dependencies = [ - "bitflags 2.9.3", - "itertools 0.14.0", - "ruff_python_ast", - "unic-ucd-category", + "thiserror 2.0.17", ] [[package]] name = "ruff_python_parser" version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" +source = "git+https://github.com/astral-sh/ruff.git?tag=0.14.1#2bffef59665ce7d2630dfd72ee99846663660db8" dependencies = [ - "bitflags 2.9.3", + "bitflags 2.9.4", "bstr", "compact_str", + "get-size2", "memchr", "ruff_python_ast", "ruff_python_trivia", @@ -2215,13 +2299,13 @@ dependencies = [ "static_assertions", "unicode-ident", "unicode-normalization", - "unicode_names2", + "unicode_names2 1.3.0", ] [[package]] name = "ruff_python_trivia" version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" +source = "git+https://github.com/astral-sh/ruff.git?tag=0.14.1#2bffef59665ce7d2630dfd72ee99846663660db8" dependencies = [ "itertools 0.14.0", "ruff_source_file", @@ -2232,7 +2316,7 @@ dependencies = [ [[package]] name = "ruff_source_file" version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" +source = "git+https://github.com/astral-sh/ruff.git?tag=0.14.1#2bffef59665ce7d2630dfd72ee99846663660db8" dependencies = [ "memchr", "ruff_text_size", @@ -2241,7 +2325,10 @@ dependencies = [ [[package]] name = "ruff_text_size" version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" +source = "git+https://github.com/astral-sh/ruff.git?tag=0.14.1#2bffef59665ce7d2630dfd72ee99846663660db8" +dependencies = [ + "get-size2", +] [[package]] name = "rustc-hash" @@ -2251,15 +2338,15 @@ checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustix" -version = "1.0.8" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" dependencies = [ - "bitflags 2.9.3", + "bitflags 2.9.4", "errno", "libc", "linux-raw-sys", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -2290,7 +2377,7 @@ name = "rustpython-codegen" version = "0.4.0" dependencies = [ "ahash", - "bitflags 2.9.3", + "bitflags 2.9.4", "indexmap", "insta", "itertools 0.14.0", @@ -2300,15 +2387,13 @@ dependencies = [ "num-complex", "num-traits", "ruff_python_ast", - "ruff_python_codegen", "ruff_python_parser", - "ruff_source_file", "ruff_text_size", "rustpython-compiler-core", "rustpython-literal", "rustpython-wtf8", - "thiserror 2.0.16", - "unicode_names2", + "thiserror 2.0.17", + "unicode_names2 2.0.0", ] [[package]] @@ -2316,17 +2401,15 @@ name = "rustpython-common" version = "0.4.0" dependencies = [ "ascii", - "bitflags 2.9.3", - "bstr", + "bitflags 2.9.4", "cfg-if", - "getrandom 0.3.3", + "getrandom 0.3.4", "itertools 0.14.0", "libc", "lock_api", "malachite-base", "malachite-bigint", "malachite-q", - "memchr", "num-complex", "num-traits", "once_cell", @@ -2335,7 +2418,7 @@ dependencies = [ "rustpython-literal", "rustpython-wtf8", "siphasher", - "unicode_names2", + "unicode_names2 2.0.0", "widestring", "windows-sys 0.59.0", ] @@ -2344,38 +2427,35 @@ dependencies = [ name = "rustpython-compiler" version = "0.4.0" dependencies = [ - "rand 0.9.2", "ruff_python_ast", "ruff_python_parser", "ruff_source_file", "ruff_text_size", "rustpython-codegen", "rustpython-compiler-core", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] name = "rustpython-compiler-core" version = "0.4.0" dependencies = [ - "bitflags 2.9.3", + "bitflags 2.9.4", "itertools 0.14.0", "lz4_flex", "malachite-bigint", "num-complex", "ruff_source_file", "rustpython-wtf8", - "serde", ] [[package]] name = "rustpython-derive" version = "0.4.0" dependencies = [ - "proc-macro2", "rustpython-compiler", "rustpython-derive-impl", - "syn 2.0.106", + "syn", ] [[package]] @@ -2388,7 +2468,7 @@ dependencies = [ "quote", "rustpython-compiler-core", "rustpython-doc", - "syn 2.0.106", + "syn", "syn-ext", "textwrap", ] @@ -2413,7 +2493,7 @@ dependencies = [ "num-traits", "rustpython-compiler-core", "rustpython-derive", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] @@ -2442,7 +2522,7 @@ dependencies = [ name = "rustpython-sre_engine" version = "0.4.0" dependencies = [ - "bitflags 2.9.3", + "bitflags 2.9.4", "criterion", "num_enum", "optional", @@ -2472,7 +2552,6 @@ dependencies = [ "hex", "indexmap", "itertools 0.14.0", - "junction", "libc", "libsqlite3-sys", "libz-rs-sys", @@ -2483,7 +2562,7 @@ dependencies = [ "memchr", "memmap2", "mt19937", - "nix 0.29.0", + "nix 0.30.1", "num-complex", "num-integer", "num-traits", @@ -2518,7 +2597,7 @@ dependencies = [ "unic-ucd-ident", "unicode-bidi-mirroring", "unicode-casing", - "unicode_names2", + "unicode_names2 2.0.0", "uuid", "widestring", "windows-sys 0.59.0", @@ -2532,7 +2611,7 @@ version = "0.4.0" dependencies = [ "ahash", "ascii", - "bitflags 2.9.3", + "bitflags 2.9.4", "bstr", "caseless", "cfg-if", @@ -2543,7 +2622,7 @@ dependencies = [ "exitcode", "flame", "flamer", - "getrandom 0.3.3", + "getrandom 0.3.4", "glob", "half", "hex", @@ -2557,8 +2636,7 @@ dependencies = [ "log", "malachite-bigint", "memchr", - "memoffset", - "nix 0.29.0", + "nix 0.30.1", "num-complex", "num-integer", "num-traits", @@ -2582,14 +2660,13 @@ dependencies = [ "rustpython-literal", "rustpython-sre_engine", "rustyline", - "schannel", "scoped-tls", "scopeguard", "serde", "static_assertions", "strum", "strum_macros", - "thiserror 2.0.16", + "thiserror 2.0.17", "thread_local", "timsort", "uname", @@ -2597,7 +2674,6 @@ dependencies = [ "unic-ucd-category", "unic-ucd-ident", "unicode-casing", - "unicode_names2", "wasm-bindgen", "which", "widestring", @@ -2642,11 +2718,11 @@ checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "rustyline" -version = "17.0.1" +version = "17.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6614df0b6d4cfb20d1d5e295332921793ce499af3ebc011bf1e393380e1e492" +checksum = "e902948a25149d50edc1a8e0141aad50f54e22ba83ff988cf8f7c9ef07f50564" dependencies = [ - "bitflags 2.9.3", + "bitflags 2.9.4", "cfg-if", "clipboard-win", "fd-lock", @@ -2688,11 +2764,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2709,46 +2785,56 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "serde" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ + "serde_core", "serde_derive", ] [[package]] name = "serde-wasm-bindgen" -version = "0.3.1" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "618365e8e586c22123d692b72a7d791d5ee697817b65a218cdf12a98870af0f7" +checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b" dependencies = [ - "fnv", "js-sys", "serde", "wasm-bindgen", ] +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + [[package]] name = "serde_derive" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] name = "serde_json" -version = "1.0.143" +version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" dependencies = [ "itoa", "memchr", "ryu", "serde", + "serde_core", ] [[package]] @@ -2806,6 +2892,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "similar" version = "2.7.0" @@ -2826,19 +2918,19 @@ checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "socket2" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] name = "stable_deref_trait" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "static_assertions" @@ -2861,7 +2953,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -2872,20 +2964,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "1.0.109" +version = "2.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" +checksum = "2a26dbd934e5451d21ef060c018dae56fc073894c5a7896f882928a76e6d081b" dependencies = [ "proc-macro2", "quote", @@ -2900,7 +2981,7 @@ checksum = "b126de4ef6c2a628a68609dd00733766c3b015894698a438ebdf374933fc31d1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -2909,7 +2990,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ - "bitflags 2.9.3", + "bitflags 2.9.4", "core-foundation", "system-configuration-sys", ] @@ -2926,9 +3007,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.13.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" +checksum = "df7f62577c25e07834649fc3b39fafdc597c0a3527dc1c60129201ccfcbaa50c" [[package]] name = "tcl-sys" @@ -2965,11 +3046,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.16" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" +checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" dependencies = [ - "thiserror-impl 2.0.16", + "thiserror-impl 2.0.17", ] [[package]] @@ -2980,18 +3061,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] name = "thiserror-impl" -version = "2.0.16" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" +checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -3097,15 +3178,15 @@ checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" [[package]] name = "twox-hash" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b907da542cbced5261bd3256de1b3a1bf340a3d37f93425a07362a1d687de56" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" [[package]] name = "typenum" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" [[package]] name = "ucd" @@ -3229,21 +3310,21 @@ dependencies = [ [[package]] name = "unicode-bidi-mirroring" -version = "0.2.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23cb788ffebc92c5948d0e997106233eeb1d8b9512f93f41651f52b6c5f5af86" +checksum = "5dfa6e8c60bb66d49db113e0125ee8711b7647b5579dc7f5f19c42357ed039fe" [[package]] name = "unicode-casing" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "623f59e6af2a98bdafeb93fa277ac8e1e40440973001ca15cf4ae1541cd16d56" +checksum = "061dbb8cc7f108532b6087a0065eff575e892a4bcb503dc57323a197457cc202" [[package]] name = "unicode-ident" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" [[package]] name = "unicode-normalization" @@ -3262,9 +3343,9 @@ checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] name = "unicode-width" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" [[package]] name = "unicode_names2" @@ -3273,7 +3354,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1673eca9782c84de5f81b82e4109dcfb3611c8ba0d52930ec4a9478f547b2dd" dependencies = [ "phf", - "unicode_names2_generator", + "unicode_names2_generator 1.3.0", +] + +[[package]] +name = "unicode_names2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d189085656ca1203291e965444e7f6a2723fbdd1dd9f34f8482e79bafd8338a0" +dependencies = [ + "phf", + "unicode_names2_generator 2.0.0", ] [[package]] @@ -3288,6 +3379,16 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "unicode_names2_generator" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1262662dc96937c71115228ce2e1d30f41db71a7a45d3459e98783ef94052214" +dependencies = [ + "phf_codegen", + "rand 0.8.5", +] + [[package]] name = "unindent" version = "0.2.4" @@ -3302,9 +3403,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.18.0" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f33196643e165781c20a5ead5582283a7dacbb87855d867fbc2df3f81eddc1be" +checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" dependencies = [ "atomic", "js-sys", @@ -3340,45 +3441,46 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] -name = "wasi" -version = "0.14.3+wasi-0.2.4" +name = "wasip2" +version = "1.0.1+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a51ae83037bdd272a9e28ce236db8c07016dd0d50c27038b3f407533c030c95" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" dependencies = [ "wit-bindgen", ] [[package]] name = "wasm-bindgen" -version = "0.2.100" +version = "0.2.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +checksum = "c1da10c01ae9f1ae40cbfac0bac3b1e724b320abfcf52229f80b547c0d250e2d" dependencies = [ "cfg-if", "once_cell", "rustversion", "wasm-bindgen-macro", + "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.100" +version = "0.2.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +checksum = "671c9a5a66f49d8a47345ab942e2cb93c7d1d0339065d4f8139c486121b43b19" dependencies = [ "bumpalo", "log", "proc-macro2", "quote", - "syn 2.0.106", + "syn", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.50" +version = "0.4.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +checksum = "7e038d41e478cc73bae0ff9b36c60cff1c98b8f38f8d7e8061e79ee63608ac5c" dependencies = [ "cfg-if", "js-sys", @@ -3389,9 +3491,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.100" +version = "0.2.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +checksum = "7ca60477e4c59f5f2986c50191cd972e3a50d8a95603bc9434501cf156a9a119" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3399,22 +3501,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.100" +version = "0.2.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +checksum = "9f07d2f20d4da7b26400c9f4a0511e6e0345b040694e8a75bd41d578fa4421d7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.100" +version = "0.2.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +checksum = "bad67dc8b2a1a6e5448428adec4c3e84c43e561d8c9ee8a9e5aabeb193ec41d1" dependencies = [ "unicode-ident", ] @@ -3433,9 +3535,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.77" +version = "0.3.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +checksum = "9367c417a924a74cae129e6a2ae3b47fabb1f8995595ab474029da749a8be120" dependencies = [ "js-sys", "wasm-bindgen", @@ -3464,9 +3566,9 @@ dependencies = [ [[package]] name = "widestring" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd7cf3379ca1aac9eea11fba24fd7e315d621f8dfe35c8d7d2be8b793726e07d" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" [[package]] name = "winapi" @@ -3486,11 +3588,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0978bf7171b3d90bac376700cb56d606feb40f251a475a5d6634613564460b22" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3520,9 +3622,9 @@ dependencies = [ [[package]] name = "windows-core" -version = "0.61.2" +version = "0.62.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ "windows-implement", "windows-interface", @@ -3533,46 +3635,46 @@ dependencies = [ [[package]] name = "windows-implement" -version = "0.60.0" +version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] name = "windows-interface" -version = "0.59.1" +version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] name = "windows-link" -version = "0.1.3" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] name = "windows-result" -version = "0.3.4" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" dependencies = [ "windows-link", ] [[package]] name = "windows-strings" -version = "0.4.2" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" dependencies = [ "windows-link", ] @@ -3601,7 +3703,16 @@ version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" dependencies = [ - "windows-targets 0.53.3", + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", ] [[package]] @@ -3622,19 +3733,19 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.53.3" +version = "0.53.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" dependencies = [ "windows-link", - "windows_aarch64_gnullvm 0.53.0", - "windows_aarch64_msvc 0.53.0", - "windows_i686_gnu 0.53.0", - "windows_i686_gnullvm 0.53.0", - "windows_i686_msvc 0.53.0", - "windows_x86_64_gnu 0.53.0", - "windows_x86_64_gnullvm 0.53.0", - "windows_x86_64_msvc 0.53.0", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", ] [[package]] @@ -3645,9 +3756,9 @@ checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_gnullvm" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" [[package]] name = "windows_aarch64_msvc" @@ -3657,9 +3768,9 @@ checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_aarch64_msvc" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" [[package]] name = "windows_i686_gnu" @@ -3669,9 +3780,9 @@ checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnu" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" [[package]] name = "windows_i686_gnullvm" @@ -3681,9 +3792,9 @@ checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_gnullvm" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" [[package]] name = "windows_i686_msvc" @@ -3693,9 +3804,9 @@ checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_i686_msvc" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" [[package]] name = "windows_x86_64_gnu" @@ -3705,9 +3816,9 @@ checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnu" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" [[package]] name = "windows_x86_64_gnullvm" @@ -3717,9 +3828,9 @@ checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_gnullvm" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" [[package]] name = "windows_x86_64_msvc" @@ -3729,9 +3840,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "windows_x86_64_msvc" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" [[package]] name = "winnow" @@ -3770,15 +3881,15 @@ checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904" [[package]] name = "wit-bindgen" -version = "0.45.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "052283831dbae3d879dc7f51f3d92703a316ca49f91540417d38591826127814" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" [[package]] name = "xml" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72e6e0a83ae73d886ab66fc2f82b598fbbb8f373357d5f2f9f783e50e4d06435" +checksum = "58a4274c410d957424a1502b21126915b45d9956b2f80a88d4f6f906af29facc" [[package]] name = "xz2" @@ -3791,26 +3902,26 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.26" +version = "0.8.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" +checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.26" +version = "0.8.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" +checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] name = "zlib-rs" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626bd9fa9734751fc50d6060752170984d7053f5a39061f524cda68023d4db8a" +checksum = "2f06ae92f42f5e5c42443fd094f245eb656abf56dd7cce9b8b263236565e00f2" diff --git a/Cargo.toml b/Cargo.toml index 70ceebf7a4..3cdc471dc3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,7 +51,7 @@ rustyline = { workspace = true } [dev-dependencies] criterion = { workspace = true } -pyo3 = { version = "0.24", features = ["auto-initialize"] } +pyo3 = { version = "0.26", features = ["auto-initialize"] } [[bench]] name = "execution" @@ -90,7 +90,7 @@ lto = "thin" git = "https://github.com/microsoft/vcpkg" # The revision of the vcpkg repository to use # https://github.com/microsoft/vcpkg/tags -rev = "2024.02.14" +rev = "2025.09.17" [package.metadata.vcpkg.target] x86_64-pc-windows-msvc = { triplet = "x64-windows-static-md", dev-dependencies = ["openssl" ] } @@ -138,7 +138,7 @@ members = [ version = "0.4.0" authors = ["RustPython Team"] edition = "2024" -rust-version = "1.87.0" +rust-version = "1.89.0" repository = "https://github.com/RustPython/RustPython" license = "MIT" @@ -158,34 +158,33 @@ rustpython-sre_engine = { path = "vm/sre_engine", version = "0.4.0" } rustpython-wtf8 = { path = "wtf8", version = "0.4.0" } rustpython-doc = { git = "https://github.com/RustPython/__doc__", tag = "0.3.0", version = "0.3.0" } -ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } -ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } -ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } -ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } -ruff_python_codegen = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } +ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" } +ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" } +ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" } +ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" } -ahash = "0.8.11" +ahash = "0.8.12" ascii = "1.1" -bitflags = "2.9.1" +bitflags = "2.9.4" bstr = "1" cfg-if = "1.0" -chrono = "0.4.39" +chrono = "0.4.42" constant_time_eq = "0.4" -criterion = { version = "0.5", features = ["html_reports"] } +criterion = { version = "0.7", features = ["html_reports"] } crossbeam-utils = "0.8.21" flame = "0.2.2" getrandom = { version = "0.3", features = ["std"] } glob = "0.3" hex = "0.4.3" -indexmap = { version = "2.10.0", features = ["std"] } +indexmap = { version = "2.11.3", features = ["std"] } insta = "1.42" itertools = "0.14.0" is-macro = "0.3.7" -junction = "1.2.0" +junction = "1.3.0" libc = "0.2.169" libffi = "4.1" -log = "0.4.27" -nix = { version = "0.29", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] } +log = "0.4.28" +nix = { version = "0.30", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] } malachite-bigint = "0.6" malachite-q = "0.6" malachite-base = "0.6" @@ -205,9 +204,9 @@ radium = "1.1.1" rand = "0.9" rand_core = { version = "0.9", features = ["os_rng"] } rustix = { version = "1.0", features = ["event"] } -rustyline = "17.0.0" -serde = { version = "1.0.133", default-features = false } -schannel = "0.1.27" +rustyline = "17.0.1" +serde = { version = "1.0.225", default-features = false } +schannel = "0.1.28" scoped-tls = "1" scopeguard = "1" static_assertions = "1.1" @@ -215,16 +214,16 @@ strum = "0.27" strum_macros = "0.27" syn = "2" thiserror = "2.0" -thread_local = "1.1.8" -unicode-casing = "0.1.0" +thread_local = "1.1.9" +unicode-casing = "0.1.1" unic-char-property = "0.9.0" unic-normal = "0.9.0" unic-ucd-age = "0.9.0" unic-ucd-bidi = "0.9.0" unic-ucd-category = "0.9.0" unic-ucd-ident = "0.9.0" -unicode_names2 = "1.3.0" -unicode-bidi-mirroring = "0.2" +unicode_names2 = "2.0.0" +unicode-bidi-mirroring = "0.4" widestring = "1.2.0" windows-sys = "0.59.0" wasm-bindgen = "0.2.100" diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py new file mode 100644 index 0000000000..b3d7b8103e --- /dev/null +++ b/Lib/_opcode_metadata.py @@ -0,0 +1,343 @@ +# This file is generated by Tools/cases_generator/py_metadata_generator.py +# from: +# Python/bytecodes.c +# Do not edit! +_specializations = { + "RESUME": [ + "RESUME_CHECK", + ], + "TO_BOOL": [ + "TO_BOOL_ALWAYS_TRUE", + "TO_BOOL_BOOL", + "TO_BOOL_INT", + "TO_BOOL_LIST", + "TO_BOOL_NONE", + "TO_BOOL_STR", + ], + "BINARY_OP": [ + "BINARY_OP_MULTIPLY_INT", + "BINARY_OP_ADD_INT", + "BINARY_OP_SUBTRACT_INT", + "BINARY_OP_MULTIPLY_FLOAT", + "BINARY_OP_ADD_FLOAT", + "BINARY_OP_SUBTRACT_FLOAT", + "BINARY_OP_ADD_UNICODE", + "BINARY_OP_INPLACE_ADD_UNICODE", + ], + "BINARY_SUBSCR": [ + "BINARY_SUBSCR_DICT", + "BINARY_SUBSCR_GETITEM", + "BINARY_SUBSCR_LIST_INT", + "BINARY_SUBSCR_STR_INT", + "BINARY_SUBSCR_TUPLE_INT", + ], + "STORE_SUBSCR": [ + "STORE_SUBSCR_DICT", + "STORE_SUBSCR_LIST_INT", + ], + "SEND": [ + "SEND_GEN", + ], + "UNPACK_SEQUENCE": [ + "UNPACK_SEQUENCE_TWO_TUPLE", + "UNPACK_SEQUENCE_TUPLE", + "UNPACK_SEQUENCE_LIST", + ], + "STORE_ATTR": [ + "STORE_ATTR_INSTANCE_VALUE", + "STORE_ATTR_SLOT", + "STORE_ATTR_WITH_HINT", + ], + "LOAD_GLOBAL": [ + "LOAD_GLOBAL_MODULE", + "LOAD_GLOBAL_BUILTIN", + ], + "LOAD_SUPER_ATTR": [ + "LOAD_SUPER_ATTR_ATTR", + "LOAD_SUPER_ATTR_METHOD", + ], + "LOAD_ATTR": [ + "LOAD_ATTR_INSTANCE_VALUE", + "LOAD_ATTR_MODULE", + "LOAD_ATTR_WITH_HINT", + "LOAD_ATTR_SLOT", + "LOAD_ATTR_CLASS", + "LOAD_ATTR_PROPERTY", + "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", + "LOAD_ATTR_METHOD_WITH_VALUES", + "LOAD_ATTR_METHOD_NO_DICT", + "LOAD_ATTR_METHOD_LAZY_DICT", + "LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", + "LOAD_ATTR_NONDESCRIPTOR_NO_DICT", + ], + "COMPARE_OP": [ + "COMPARE_OP_FLOAT", + "COMPARE_OP_INT", + "COMPARE_OP_STR", + ], + "CONTAINS_OP": [ + "CONTAINS_OP_SET", + "CONTAINS_OP_DICT", + ], + "FOR_ITER": [ + "FOR_ITER_LIST", + "FOR_ITER_TUPLE", + "FOR_ITER_RANGE", + "FOR_ITER_GEN", + ], + "CALL": [ + "CALL_BOUND_METHOD_EXACT_ARGS", + "CALL_PY_EXACT_ARGS", + "CALL_TYPE_1", + "CALL_STR_1", + "CALL_TUPLE_1", + "CALL_BUILTIN_CLASS", + "CALL_BUILTIN_O", + "CALL_BUILTIN_FAST", + "CALL_BUILTIN_FAST_WITH_KEYWORDS", + "CALL_LEN", + "CALL_ISINSTANCE", + "CALL_LIST_APPEND", + "CALL_METHOD_DESCRIPTOR_O", + "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + "CALL_METHOD_DESCRIPTOR_NOARGS", + "CALL_METHOD_DESCRIPTOR_FAST", + "CALL_ALLOC_AND_ENTER_INIT", + "CALL_PY_GENERAL", + "CALL_BOUND_METHOD_GENERAL", + "CALL_NON_PY_GENERAL", + ], +} + +_specialized_opmap = { + 'BINARY_OP_ADD_FLOAT': 150, + 'BINARY_OP_ADD_INT': 151, + 'BINARY_OP_ADD_UNICODE': 152, + 'BINARY_OP_INPLACE_ADD_UNICODE': 3, + 'BINARY_OP_MULTIPLY_FLOAT': 153, + 'BINARY_OP_MULTIPLY_INT': 154, + 'BINARY_OP_SUBTRACT_FLOAT': 155, + 'BINARY_OP_SUBTRACT_INT': 156, + 'BINARY_SUBSCR_DICT': 157, + 'BINARY_SUBSCR_GETITEM': 158, + 'BINARY_SUBSCR_LIST_INT': 159, + 'BINARY_SUBSCR_STR_INT': 160, + 'BINARY_SUBSCR_TUPLE_INT': 161, + 'CALL_ALLOC_AND_ENTER_INIT': 162, + 'CALL_BOUND_METHOD_EXACT_ARGS': 163, + 'CALL_BOUND_METHOD_GENERAL': 164, + 'CALL_BUILTIN_CLASS': 165, + 'CALL_BUILTIN_FAST': 166, + 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 167, + 'CALL_BUILTIN_O': 168, + 'CALL_ISINSTANCE': 169, + 'CALL_LEN': 170, + 'CALL_LIST_APPEND': 171, + 'CALL_METHOD_DESCRIPTOR_FAST': 172, + 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 173, + 'CALL_METHOD_DESCRIPTOR_NOARGS': 174, + 'CALL_METHOD_DESCRIPTOR_O': 175, + 'CALL_NON_PY_GENERAL': 176, + 'CALL_PY_EXACT_ARGS': 177, + 'CALL_PY_GENERAL': 178, + 'CALL_STR_1': 179, + 'CALL_TUPLE_1': 180, + 'CALL_TYPE_1': 181, + 'COMPARE_OP_FLOAT': 182, + 'COMPARE_OP_INT': 183, + 'COMPARE_OP_STR': 184, + 'CONTAINS_OP_DICT': 185, + 'CONTAINS_OP_SET': 186, + 'FOR_ITER_GEN': 187, + 'FOR_ITER_LIST': 188, + 'FOR_ITER_RANGE': 189, + 'FOR_ITER_TUPLE': 190, + 'LOAD_ATTR_CLASS': 191, + 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 192, + 'LOAD_ATTR_INSTANCE_VALUE': 193, + 'LOAD_ATTR_METHOD_LAZY_DICT': 194, + 'LOAD_ATTR_METHOD_NO_DICT': 195, + 'LOAD_ATTR_METHOD_WITH_VALUES': 196, + 'LOAD_ATTR_MODULE': 197, + 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 198, + 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 199, + 'LOAD_ATTR_PROPERTY': 200, + 'LOAD_ATTR_SLOT': 201, + 'LOAD_ATTR_WITH_HINT': 202, + 'LOAD_GLOBAL_BUILTIN': 203, + 'LOAD_GLOBAL_MODULE': 204, + 'LOAD_SUPER_ATTR_ATTR': 205, + 'LOAD_SUPER_ATTR_METHOD': 206, + 'RESUME_CHECK': 207, + 'SEND_GEN': 208, + 'STORE_ATTR_INSTANCE_VALUE': 209, + 'STORE_ATTR_SLOT': 210, + 'STORE_ATTR_WITH_HINT': 211, + 'STORE_SUBSCR_DICT': 212, + 'STORE_SUBSCR_LIST_INT': 213, + 'TO_BOOL_ALWAYS_TRUE': 214, + 'TO_BOOL_BOOL': 215, + 'TO_BOOL_INT': 216, + 'TO_BOOL_LIST': 217, + 'TO_BOOL_NONE': 218, + 'TO_BOOL_STR': 219, + 'UNPACK_SEQUENCE_LIST': 220, + 'UNPACK_SEQUENCE_TUPLE': 221, + 'UNPACK_SEQUENCE_TWO_TUPLE': 222, +} + +opmap = { + 'CACHE': 0, + 'RESERVED': 17, + 'RESUME': 149, + 'INSTRUMENTED_LINE': 254, + 'BEFORE_ASYNC_WITH': 1, + 'BEFORE_WITH': 2, + 'BINARY_SLICE': 4, + 'BINARY_SUBSCR': 5, + 'CHECK_EG_MATCH': 6, + 'CHECK_EXC_MATCH': 7, + 'CLEANUP_THROW': 8, + 'DELETE_SUBSCR': 9, + 'END_ASYNC_FOR': 10, + 'END_FOR': 11, + 'END_SEND': 12, + 'EXIT_INIT_CHECK': 13, + 'FORMAT_SIMPLE': 14, + 'FORMAT_WITH_SPEC': 15, + 'GET_AITER': 16, + 'GET_ANEXT': 18, + 'GET_ITER': 19, + 'GET_LEN': 20, + 'GET_YIELD_FROM_ITER': 21, + 'INTERPRETER_EXIT': 22, + 'LOAD_ASSERTION_ERROR': 23, + 'LOAD_BUILD_CLASS': 24, + 'LOAD_LOCALS': 25, + 'MAKE_FUNCTION': 26, + 'MATCH_KEYS': 27, + 'MATCH_MAPPING': 28, + 'MATCH_SEQUENCE': 29, + 'NOP': 30, + 'POP_EXCEPT': 31, + 'POP_TOP': 32, + 'PUSH_EXC_INFO': 33, + 'PUSH_NULL': 34, + 'RETURN_GENERATOR': 35, + 'RETURN_VALUE': 36, + 'SETUP_ANNOTATIONS': 37, + 'STORE_SLICE': 38, + 'STORE_SUBSCR': 39, + 'TO_BOOL': 40, + 'UNARY_INVERT': 41, + 'UNARY_NEGATIVE': 42, + 'UNARY_NOT': 43, + 'WITH_EXCEPT_START': 44, + 'BINARY_OP': 45, + 'BUILD_CONST_KEY_MAP': 46, + 'BUILD_LIST': 47, + 'BUILD_MAP': 48, + 'BUILD_SET': 49, + 'BUILD_SLICE': 50, + 'BUILD_STRING': 51, + 'BUILD_TUPLE': 52, + 'CALL': 53, + 'CALL_FUNCTION_EX': 54, + 'CALL_INTRINSIC_1': 55, + 'CALL_INTRINSIC_2': 56, + 'CALL_KW': 57, + 'COMPARE_OP': 58, + 'CONTAINS_OP': 59, + 'CONVERT_VALUE': 60, + 'COPY': 61, + 'COPY_FREE_VARS': 62, + 'DELETE_ATTR': 63, + 'DELETE_DEREF': 64, + 'DELETE_FAST': 65, + 'DELETE_GLOBAL': 66, + 'DELETE_NAME': 67, + 'DICT_MERGE': 68, + 'DICT_UPDATE': 69, + 'ENTER_EXECUTOR': 70, + 'EXTENDED_ARG': 71, + 'FOR_ITER': 72, + 'GET_AWAITABLE': 73, + 'IMPORT_FROM': 74, + 'IMPORT_NAME': 75, + 'IS_OP': 76, + 'JUMP_BACKWARD': 77, + 'JUMP_BACKWARD_NO_INTERRUPT': 78, + 'JUMP_FORWARD': 79, + 'LIST_APPEND': 80, + 'LIST_EXTEND': 81, + 'LOAD_ATTR': 82, + 'LOAD_CONST': 83, + 'LOAD_DEREF': 84, + 'LOAD_FAST': 85, + 'LOAD_FAST_AND_CLEAR': 86, + 'LOAD_FAST_CHECK': 87, + 'LOAD_FAST_LOAD_FAST': 88, + 'LOAD_FROM_DICT_OR_DEREF': 89, + 'LOAD_FROM_DICT_OR_GLOBALS': 90, + 'LOAD_GLOBAL': 91, + 'LOAD_NAME': 92, + 'LOAD_SUPER_ATTR': 93, + 'MAKE_CELL': 94, + 'MAP_ADD': 95, + 'MATCH_CLASS': 96, + 'POP_JUMP_IF_FALSE': 97, + 'POP_JUMP_IF_NONE': 98, + 'POP_JUMP_IF_NOT_NONE': 99, + 'POP_JUMP_IF_TRUE': 100, + 'RAISE_VARARGS': 101, + 'RERAISE': 102, + 'RETURN_CONST': 103, + 'SEND': 104, + 'SET_ADD': 105, + 'SET_FUNCTION_ATTRIBUTE': 106, + 'SET_UPDATE': 107, + 'STORE_ATTR': 108, + 'STORE_DEREF': 109, + 'STORE_FAST': 110, + 'STORE_FAST_LOAD_FAST': 111, + 'STORE_FAST_STORE_FAST': 112, + 'STORE_GLOBAL': 113, + 'STORE_NAME': 114, + 'SWAP': 115, + 'UNPACK_EX': 116, + 'UNPACK_SEQUENCE': 117, + 'YIELD_VALUE': 118, + 'INSTRUMENTED_RESUME': 236, + 'INSTRUMENTED_END_FOR': 237, + 'INSTRUMENTED_END_SEND': 238, + 'INSTRUMENTED_RETURN_VALUE': 239, + 'INSTRUMENTED_RETURN_CONST': 240, + 'INSTRUMENTED_YIELD_VALUE': 241, + 'INSTRUMENTED_LOAD_SUPER_ATTR': 242, + 'INSTRUMENTED_FOR_ITER': 243, + 'INSTRUMENTED_CALL': 244, + 'INSTRUMENTED_CALL_KW': 245, + 'INSTRUMENTED_CALL_FUNCTION_EX': 246, + 'INSTRUMENTED_INSTRUCTION': 247, + 'INSTRUMENTED_JUMP_FORWARD': 248, + 'INSTRUMENTED_JUMP_BACKWARD': 249, + 'INSTRUMENTED_POP_JUMP_IF_TRUE': 250, + 'INSTRUMENTED_POP_JUMP_IF_FALSE': 251, + 'INSTRUMENTED_POP_JUMP_IF_NONE': 252, + 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 253, + 'JUMP': 256, + 'JUMP_NO_INTERRUPT': 257, + 'LOAD_CLOSURE': 258, + 'LOAD_METHOD': 259, + 'LOAD_SUPER_METHOD': 260, + 'LOAD_ZERO_SUPER_ATTR': 261, + 'LOAD_ZERO_SUPER_METHOD': 262, + 'POP_BLOCK': 263, + 'SETUP_CLEANUP': 264, + 'SETUP_FINALLY': 265, + 'SETUP_WITH': 266, + 'STORE_FAST_MAYBE_NULL': 267, +} + +HAVE_ARGUMENT = 44 +MIN_INSTRUMENTED_OPCODE = 236 diff --git a/Lib/_strptime.py b/Lib/_strptime.py index 798cf9f9d3..a07eb5f923 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -10,10 +10,13 @@ strptime -- Calculates the time struct represented by the passed-in string """ +import os import time import locale import calendar +import re from re import compile as re_compile +from re import sub as re_sub from re import IGNORECASE from re import escape as re_escape from datetime import (date as datetime_date, @@ -27,6 +30,41 @@ def _getlang(): # Figure out what the current language is set to. return locale.getlocale(locale.LC_TIME) +def _findall(haystack, needle): + # Find all positions of needle in haystack. + if not needle: + return + i = 0 + while True: + i = haystack.find(needle, i) + if i < 0: + break + yield i + i += len(needle) + +def _fixmonths(months): + yield from months + # The lower case of 'İ' ('\u0130') is 'i\u0307'. + # The re module only supports 1-to-1 character matching in + # case-insensitive mode. + for s in months: + if 'i\u0307' in s: + yield s.replace('i\u0307', '\u0130') + +lzh_TW_alt_digits = ( + # 〇:一:二:三:四:五:六:七:八:九 + '\u3007', '\u4e00', '\u4e8c', '\u4e09', '\u56db', + '\u4e94', '\u516d', '\u4e03', '\u516b', '\u4e5d', + # 十:十一:十二:十三:十四:十五:十六:十七:十八:十九 + '\u5341', '\u5341\u4e00', '\u5341\u4e8c', '\u5341\u4e09', '\u5341\u56db', + '\u5341\u4e94', '\u5341\u516d', '\u5341\u4e03', '\u5341\u516b', '\u5341\u4e5d', + # 廿:廿一:廿二:廿三:廿四:廿五:廿六:廿七:廿八:廿九 + '\u5eff', '\u5eff\u4e00', '\u5eff\u4e8c', '\u5eff\u4e09', '\u5eff\u56db', + '\u5eff\u4e94', '\u5eff\u516d', '\u5eff\u4e03', '\u5eff\u516b', '\u5eff\u4e5d', + # 卅:卅一 + '\u5345', '\u5345\u4e00') + + class LocaleTime(object): """Stores and handles locale-specific information related to time. @@ -70,6 +108,7 @@ def __init__(self): self.__calc_weekday() self.__calc_month() self.__calc_am_pm() + self.__calc_alt_digits() self.__calc_timezone() self.__calc_date_time() if _getlang() != self.lang: @@ -101,53 +140,184 @@ def __calc_am_pm(self): am_pm = [] for hour in (1, 22): time_tuple = time.struct_time((1999,3,17,hour,44,55,2,76,0)) - am_pm.append(time.strftime("%p", time_tuple).lower()) + # br_FR has AM/PM info (' ',' '). + am_pm.append(time.strftime("%p", time_tuple).lower().strip()) self.am_pm = am_pm + def __calc_alt_digits(self): + # Set self.LC_alt_digits by using time.strftime(). + + # The magic data should contain all decimal digits. + time_tuple = time.struct_time((1998, 1, 27, 10, 43, 56, 1, 27, 0)) + s = time.strftime("%x%X", time_tuple) + if s.isascii(): + # Fast path -- all digits are ASCII. + self.LC_alt_digits = () + return + + digits = ''.join(sorted(set(re.findall(r'\d', s)))) + if len(digits) == 10 and ord(digits[-1]) == ord(digits[0]) + 9: + # All 10 decimal digits from the same set. + if digits.isascii(): + # All digits are ASCII. + self.LC_alt_digits = () + return + + self.LC_alt_digits = [a + b for a in digits for b in digits] + # Test whether the numbers contain leading zero. + time_tuple2 = time.struct_time((2000, 1, 1, 1, 1, 1, 5, 1, 0)) + if self.LC_alt_digits[1] not in time.strftime("%x %X", time_tuple2): + self.LC_alt_digits[:10] = digits + return + + # Either non-Gregorian calendar or non-decimal numbers. + if {'\u4e00', '\u4e03', '\u4e5d', '\u5341', '\u5eff'}.issubset(s): + # lzh_TW + self.LC_alt_digits = lzh_TW_alt_digits + return + + self.LC_alt_digits = None + def __calc_date_time(self): - # Set self.date_time, self.date, & self.time by using - # time.strftime(). + # Set self.LC_date_time, self.LC_date, self.LC_time and + # self.LC_time_ampm by using time.strftime(). # Use (1999,3,17,22,44,55,2,76,0) for magic date because the amount of # overloaded numbers is minimized. The order in which searches for # values within the format string is very important; it eliminates # possible ambiguity for what something represents. time_tuple = time.struct_time((1999,3,17,22,44,55,2,76,0)) - date_time = [None, None, None] - date_time[0] = time.strftime("%c", time_tuple).lower() - date_time[1] = time.strftime("%x", time_tuple).lower() - date_time[2] = time.strftime("%X", time_tuple).lower() - replacement_pairs = [('%', '%%'), (self.f_weekday[2], '%A'), - (self.f_month[3], '%B'), (self.a_weekday[2], '%a'), - (self.a_month[3], '%b'), (self.am_pm[1], '%p'), - ('1999', '%Y'), ('99', '%y'), ('22', '%H'), - ('44', '%M'), ('55', '%S'), ('76', '%j'), - ('17', '%d'), ('03', '%m'), ('3', '%m'), - # '3' needed for when no leading zero. - ('2', '%w'), ('10', '%I')] - replacement_pairs.extend([(tz, "%Z") for tz_values in self.timezone - for tz in tz_values]) - for offset,directive in ((0,'%c'), (1,'%x'), (2,'%X')): - current_format = date_time[offset] - for old, new in replacement_pairs: + time_tuple2 = time.struct_time((1999,1,3,1,1,1,6,3,0)) + replacement_pairs = [] + + # Non-ASCII digits + if self.LC_alt_digits or self.LC_alt_digits is None: + for n, d in [(19, '%OC'), (99, '%Oy'), (22, '%OH'), + (44, '%OM'), (55, '%OS'), (17, '%Od'), + (3, '%Om'), (2, '%Ow'), (10, '%OI')]: + if self.LC_alt_digits is None: + s = chr(0x660 + n // 10) + chr(0x660 + n % 10) + replacement_pairs.append((s, d)) + if n < 10: + replacement_pairs.append((s[1], d)) + elif len(self.LC_alt_digits) > n: + replacement_pairs.append((self.LC_alt_digits[n], d)) + else: + replacement_pairs.append((time.strftime(d, time_tuple), d)) + replacement_pairs += [ + ('1999', '%Y'), ('99', '%y'), ('22', '%H'), + ('44', '%M'), ('55', '%S'), ('76', '%j'), + ('17', '%d'), ('03', '%m'), ('3', '%m'), + # '3' needed for when no leading zero. + ('2', '%w'), ('10', '%I'), + ] + + date_time = [] + for directive in ('%c', '%x', '%X', '%r'): + current_format = time.strftime(directive, time_tuple).lower() + current_format = current_format.replace('%', '%%') + # The month and the day of the week formats are treated specially + # because of a possible ambiguity in some locales where the full + # and abbreviated names are equal or names of different types + # are equal. See doc of __find_month_format for more details. + lst, fmt = self.__find_weekday_format(directive) + if lst: + current_format = current_format.replace(lst[2], fmt, 1) + lst, fmt = self.__find_month_format(directive) + if lst: + current_format = current_format.replace(lst[3], fmt, 1) + if self.am_pm[1]: # Must deal with possible lack of locale info # manifesting itself as the empty string (e.g., Swedish's # lack of AM/PM info) or a platform returning a tuple of empty # strings (e.g., MacOS 9 having timezone as ('','')). - if old: - current_format = current_format.replace(old, new) + current_format = current_format.replace(self.am_pm[1], '%p') + for tz_values in self.timezone: + for tz in tz_values: + if tz: + current_format = current_format.replace(tz, "%Z") + # Transform all non-ASCII digits to digits in range U+0660 to U+0669. + if not current_format.isascii() and self.LC_alt_digits is None: + current_format = re_sub(r'\d(?3[0-1]|[1-2]\d|0[1-9]|[1-9]| [1-9])", 'f': r"(?P[0-9]{1,6})", - 'H': r"(?P2[0-3]|[0-1]\d|\d)", - 'I': r"(?P1[0-2]|0[1-9]|[1-9])", + 'H': r"(?P2[0-3]|[0-1]\d|\d| \d)", + 'k': r"(?P2[0-3]|[0-1]\d|\d| \d)", + 'I': r"(?P1[0-2]|0[1-9]|[1-9]| [1-9])", + 'l': r"(?P1[0-2]|0[1-9]|[1-9]| [1-9])", 'G': r"(?P\d\d\d\d)", 'j': r"(?P36[0-6]|3[0-5]\d|[1-2]\d\d|0[1-9]\d|00[1-9]|[1-9]\d|0[1-9]|[1-9])", 'm': r"(?P1[0-2]|0[1-9]|[1-9])", @@ -198,25 +370,60 @@ def __init__(self, locale_time=None): 'V': r"(?P5[0-3]|0[1-9]|[1-4]\d|\d)", # W is set below by using 'U' 'y': r"(?P\d\d)", - #XXX: Does 'Y' need to worry about having less or more than - # 4 digits? 'Y': r"(?P\d\d\d\d)", 'z': r"(?P[+-]\d\d:?[0-5]\d(:?[0-5]\d(\.\d{1,6})?)?|(?-i:Z))", 'A': self.__seqToRE(self.locale_time.f_weekday, 'A'), 'a': self.__seqToRE(self.locale_time.a_weekday, 'a'), - 'B': self.__seqToRE(self.locale_time.f_month[1:], 'B'), - 'b': self.__seqToRE(self.locale_time.a_month[1:], 'b'), + 'B': self.__seqToRE(_fixmonths(self.locale_time.f_month[1:]), 'B'), + 'b': self.__seqToRE(_fixmonths(self.locale_time.a_month[1:]), 'b'), 'p': self.__seqToRE(self.locale_time.am_pm, 'p'), 'Z': self.__seqToRE((tz for tz_names in self.locale_time.timezone for tz in tz_names), 'Z'), - '%': '%'}) - base.__setitem__('W', base.__getitem__('U').replace('U', 'W')) - base.__setitem__('c', self.pattern(self.locale_time.LC_date_time)) - base.__setitem__('x', self.pattern(self.locale_time.LC_date)) + '%': '%'} + if self.locale_time.LC_alt_digits is None: + for d in 'dmyCHIMS': + mapping['O' + d] = r'(?P<%s>\d\d|\d| \d)' % d + mapping['Ow'] = r'(?P\d)' + else: + mapping.update({ + 'Od': self.__seqToRE(self.locale_time.LC_alt_digits[1:32], 'd', + '3[0-1]|[1-2][0-9]|0[1-9]|[1-9]'), + 'Om': self.__seqToRE(self.locale_time.LC_alt_digits[1:13], 'm', + '1[0-2]|0[1-9]|[1-9]'), + 'Ow': self.__seqToRE(self.locale_time.LC_alt_digits[:7], 'w', + '[0-6]'), + 'Oy': self.__seqToRE(self.locale_time.LC_alt_digits, 'y', + '[0-9][0-9]'), + 'OC': self.__seqToRE(self.locale_time.LC_alt_digits, 'C', + '[0-9][0-9]'), + 'OH': self.__seqToRE(self.locale_time.LC_alt_digits[:24], 'H', + '2[0-3]|[0-1][0-9]|[0-9]'), + 'OI': self.__seqToRE(self.locale_time.LC_alt_digits[1:13], 'I', + '1[0-2]|0[1-9]|[1-9]'), + 'OM': self.__seqToRE(self.locale_time.LC_alt_digits[:60], 'M', + '[0-5][0-9]|[0-9]'), + 'OS': self.__seqToRE(self.locale_time.LC_alt_digits[:62], 'S', + '6[0-1]|[0-5][0-9]|[0-9]'), + }) + mapping.update({ + 'e': mapping['d'], + 'Oe': mapping['Od'], + 'P': mapping['p'], + 'Op': mapping['p'], + 'W': mapping['U'].replace('U', 'W'), + }) + mapping['W'] = mapping['U'].replace('U', 'W') + + base.__init__(mapping) + base.__setitem__('T', self.pattern('%H:%M:%S')) + base.__setitem__('R', self.pattern('%H:%M')) + base.__setitem__('r', self.pattern(self.locale_time.LC_time_ampm)) base.__setitem__('X', self.pattern(self.locale_time.LC_time)) + base.__setitem__('x', self.pattern(self.locale_time.LC_date)) + base.__setitem__('c', self.pattern(self.locale_time.LC_date_time)) - def __seqToRE(self, to_convert, directive): + def __seqToRE(self, to_convert, directive, altregex=None): """Convert a list to a regex string for matching a directive. Want possible matching values to be from longest to shortest. This @@ -232,8 +439,9 @@ def __seqToRE(self, to_convert, directive): else: return '' regex = '|'.join(re_escape(stuff) for stuff in to_convert) - regex = '(?P<%s>%s' % (directive, regex) - return '%s)' % regex + if altregex is not None: + regex += '|' + altregex + return '(?P<%s>%s)' % (directive, regex) def pattern(self, format): """Return regex pattern for the format string. @@ -242,21 +450,36 @@ def pattern(self, format): regex syntax are escaped. """ - processed_format = '' # The sub() call escapes all characters that might be misconstrued # as regex syntax. Cannot use re.escape since we have to deal with # format directives (%m, etc.). - regex_chars = re_compile(r"([\\.^$*+?\(\){}\[\]|])") - format = regex_chars.sub(r"\\\1", format) - whitespace_replacement = re_compile(r'\s+') - format = whitespace_replacement.sub(r'\\s+', format) - while '%' in format: - directive_index = format.index('%')+1 - processed_format = "%s%s%s" % (processed_format, - format[:directive_index-1], - self[format[directive_index]]) - format = format[directive_index+1:] - return "%s%s" % (processed_format, format) + format = re_sub(r"([\\.^$*+?\(\){}\[\]|])", r"\\\1", format) + format = re_sub(r'\s+', r'\\s+', format) + format = re_sub(r"'", "['\u02bc]", format) # needed for br_FR + year_in_format = False + day_of_month_in_format = False + def repl(m): + format_char = m[1] + match format_char: + case 'Y' | 'y' | 'G': + nonlocal year_in_format + year_in_format = True + case 'd': + nonlocal day_of_month_in_format + day_of_month_in_format = True + return self[format_char] + format = re_sub(r'%[-_0^#]*[0-9]*([OE]?\\?.?)', repl, format) + if day_of_month_in_format and not year_in_format: + import warnings + warnings.warn("""\ +Parsing dates involving a day of month without a year specified is ambiguious +and fails to parse leap day. The default behavior will change in Python 3.15 +to either always raise an exception or to use a different default year (TBD). +To avoid trouble, add a specific year to the input & format. +See https://github.com/python/cpython/issues/70647.""", + DeprecationWarning, + skip_file_prefixes=(os.path.dirname(__file__),)) + return format def compile(self, format): """Return a compiled re object for the format string.""" @@ -319,14 +542,13 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): # \\, in which case it was a stray % but with a space after it except KeyError as err: bad_directive = err.args[0] - if bad_directive == "\\": - bad_directive = "%" del err + bad_directive = bad_directive.replace('\\s', '') + if not bad_directive: + raise ValueError("stray %% in format '%s'" % format) from None + bad_directive = bad_directive.replace('\\', '', 1) raise ValueError("'%s' is a bad directive in format '%s'" % (bad_directive, format)) from None - # IndexError only occurs when the format string is "%" - except IndexError: - raise ValueError("stray %% in format '%s'" % format) from None _regex_cache[format] = format_regex found = format_regex.match(data_string) if not found: @@ -348,6 +570,15 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): # values weekday = julian = None found_dict = found.groupdict() + if locale_time.LC_alt_digits: + def parse_int(s): + try: + return locale_time.LC_alt_digits.index(s) + except ValueError: + return int(s) + else: + parse_int = int + for group_key in found_dict.keys(): # Directives not explicitly handled below: # c, x, X @@ -355,30 +586,34 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): # U, W # worthless without day of the week if group_key == 'y': - year = int(found_dict['y']) - # Open Group specification for strptime() states that a %y - #value in the range of [00, 68] is in the century 2000, while - #[69,99] is in the century 1900 - if year <= 68: - year += 2000 + year = parse_int(found_dict['y']) + if 'C' in found_dict: + century = parse_int(found_dict['C']) + year += century * 100 else: - year += 1900 + # Open Group specification for strptime() states that a %y + #value in the range of [00, 68] is in the century 2000, while + #[69,99] is in the century 1900 + if year <= 68: + year += 2000 + else: + year += 1900 elif group_key == 'Y': year = int(found_dict['Y']) elif group_key == 'G': iso_year = int(found_dict['G']) elif group_key == 'm': - month = int(found_dict['m']) + month = parse_int(found_dict['m']) elif group_key == 'B': month = locale_time.f_month.index(found_dict['B'].lower()) elif group_key == 'b': month = locale_time.a_month.index(found_dict['b'].lower()) elif group_key == 'd': - day = int(found_dict['d']) + day = parse_int(found_dict['d']) elif group_key == 'H': - hour = int(found_dict['H']) + hour = parse_int(found_dict['H']) elif group_key == 'I': - hour = int(found_dict['I']) + hour = parse_int(found_dict['I']) ampm = found_dict.get('p', '').lower() # If there was no AM/PM indicator, we'll treat this like AM if ampm in ('', locale_time.am_pm[0]): @@ -394,9 +629,9 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): if hour != 12: hour += 12 elif group_key == 'M': - minute = int(found_dict['M']) + minute = parse_int(found_dict['M']) elif group_key == 'S': - second = int(found_dict['S']) + second = parse_int(found_dict['S']) elif group_key == 'f': s = found_dict['f'] # Pad to always return microseconds. diff --git a/Lib/codecs.py b/Lib/codecs.py index 82f23983e7..e4f4e1b5c0 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -111,6 +111,9 @@ def __repr__(self): (self.__class__.__module__, self.__class__.__qualname__, self.name, id(self)) + def __getnewargs__(self): + return tuple(self) + class Codec: """ Defines the interface for stateless encoders/decoders. @@ -615,7 +618,7 @@ def readlines(self, sizehint=None, keepends=True): method and are included in the list entries. sizehint, if given, is ignored since there is no efficient - way to finding the true end-of-line. + way of finding the true end-of-line. """ data = self.read() @@ -706,13 +709,13 @@ def read(self, size=-1): return self.reader.read(size) - def readline(self, size=None): + def readline(self, size=None, keepends=True): - return self.reader.readline(size) + return self.reader.readline(size, keepends) - def readlines(self, sizehint=None): + def readlines(self, sizehint=None, keepends=True): - return self.reader.readlines(sizehint) + return self.reader.readlines(sizehint, keepends) def __next__(self): diff --git a/Lib/encodings/aliases.py b/Lib/encodings/aliases.py index d85afd6d5c..6a5ca046b5 100644 --- a/Lib/encodings/aliases.py +++ b/Lib/encodings/aliases.py @@ -209,6 +209,7 @@ 'ms932' : 'cp932', 'mskanji' : 'cp932', 'ms_kanji' : 'cp932', + 'windows_31j' : 'cp932', # cp949 codec '949' : 'cp949', diff --git a/Lib/encodings/idna.py b/Lib/encodings/idna.py index 5396047a7f..0c90b4c9fe 100644 --- a/Lib/encodings/idna.py +++ b/Lib/encodings/idna.py @@ -11,7 +11,7 @@ sace_prefix = "xn--" # This assumes query strings, so AllowUnassigned is true -def nameprep(label): +def nameprep(label): # type: (str) -> str # Map newlabel = [] for c in label: @@ -25,7 +25,7 @@ def nameprep(label): label = unicodedata.normalize("NFKC", label) # Prohibit - for c in label: + for i, c in enumerate(label): if stringprep.in_table_c12(c) or \ stringprep.in_table_c22(c) or \ stringprep.in_table_c3(c) or \ @@ -35,7 +35,7 @@ def nameprep(label): stringprep.in_table_c7(c) or \ stringprep.in_table_c8(c) or \ stringprep.in_table_c9(c): - raise UnicodeError("Invalid character %r" % c) + raise UnicodeEncodeError("idna", label, i, i+1, f"Invalid character {c!r}") # Check bidi RandAL = [stringprep.in_table_d1(x) for x in label] @@ -46,29 +46,38 @@ def nameprep(label): # This is table C.8, which was already checked # 2) If a string contains any RandALCat character, the string # MUST NOT contain any LCat character. - if any(stringprep.in_table_d2(x) for x in label): - raise UnicodeError("Violation of BIDI requirement 2") + for i, x in enumerate(label): + if stringprep.in_table_d2(x): + raise UnicodeEncodeError("idna", label, i, i+1, + "Violation of BIDI requirement 2") # 3) If a string contains any RandALCat character, a # RandALCat character MUST be the first character of the # string, and a RandALCat character MUST be the last # character of the string. - if not RandAL[0] or not RandAL[-1]: - raise UnicodeError("Violation of BIDI requirement 3") + if not RandAL[0]: + raise UnicodeEncodeError("idna", label, 0, 1, + "Violation of BIDI requirement 3") + if not RandAL[-1]: + raise UnicodeEncodeError("idna", label, len(label)-1, len(label), + "Violation of BIDI requirement 3") return label -def ToASCII(label): +def ToASCII(label): # type: (str) -> bytes try: # Step 1: try ASCII - label = label.encode("ascii") - except UnicodeError: + label_ascii = label.encode("ascii") + except UnicodeEncodeError: pass else: # Skip to step 3: UseSTD3ASCIIRules is false, so # Skip to step 8. - if 0 < len(label) < 64: - return label - raise UnicodeError("label empty or too long") + if 0 < len(label_ascii) < 64: + return label_ascii + if len(label) == 0: + raise UnicodeEncodeError("idna", label, 0, 1, "label empty") + else: + raise UnicodeEncodeError("idna", label, 0, len(label), "label too long") # Step 2: nameprep label = nameprep(label) @@ -76,29 +85,34 @@ def ToASCII(label): # Step 3: UseSTD3ASCIIRules is false # Step 4: try ASCII try: - label = label.encode("ascii") - except UnicodeError: + label_ascii = label.encode("ascii") + except UnicodeEncodeError: pass else: # Skip to step 8. if 0 < len(label) < 64: - return label - raise UnicodeError("label empty or too long") + return label_ascii + if len(label) == 0: + raise UnicodeEncodeError("idna", label, 0, 1, "label empty") + else: + raise UnicodeEncodeError("idna", label, 0, len(label), "label too long") # Step 5: Check ACE prefix - if label.startswith(sace_prefix): - raise UnicodeError("Label starts with ACE prefix") + if label.lower().startswith(sace_prefix): + raise UnicodeEncodeError( + "idna", label, 0, len(sace_prefix), "Label starts with ACE prefix") # Step 6: Encode with PUNYCODE - label = label.encode("punycode") + label_ascii = label.encode("punycode") # Step 7: Prepend ACE prefix - label = ace_prefix + label + label_ascii = ace_prefix + label_ascii # Step 8: Check size - if 0 < len(label) < 64: - return label - raise UnicodeError("label empty or too long") + # do not check for empty as we prepend ace_prefix. + if len(label_ascii) < 64: + return label_ascii + raise UnicodeEncodeError("idna", label, 0, len(label), "label too long") def ToUnicode(label): if len(label) > 1024: @@ -110,7 +124,9 @@ def ToUnicode(label): # per https://www.rfc-editor.org/rfc/rfc3454#section-3.1 while still # preventing us from wasting time decoding a big thing that'll just # hit the actual <= 63 length limit in Step 6. - raise UnicodeError("label way too long") + if isinstance(label, str): + label = label.encode("utf-8", errors="backslashreplace") + raise UnicodeDecodeError("idna", label, 0, len(label), "label way too long") # Step 1: Check for ASCII if isinstance(label, bytes): pure_ascii = True @@ -118,25 +134,32 @@ def ToUnicode(label): try: label = label.encode("ascii") pure_ascii = True - except UnicodeError: + except UnicodeEncodeError: pure_ascii = False if not pure_ascii: + assert isinstance(label, str) # Step 2: Perform nameprep label = nameprep(label) # It doesn't say this, but apparently, it should be ASCII now try: label = label.encode("ascii") - except UnicodeError: - raise UnicodeError("Invalid character in IDN label") + except UnicodeEncodeError as exc: + raise UnicodeEncodeError("idna", label, exc.start, exc.end, + "Invalid character in IDN label") # Step 3: Check for ACE prefix - if not label.startswith(ace_prefix): + assert isinstance(label, bytes) + if not label.lower().startswith(ace_prefix): return str(label, "ascii") # Step 4: Remove ACE prefix label1 = label[len(ace_prefix):] # Step 5: Decode using PUNYCODE - result = label1.decode("punycode") + try: + result = label1.decode("punycode") + except UnicodeDecodeError as exc: + offset = len(ace_prefix) + raise UnicodeDecodeError("idna", label, offset+exc.start, offset+exc.end, exc.reason) # Step 6: Apply ToASCII label2 = ToASCII(result) @@ -144,7 +167,8 @@ def ToUnicode(label): # Step 7: Compare the result of step 6 with the one of step 3 # label2 will already be in lower case. if str(label, "ascii").lower() != str(label2, "ascii"): - raise UnicodeError("IDNA does not round-trip", label, label2) + raise UnicodeDecodeError("idna", label, 0, len(label), + f"IDNA does not round-trip, '{label!r}' != '{label2!r}'") # Step 8: return the result of step 5 return result @@ -156,7 +180,7 @@ def encode(self, input, errors='strict'): if errors != 'strict': # IDNA is quite clear that implementations must be strict - raise UnicodeError("unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") if not input: return b'', 0 @@ -168,11 +192,16 @@ def encode(self, input, errors='strict'): else: # ASCII name: fast path labels = result.split(b'.') - for label in labels[:-1]: - if not (0 < len(label) < 64): - raise UnicodeError("label empty or too long") - if len(labels[-1]) >= 64: - raise UnicodeError("label too long") + for i, label in enumerate(labels[:-1]): + if len(label) == 0: + offset = sum(len(l) for l in labels[:i]) + i + raise UnicodeEncodeError("idna", input, offset, offset+1, + "label empty") + for i, label in enumerate(labels): + if len(label) >= 64: + offset = sum(len(l) for l in labels[:i]) + i + raise UnicodeEncodeError("idna", input, offset, offset+len(label), + "label too long") return result, len(input) result = bytearray() @@ -182,17 +211,27 @@ def encode(self, input, errors='strict'): del labels[-1] else: trailing_dot = b'' - for label in labels: + for i, label in enumerate(labels): if result: # Join with U+002E result.extend(b'.') - result.extend(ToASCII(label)) + try: + result.extend(ToASCII(label)) + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + offset = sum(len(l) for l in labels[:i]) + i + raise UnicodeEncodeError( + "idna", + input, + offset + exc.start, + offset + exc.end, + exc.reason, + ) return bytes(result+trailing_dot), len(input) def decode(self, input, errors='strict'): if errors != 'strict': - raise UnicodeError("Unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") if not input: return "", 0 @@ -202,7 +241,7 @@ def decode(self, input, errors='strict'): # XXX obviously wrong, see #3232 input = bytes(input) - if ace_prefix not in input: + if ace_prefix not in input.lower(): # Fast path try: return input.decode('ascii'), len(input) @@ -218,8 +257,15 @@ def decode(self, input, errors='strict'): trailing_dot = '' result = [] - for label in labels: - result.append(ToUnicode(label)) + for i, label in enumerate(labels): + try: + u_label = ToUnicode(label) + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + offset = sum(len(x) for x in labels[:i]) + len(labels[:i]) + raise UnicodeDecodeError( + "idna", input, offset+exc.start, offset+exc.end, exc.reason) + else: + result.append(u_label) return ".".join(result)+trailing_dot, len(input) @@ -227,7 +273,7 @@ class IncrementalEncoder(codecs.BufferedIncrementalEncoder): def _buffer_encode(self, input, errors, final): if errors != 'strict': # IDNA is quite clear that implementations must be strict - raise UnicodeError("unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") if not input: return (b'', 0) @@ -251,7 +297,16 @@ def _buffer_encode(self, input, errors, final): # Join with U+002E result.extend(b'.') size += 1 - result.extend(ToASCII(label)) + try: + result.extend(ToASCII(label)) + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + raise UnicodeEncodeError( + "idna", + input, + size + exc.start, + size + exc.end, + exc.reason, + ) size += len(label) result += trailing_dot @@ -261,7 +316,7 @@ def _buffer_encode(self, input, errors, final): class IncrementalDecoder(codecs.BufferedIncrementalDecoder): def _buffer_decode(self, input, errors, final): if errors != 'strict': - raise UnicodeError("Unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") if not input: return ("", 0) @@ -271,7 +326,11 @@ def _buffer_decode(self, input, errors, final): labels = dots.split(input) else: # Must be ASCII string - input = str(input, "ascii") + try: + input = str(input, "ascii") + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + raise UnicodeDecodeError("idna", input, + exc.start, exc.end, exc.reason) labels = input.split(".") trailing_dot = '' @@ -288,7 +347,18 @@ def _buffer_decode(self, input, errors, final): result = [] size = 0 for label in labels: - result.append(ToUnicode(label)) + try: + u_label = ToUnicode(label) + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + raise UnicodeDecodeError( + "idna", + input.encode("ascii", errors="backslashreplace"), + size + exc.start, + size + exc.end, + exc.reason, + ) + else: + result.append(u_label) if size: size += 1 size += len(label) diff --git a/Lib/encodings/palmos.py b/Lib/encodings/palmos.py index c506d65452..df164ca5b9 100644 --- a/Lib/encodings/palmos.py +++ b/Lib/encodings/palmos.py @@ -201,7 +201,7 @@ def getregentry(): '\u02dc' # 0x98 -> SMALL TILDE '\u2122' # 0x99 -> TRADE MARK SIGN '\u0161' # 0x9A -> LATIN SMALL LETTER S WITH CARON - '\x9b' # 0x9B -> + '\u203a' # 0x9B -> SINGLE RIGHT-POINTING ANGLE QUOTATION MARK '\u0153' # 0x9C -> LATIN SMALL LIGATURE OE '\x9d' # 0x9D -> '\x9e' # 0x9E -> diff --git a/Lib/encodings/punycode.py b/Lib/encodings/punycode.py index 1c57264470..4622fc8c92 100644 --- a/Lib/encodings/punycode.py +++ b/Lib/encodings/punycode.py @@ -1,4 +1,4 @@ -""" Codec for the Punicode encoding, as specified in RFC 3492 +""" Codec for the Punycode encoding, as specified in RFC 3492 Written by Martin v. Löwis. """ @@ -131,10 +131,11 @@ def decode_generalized_number(extended, extpos, bias, errors): j = 0 while 1: try: - char = ord(extended[extpos]) + char = extended[extpos] except IndexError: if errors == "strict": - raise UnicodeError("incomplete punicode string") + raise UnicodeDecodeError("punycode", extended, extpos, extpos+1, + "incomplete punycode string") return extpos + 1, None extpos += 1 if 0x41 <= char <= 0x5A: # A-Z @@ -142,8 +143,8 @@ def decode_generalized_number(extended, extpos, bias, errors): elif 0x30 <= char <= 0x39: digit = char - 22 # 0x30-26 elif errors == "strict": - raise UnicodeError("Invalid extended code point '%s'" - % extended[extpos-1]) + raise UnicodeDecodeError("punycode", extended, extpos-1, extpos, + f"Invalid extended code point '{extended[extpos-1]}'") else: return extpos, None t = T(j, bias) @@ -155,11 +156,14 @@ def decode_generalized_number(extended, extpos, bias, errors): def insertion_sort(base, extended, errors): - """3.2 Insertion unsort coding""" + """3.2 Insertion sort coding""" + # This function raises UnicodeDecodeError with position in the extended. + # Caller should add the offset. char = 0x80 pos = -1 bias = 72 extpos = 0 + while extpos < len(extended): newpos, delta = decode_generalized_number(extended, extpos, bias, errors) @@ -171,7 +175,9 @@ def insertion_sort(base, extended, errors): char += pos // (len(base) + 1) if char > 0x10FFFF: if errors == "strict": - raise UnicodeError("Invalid character U+%x" % char) + raise UnicodeDecodeError( + "punycode", extended, pos-1, pos, + f"Invalid character U+{char:x}") char = ord('?') pos = pos % (len(base) + 1) base = base[:pos] + chr(char) + base[pos:] @@ -187,11 +193,21 @@ def punycode_decode(text, errors): pos = text.rfind(b"-") if pos == -1: base = "" - extended = str(text, "ascii").upper() + extended = text.upper() else: - base = str(text[:pos], "ascii", errors) - extended = str(text[pos+1:], "ascii").upper() - return insertion_sort(base, extended, errors) + try: + base = str(text[:pos], "ascii", errors) + except UnicodeDecodeError as exc: + raise UnicodeDecodeError("ascii", text, exc.start, exc.end, + exc.reason) from None + extended = text[pos+1:].upper() + try: + return insertion_sort(base, extended, errors) + except UnicodeDecodeError as exc: + offset = pos + 1 + raise UnicodeDecodeError("punycode", text, + offset+exc.start, offset+exc.end, + exc.reason) from None ### Codec APIs @@ -203,7 +219,7 @@ def encode(self, input, errors='strict'): def decode(self, input, errors='strict'): if errors not in ('strict', 'replace', 'ignore'): - raise UnicodeError("Unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") res = punycode_decode(input, errors) return res, len(input) @@ -214,7 +230,7 @@ def encode(self, input, final=False): class IncrementalDecoder(codecs.IncrementalDecoder): def decode(self, input, final=False): if self.errors not in ('strict', 'replace', 'ignore'): - raise UnicodeError("Unsupported error handling "+self.errors) + raise UnicodeError(f"Unsupported error handling: {self.errors}") return punycode_decode(input, self.errors) class StreamWriter(Codec,codecs.StreamWriter): diff --git a/Lib/encodings/undefined.py b/Lib/encodings/undefined.py index 4690288355..082771e1c8 100644 --- a/Lib/encodings/undefined.py +++ b/Lib/encodings/undefined.py @@ -1,6 +1,6 @@ """ Python 'undefined' Codec - This codec will always raise a ValueError exception when being + This codec will always raise a UnicodeError exception when being used. It is intended for use by the site.py file to switch off automatic string to Unicode coercion. diff --git a/Lib/encodings/utf_16.py b/Lib/encodings/utf_16.py index c61248242b..d3b9980026 100644 --- a/Lib/encodings/utf_16.py +++ b/Lib/encodings/utf_16.py @@ -64,7 +64,7 @@ def _buffer_decode(self, input, errors, final): elif byteorder == 1: self.decoder = codecs.utf_16_be_decode elif consumed >= 2: - raise UnicodeError("UTF-16 stream does not start with BOM") + raise UnicodeDecodeError("utf-16", input, 0, 2, "Stream does not start with BOM") return (output, consumed) return self.decoder(input, self.errors, final) @@ -138,7 +138,7 @@ def decode(self, input, errors='strict'): elif byteorder == 1: self.decode = codecs.utf_16_be_decode elif consumed>=2: - raise UnicodeError("UTF-16 stream does not start with BOM") + raise UnicodeDecodeError("utf-16", input, 0, 2, "Stream does not start with BOM") return (object, consumed) ### encodings module API diff --git a/Lib/encodings/utf_32.py b/Lib/encodings/utf_32.py index cdf84d1412..1924bedbb7 100644 --- a/Lib/encodings/utf_32.py +++ b/Lib/encodings/utf_32.py @@ -59,7 +59,7 @@ def _buffer_decode(self, input, errors, final): elif byteorder == 1: self.decoder = codecs.utf_32_be_decode elif consumed >= 4: - raise UnicodeError("UTF-32 stream does not start with BOM") + raise UnicodeDecodeError("utf-32", input, 0, 4, "Stream does not start with BOM") return (output, consumed) return self.decoder(input, self.errors, final) @@ -132,8 +132,8 @@ def decode(self, input, errors='strict'): self.decode = codecs.utf_32_le_decode elif byteorder == 1: self.decode = codecs.utf_32_be_decode - elif consumed>=4: - raise UnicodeError("UTF-32 stream does not start with BOM") + elif consumed >= 4: + raise UnicodeDecodeError("utf-32", input, 0, 4, "Stream does not start with BOM") return (object, consumed) ### encodings module API diff --git a/Lib/fnmatch.py b/Lib/fnmatch.py index fee59bf73f..73acb1fe8d 100644 --- a/Lib/fnmatch.py +++ b/Lib/fnmatch.py @@ -16,12 +16,6 @@ __all__ = ["filter", "fnmatch", "fnmatchcase", "translate"] -# Build a thread-safe incrementing counter to help create unique regexp group -# names across calls. -from itertools import count -_nextgroupnum = count().__next__ -del count - def fnmatch(name, pat): """Test whether FILENAME matches PATTERN. @@ -41,7 +35,7 @@ def fnmatch(name, pat): pat = os.path.normcase(pat) return fnmatchcase(name, pat) -@functools.lru_cache(maxsize=256, typed=True) +@functools.lru_cache(maxsize=32768, typed=True) def _compile_pattern(pat): if isinstance(pat, bytes): pat_str = str(pat, 'ISO-8859-1') @@ -84,6 +78,11 @@ def translate(pat): """ STAR = object() + parts = _translate(pat, STAR, '.') + return _join_translated_parts(parts, STAR) + + +def _translate(pat, STAR, QUESTION_MARK): res = [] add = res.append i, n = 0, len(pat) @@ -95,7 +94,7 @@ def translate(pat): if (not res) or res[-1] is not STAR: add(STAR) elif c == '?': - add('.') + add(QUESTION_MARK) elif c == '[': j = i if j < n and pat[j] == '!': @@ -152,9 +151,11 @@ def translate(pat): else: add(re.escape(c)) assert i == n + return res + +def _join_translated_parts(inp, STAR): # Deal with STARs. - inp = res res = [] add = res.append i, n = 0, len(inp) @@ -165,17 +166,10 @@ def translate(pat): # Now deal with STAR fixed STAR fixed ... # For an interior `STAR fixed` pairing, we want to do a minimal # .*? match followed by `fixed`, with no possibility of backtracking. - # We can't spell that directly, but can trick it into working by matching - # .*?fixed - # in a lookahead assertion, save the matched part in a group, then - # consume that group via a backreference. If the overall match fails, - # the lookahead assertion won't try alternatives. So the translation is: - # (?=(?P.*?fixed))(?P=name) - # Group names are created as needed: g0, g1, g2, ... - # The numbers are obtained from _nextgroupnum() to ensure they're unique - # across calls and across threads. This is because people rely on the - # undocumented ability to join multiple translate() results together via - # "|" to build large regexps matching "one of many" shell patterns. + # Atomic groups ("(?>...)") allow us to spell that directly. + # Note: people rely on the undocumented ability to join multiple + # translate() results together via "|" to build large regexps matching + # "one of many" shell patterns. while i < n: assert inp[i] is STAR i += 1 @@ -192,8 +186,7 @@ def translate(pat): add(".*") add(fixed) else: - groupnum = _nextgroupnum() - add(f"(?=(?P.*?{fixed}))(?P=g{groupnum})") + add(f"(?>.*?{fixed})") assert i == n res = "".join(res) return fr'(?s:{res})\Z' diff --git a/Lib/glob.py b/Lib/glob.py index 50beef37f4..c506e0e215 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -4,11 +4,14 @@ import os import re import fnmatch +import functools import itertools +import operator import stat import sys -__all__ = ["glob", "iglob", "escape"] + +__all__ = ["glob", "iglob", "escape", "translate"] def glob(pathname, *, root_dir=None, dir_fd=None, recursive=False, include_hidden=False): @@ -104,8 +107,8 @@ def _iglob(pathname, root_dir, dir_fd, recursive, dironly, def _glob1(dirname, pattern, dir_fd, dironly, include_hidden=False): names = _listdir(dirname, dir_fd, dironly) - if include_hidden or not _ishidden(pattern): - names = (x for x in names if include_hidden or not _ishidden(x)) + if not (include_hidden or _ishidden(pattern)): + names = (x for x in names if not _ishidden(x)) return fnmatch.filter(names, pattern) def _glob0(dirname, basename, dir_fd, dironly, include_hidden=False): @@ -119,12 +122,19 @@ def _glob0(dirname, basename, dir_fd, dironly, include_hidden=False): return [basename] return [] -# Following functions are not public but can be used by third-party code. +_deprecated_function_message = ( + "{name} is deprecated and will be removed in Python {remove}. Use " + "glob.glob and pass a directory to its root_dir argument instead." +) def glob0(dirname, pattern): + import warnings + warnings._deprecated("glob.glob0", _deprecated_function_message, remove=(3, 15)) return _glob0(dirname, pattern, None, False) def glob1(dirname, pattern): + import warnings + warnings._deprecated("glob.glob1", _deprecated_function_message, remove=(3, 15)) return _glob1(dirname, pattern, None, False) # This helper function recursively yields relative pathnames inside a literal @@ -249,4 +259,287 @@ def escape(pathname): return drive + pathname +_special_parts = ('', '.', '..') _dir_open_flags = os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0) +_no_recurse_symlinks = object() + + +def translate(pat, *, recursive=False, include_hidden=False, seps=None): + """Translate a pathname with shell wildcards to a regular expression. + + If `recursive` is true, the pattern segment '**' will match any number of + path segments. + + If `include_hidden` is true, wildcards can match path segments beginning + with a dot ('.'). + + If a sequence of separator characters is given to `seps`, they will be + used to split the pattern into segments and match path separators. If not + given, os.path.sep and os.path.altsep (where available) are used. + """ + if not seps: + if os.path.altsep: + seps = (os.path.sep, os.path.altsep) + else: + seps = os.path.sep + escaped_seps = ''.join(map(re.escape, seps)) + any_sep = f'[{escaped_seps}]' if len(seps) > 1 else escaped_seps + not_sep = f'[^{escaped_seps}]' + if include_hidden: + one_last_segment = f'{not_sep}+' + one_segment = f'{one_last_segment}{any_sep}' + any_segments = f'(?:.+{any_sep})?' + any_last_segments = '.*' + else: + one_last_segment = f'[^{escaped_seps}.]{not_sep}*' + one_segment = f'{one_last_segment}{any_sep}' + any_segments = f'(?:{one_segment})*' + any_last_segments = f'{any_segments}(?:{one_last_segment})?' + + results = [] + parts = re.split(any_sep, pat) + last_part_idx = len(parts) - 1 + for idx, part in enumerate(parts): + if part == '*': + results.append(one_segment if idx < last_part_idx else one_last_segment) + elif recursive and part == '**': + if idx < last_part_idx: + if parts[idx + 1] != '**': + results.append(any_segments) + else: + results.append(any_last_segments) + else: + if part: + if not include_hidden and part[0] in '*?': + results.append(r'(?!\.)') + results.extend(fnmatch._translate(part, f'{not_sep}*', not_sep)) + if idx < last_part_idx: + results.append(any_sep) + res = ''.join(results) + return fr'(?s:{res})\Z' + + +@functools.lru_cache(maxsize=512) +def _compile_pattern(pat, sep, case_sensitive, recursive=True): + """Compile given glob pattern to a re.Pattern object (observing case + sensitivity).""" + flags = re.NOFLAG if case_sensitive else re.IGNORECASE + regex = translate(pat, recursive=recursive, include_hidden=True, seps=sep) + return re.compile(regex, flags=flags).match + + +class _Globber: + """Class providing shell-style pattern matching and globbing. + """ + + def __init__(self, sep, case_sensitive, case_pedantic=False, recursive=False): + self.sep = sep + self.case_sensitive = case_sensitive + self.case_pedantic = case_pedantic + self.recursive = recursive + + # Low-level methods + + lstat = operator.methodcaller('lstat') + add_slash = operator.methodcaller('joinpath', '') + + @staticmethod + def scandir(path): + """Emulates os.scandir(), which returns an object that can be used as + a context manager. This method is called by walk() and glob(). + """ + return contextlib.nullcontext(path.iterdir()) + + @staticmethod + def concat_path(path, text): + """Appends text to the given path. + """ + return path.with_segments(path._raw_path + text) + + @staticmethod + def parse_entry(entry): + """Returns the path of an entry yielded from scandir(). + """ + return entry + + # High-level methods + + def compile(self, pat): + return _compile_pattern(pat, self.sep, self.case_sensitive, self.recursive) + + def selector(self, parts): + """Returns a function that selects from a given path, walking and + filtering according to the glob-style pattern parts in *parts*. + """ + if not parts: + return self.select_exists + part = parts.pop() + if self.recursive and part == '**': + selector = self.recursive_selector + elif part in _special_parts: + selector = self.special_selector + elif not self.case_pedantic and magic_check.search(part) is None: + selector = self.literal_selector + else: + selector = self.wildcard_selector + return selector(part, parts) + + def special_selector(self, part, parts): + """Returns a function that selects special children of the given path. + """ + select_next = self.selector(parts) + + def select_special(path, exists=False): + path = self.concat_path(self.add_slash(path), part) + return select_next(path, exists) + return select_special + + def literal_selector(self, part, parts): + """Returns a function that selects a literal descendant of a path. + """ + + # Optimization: consume and join any subsequent literal parts here, + # rather than leaving them for the next selector. This reduces the + # number of string concatenation operations and calls to add_slash(). + while parts and magic_check.search(parts[-1]) is None: + part += self.sep + parts.pop() + + select_next = self.selector(parts) + + def select_literal(path, exists=False): + path = self.concat_path(self.add_slash(path), part) + return select_next(path, exists=False) + return select_literal + + def wildcard_selector(self, part, parts): + """Returns a function that selects direct children of a given path, + filtering by pattern. + """ + + match = None if part == '*' else self.compile(part) + dir_only = bool(parts) + if dir_only: + select_next = self.selector(parts) + + def select_wildcard(path, exists=False): + try: + # We must close the scandir() object before proceeding to + # avoid exhausting file descriptors when globbing deep trees. + with self.scandir(path) as scandir_it: + entries = list(scandir_it) + except OSError: + pass + else: + for entry in entries: + if match is None or match(entry.name): + if dir_only: + try: + if not entry.is_dir(): + continue + except OSError: + continue + entry_path = self.parse_entry(entry) + if dir_only: + yield from select_next(entry_path, exists=True) + else: + yield entry_path + return select_wildcard + + def recursive_selector(self, part, parts): + """Returns a function that selects a given path and all its children, + recursively, filtering by pattern. + """ + # Optimization: consume following '**' parts, which have no effect. + while parts and parts[-1] == '**': + parts.pop() + + # Optimization: consume and join any following non-special parts here, + # rather than leaving them for the next selector. They're used to + # build a regular expression, which we use to filter the results of + # the recursive walk. As a result, non-special pattern segments + # following a '**' wildcard don't require additional filesystem access + # to expand. + follow_symlinks = self.recursive is not _no_recurse_symlinks + if follow_symlinks: + while parts and parts[-1] not in _special_parts: + part += self.sep + parts.pop() + + match = None if part == '**' else self.compile(part) + dir_only = bool(parts) + select_next = self.selector(parts) + + def select_recursive(path, exists=False): + path = self.add_slash(path) + match_pos = len(str(path)) + if match is None or match(str(path), match_pos): + yield from select_next(path, exists) + stack = [path] + while stack: + yield from select_recursive_step(stack, match_pos) + + def select_recursive_step(stack, match_pos): + path = stack.pop() + try: + # We must close the scandir() object before proceeding to + # avoid exhausting file descriptors when globbing deep trees. + with self.scandir(path) as scandir_it: + entries = list(scandir_it) + except OSError: + pass + else: + for entry in entries: + is_dir = False + try: + if entry.is_dir(follow_symlinks=follow_symlinks): + is_dir = True + except OSError: + pass + + if is_dir or not dir_only: + entry_path = self.parse_entry(entry) + if match is None or match(str(entry_path), match_pos): + if dir_only: + yield from select_next(entry_path, exists=True) + else: + # Optimization: directly yield the path if this is + # last pattern part. + yield entry_path + if is_dir: + stack.append(entry_path) + + return select_recursive + + def select_exists(self, path, exists=False): + """Yields the given path, if it exists. + """ + if exists: + # Optimization: this path is already known to exist, e.g. because + # it was returned from os.scandir(), so we skip calling lstat(). + yield path + else: + try: + self.lstat(path) + yield path + except OSError: + pass + + +class _StringGlobber(_Globber): + lstat = staticmethod(os.lstat) + scandir = staticmethod(os.scandir) + parse_entry = operator.attrgetter('path') + concat_path = operator.add + + if os.name == 'nt': + @staticmethod + def add_slash(pathname): + tail = os.path.splitroot(pathname)[2] + if not tail or tail[-1] in '\\/': + return pathname + return f'{pathname}\\' + else: + @staticmethod + def add_slash(pathname): + if not pathname or pathname[-1] == '/': + return pathname + return f'{pathname}/' diff --git a/Lib/html/parser.py b/Lib/html/parser.py index 1e30956fe2..5d03c98df5 100644 --- a/Lib/html/parser.py +++ b/Lib/html/parser.py @@ -27,18 +27,48 @@ attr_charref = re.compile(r'&(#[0-9]+|#[xX][0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*)[;=]?') starttagopen = re.compile('<[a-zA-Z]') +endtagopen = re.compile('') -commentclose = re.compile(r'--\s*>') +commentclose = re.compile(r'--!?>') +commentabruptclose = re.compile(r'-?>') # Note: -# 1) if you change tagfind/attrfind remember to update locatestarttagend too; -# 2) if you change tagfind/attrfind and/or locatestarttagend the parser will +# 1) if you change tagfind/attrfind remember to update locatetagend too; +# 2) if you change tagfind/attrfind and/or locatetagend the parser will # explode, so don't do it. -# see http://www.w3.org/TR/html5/tokenization.html#tag-open-state -# and http://www.w3.org/TR/html5/tokenization.html#tag-name-state -tagfind_tolerant = re.compile(r'([a-zA-Z][^\t\n\r\f />\x00]*)(?:\s|/(?!>))*') -attrfind_tolerant = re.compile( - r'((?<=[\'"\s/])[^\s/>][^\s/=>]*)(\s*=+\s*' - r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?(?:\s|/(?!>))*') +# see the HTML5 specs section "13.2.5.6 Tag open state", +# "13.2.5.8 Tag name state" and "13.2.5.33 Attribute name state". +# https://html.spec.whatwg.org/multipage/parsing.html#tag-open-state +# https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state +# https://html.spec.whatwg.org/multipage/parsing.html#attribute-name-state +tagfind_tolerant = re.compile(r'([a-zA-Z][^\t\n\r\f />]*)(?:[\t\n\r\f ]|/(?!>))*') +attrfind_tolerant = re.compile(r""" + ( + (?<=['"\t\n\r\f /])[^\t\n\r\f />][^\t\n\r\f /=>]* # attribute name + ) + ([\t\n\r\f ]*=[\t\n\r\f ]* # value indicator + ('[^']*' # LITA-enclosed value + |"[^"]*" # LIT-enclosed value + |(?!['"])[^>\t\n\r\f ]* # bare value + ) + )? + (?:[\t\n\r\f ]|/(?!>))* # possibly followed by a space +""", re.VERBOSE) +locatetagend = re.compile(r""" + [a-zA-Z][^\t\n\r\f />]* # tag name + [\t\n\r\f /]* # optional whitespace before attribute name + (?:(?<=['"\t\n\r\f /])[^\t\n\r\f />][^\t\n\r\f /=>]* # attribute name + (?:[\t\n\r\f ]*=[\t\n\r\f ]* # value indicator + (?:'[^']*' # LITA-enclosed value + |"[^"]*" # LIT-enclosed value + |(?!['"])[^>\t\n\r\f ]* # bare value + ) + )? + [\t\n\r\f /]* # possibly followed by a space + )* + >? +""", re.VERBOSE) +# The following variables are not used, but are temporarily left for +# backward compatibility. locatestarttagend_tolerant = re.compile(r""" <[a-zA-Z][^\t\n\r\f />\x00]* # tag name (?:[\s/]* # optional whitespace before attribute name @@ -55,8 +85,6 @@ \s* # trailing whitespace """, re.VERBOSE) endendtag = re.compile('>') -# the HTML 5 spec, section 8.1.2.2, doesn't allow spaces between -# ') # Character reference processing logic specific to attribute values @@ -100,6 +128,7 @@ class HTMLParser(_markupbase.ParserBase): """ CDATA_CONTENT_ELEMENTS = ("script", "style") + RCDATA_CONTENT_ELEMENTS = ("textarea", "title") def __init__(self, *, convert_charrefs=True): """Initialize and reset this instance. @@ -117,6 +146,7 @@ def reset(self): self.lasttag = '???' self.interesting = interesting_normal self.cdata_elem = None + self._escapable = True super().reset() def feed(self, data): @@ -138,13 +168,20 @@ def get_starttag_text(self): """Return full source of start tag: '<...>'.""" return self.__starttag_text - def set_cdata_mode(self, elem): + def set_cdata_mode(self, elem, *, escapable=False): self.cdata_elem = elem.lower() - self.interesting = re.compile(r'' % self.cdata_elem, re.I) + self._escapable = escapable + if escapable and not self.convert_charrefs: + self.interesting = re.compile(r'&|])' % self.cdata_elem, + re.IGNORECASE|re.ASCII) + else: + self.interesting = re.compile(r'])' % self.cdata_elem, + re.IGNORECASE|re.ASCII) def clear_cdata_mode(self): self.interesting = interesting_normal self.cdata_elem = None + self._escapable = True # Internal -- handle data as far as reasonable. May leave state # and data to be processed by a subsequent call. If 'end' is @@ -165,7 +202,7 @@ def goahead(self, end): # & near the end and see if it's followed by a space or ;. amppos = rawdata.rfind('&', max(i, n-34)) if (amppos >= 0 and - not re.compile(r'[\s;]').search(rawdata, amppos)): + not re.compile(r'[\t\n\r\f ;]').search(rawdata, amppos)): break # wait till we get all the text j = n else: @@ -177,7 +214,7 @@ def goahead(self, end): break j = n if i < j: - if self.convert_charrefs and not self.cdata_elem: + if self.convert_charrefs and self._escapable: self.handle_data(unescape(rawdata[i:j])) else: self.handle_data(rawdata[i:j]) @@ -195,7 +232,7 @@ def goahead(self, end): k = self.parse_pi(i) elif startswith("', i + 1) - if k < 0: - k = rawdata.find('<', i + 1) - if k < 0: - k = i + 1 - else: - k += 1 - if self.convert_charrefs and not self.cdata_elem: - self.handle_data(unescape(rawdata[i:k])) + if starttagopen.match(rawdata, i): # < + letter + pass + elif startswith("": - return j + 1 - if next == "/": - if rawdata.startswith("/>", j): - return j + 2 - if rawdata.startswith("/", j): - # buffer boundary - return -1 - # else bogus input - if j > i: - return j - else: - return i + 1 - if next == "": - # end of input - return -1 - if next in ("abcdefghijklmnopqrstuvwxyz=/" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ"): - # end of input in or before attribute value, or we have the - # '/' from a '/>' ending - return -1 - if j > i: - return j - else: - return i + 1 - raise AssertionError("we should not get here!") + match = locatetagend.match(rawdata, i+1) + assert match + j = match.end() + if rawdata[j-1] != ">": + return -1 + return j # Internal -- parse endtag, return end or -1 if incomplete def parse_endtag(self, i): + # See the HTML5 specs section "13.2.5.7 End tag open state" + # https://html.spec.whatwg.org/multipage/parsing.html#end-tag-open-state rawdata = self.rawdata assert rawdata[i:i+2] == " - if not match: + if rawdata.find('>', i+2) < 0: # fast check return -1 - gtpos = match.end() - match = endtagfind.match(rawdata, i) # - if not match: - if self.cdata_elem is not None: - self.handle_data(rawdata[i:gtpos]) - return gtpos - # find the name: w3.org/TR/html5/tokenization.html#tag-name-state - namematch = tagfind_tolerant.match(rawdata, i+2) - if not namematch: - # w3.org/TR/html5/tokenization.html#end-tag-open-state - if rawdata[i:i+3] == '': - return i+3 - else: - return self.parse_bogus_comment(i) - tagname = namematch.group(1).lower() - # consume and ignore other stuff between the name and the > - # Note: this is not 100% correct, since we might have things like - # , but looking for > after the name should cover - # most of the cases and is much simpler - gtpos = rawdata.find('>', namematch.end()) - self.handle_endtag(tagname) - return gtpos+1 + if not endtagopen.match(rawdata, i): # ': # is ignored + # "missing-end-tag-name" parser error + return i+3 + else: + return self.parse_bogus_comment(i) - elem = match.group(1).lower() # script or style - if self.cdata_elem is not None: - if elem != self.cdata_elem: - self.handle_data(rawdata[i:gtpos]) - return gtpos + match = locatetagend.match(rawdata, i+2) + assert match + j = match.end() + if rawdata[j-1] != ">": + return -1 - self.handle_endtag(elem) + # find the name: "13.2.5.8 Tag name state" + # https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state + match = tagfind_tolerant.match(rawdata, i+2) + assert match + tag = match.group(1).lower() + self.handle_endtag(tag) self.clear_cdata_mode() - return gtpos + return j # Overridable -- finish processing of start+end tag: def handle_startendtag(self, tag, attrs): diff --git a/Lib/io.py b/Lib/io.py index c2812876d3..f0e2fa15d5 100644 --- a/Lib/io.py +++ b/Lib/io.py @@ -46,23 +46,17 @@ "BufferedReader", "BufferedWriter", "BufferedRWPair", "BufferedRandom", "TextIOBase", "TextIOWrapper", "UnsupportedOperation", "SEEK_SET", "SEEK_CUR", "SEEK_END", - "DEFAULT_BUFFER_SIZE", "text_encoding", - "IncrementalNewlineDecoder" - ] + "DEFAULT_BUFFER_SIZE", "text_encoding", "IncrementalNewlineDecoder"] import _io import abc from _io import (DEFAULT_BUFFER_SIZE, BlockingIOError, UnsupportedOperation, - open, open_code, BytesIO, StringIO, BufferedReader, + open, open_code, FileIO, BytesIO, StringIO, BufferedReader, BufferedWriter, BufferedRWPair, BufferedRandom, IncrementalNewlineDecoder, text_encoding, TextIOWrapper) -try: - from _io import FileIO -except ImportError: - pass # Pretend this exception was created here. UnsupportedOperation.__module__ = "io" @@ -87,10 +81,7 @@ class BufferedIOBase(_io._BufferedIOBase, IOBase): class TextIOBase(_io._TextIOBase, IOBase): __doc__ = _io._TextIOBase.__doc__ -try: - RawIOBase.register(FileIO) -except NameError: - pass +RawIOBase.register(FileIO) for klass in (BytesIO, BufferedReader, BufferedWriter, BufferedRandom, BufferedRWPair): diff --git a/Lib/numbers.py b/Lib/numbers.py index a2913e32cf..37fddb8917 100644 --- a/Lib/numbers.py +++ b/Lib/numbers.py @@ -290,18 +290,27 @@ def conjugate(self): class Rational(Real): - """.numerator and .denominator should be in lowest terms.""" + """To Real, Rational adds numerator and denominator properties. + + The numerator and denominator values should be in lowest terms, + with a positive denominator. + """ __slots__ = () @property @abstractmethod def numerator(self): + """The numerator of a rational number in lowest terms.""" raise NotImplementedError @property @abstractmethod def denominator(self): + """The denominator of a rational number in lowest terms. + + This denominator should be positive. + """ raise NotImplementedError # Concrete implementation of Real's conversion to float. diff --git a/Lib/opcode.py b/Lib/opcode.py index ab6b765b4b..5735686fa7 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -4,404 +4,47 @@ operate on bytecodes (e.g. peephole optimizers). """ -__all__ = ["cmp_op", "hasarg", "hasconst", "hasname", "hasjrel", "hasjabs", - "haslocal", "hascompare", "hasfree", "hasexc", "opname", "opmap", - "HAVE_ARGUMENT", "EXTENDED_ARG"] -# It's a chicken-and-egg I'm afraid: -# We're imported before _opcode's made. -# With exception unheeded -# (stack_effect is not needed) -# Both our chickens and eggs are allayed. -# --Larry Hastings, 2013/11/23 +__all__ = ["cmp_op", "stack_effect", "hascompare", "opname", "opmap", + "HAVE_ARGUMENT", "EXTENDED_ARG", "hasarg", "hasconst", "hasname", + "hasjump", "hasjrel", "hasjabs", "hasfree", "haslocal", "hasexc"] -try: - from _opcode import stack_effect - __all__.append('stack_effect') -except ImportError: - pass +import _opcode +from _opcode import stack_effect -cmp_op = ('<', '<=', '==', '!=', '>', '>=') - -hasarg = [] -hasconst = [] -hasname = [] -hasjrel = [] -hasjabs = [] -haslocal = [] -hascompare = [] -hasfree = [] -hasexc = [] - -def is_pseudo(op): - return op >= MIN_PSEUDO_OPCODE and op <= MAX_PSEUDO_OPCODE - -oplists = [hasarg, hasconst, hasname, hasjrel, hasjabs, - haslocal, hascompare, hasfree, hasexc] - -opmap = {} - -## pseudo opcodes (used in the compiler) mapped to the values -## they can become in the actual code. -_pseudo_ops = {} - -def def_op(name, op): - opmap[name] = op - -def name_op(name, op): - def_op(name, op) - hasname.append(op) - -def jrel_op(name, op): - def_op(name, op) - hasjrel.append(op) - -def jabs_op(name, op): - def_op(name, op) - hasjabs.append(op) - -def pseudo_op(name, op, real_ops): - def_op(name, op) - _pseudo_ops[name] = real_ops - # add the pseudo opcode to the lists its targets are in - for oplist in oplists: - res = [opmap[rop] in oplist for rop in real_ops] - if any(res): - assert all(res) - oplist.append(op) - - -# Instruction opcodes for compiled code -# Blank lines correspond to available opcodes - -def_op('CACHE', 0) -def_op('POP_TOP', 1) -def_op('PUSH_NULL', 2) - -def_op('NOP', 9) -def_op('UNARY_POSITIVE', 10) -def_op('UNARY_NEGATIVE', 11) -def_op('UNARY_NOT', 12) - -def_op('UNARY_INVERT', 15) - -def_op('BINARY_SUBSCR', 25) -def_op('BINARY_SLICE', 26) -def_op('STORE_SLICE', 27) - -def_op('GET_LEN', 30) -def_op('MATCH_MAPPING', 31) -def_op('MATCH_SEQUENCE', 32) -def_op('MATCH_KEYS', 33) - -def_op('PUSH_EXC_INFO', 35) -def_op('CHECK_EXC_MATCH', 36) -def_op('CHECK_EG_MATCH', 37) - -def_op('WITH_EXCEPT_START', 49) -def_op('GET_AITER', 50) -def_op('GET_ANEXT', 51) -def_op('BEFORE_ASYNC_WITH', 52) -def_op('BEFORE_WITH', 53) -def_op('END_ASYNC_FOR', 54) -def_op('CLEANUP_THROW', 55) - -def_op('STORE_SUBSCR', 60) -def_op('DELETE_SUBSCR', 61) - -# TODO: RUSTPYTHON -# Delete below def_op after updating coroutines.py -def_op('YIELD_FROM', 72) - -def_op('GET_ITER', 68) -def_op('GET_YIELD_FROM_ITER', 69) -def_op('PRINT_EXPR', 70) -def_op('LOAD_BUILD_CLASS', 71) - -def_op('LOAD_ASSERTION_ERROR', 74) -def_op('RETURN_GENERATOR', 75) - -def_op('LIST_TO_TUPLE', 82) -def_op('RETURN_VALUE', 83) -def_op('IMPORT_STAR', 84) -def_op('SETUP_ANNOTATIONS', 85) - -def_op('ASYNC_GEN_WRAP', 87) -def_op('PREP_RERAISE_STAR', 88) -def_op('POP_EXCEPT', 89) - -HAVE_ARGUMENT = 90 # real opcodes from here have an argument: - -name_op('STORE_NAME', 90) # Index in name list -name_op('DELETE_NAME', 91) # "" -def_op('UNPACK_SEQUENCE', 92) # Number of tuple items -jrel_op('FOR_ITER', 93) -def_op('UNPACK_EX', 94) -name_op('STORE_ATTR', 95) # Index in name list -name_op('DELETE_ATTR', 96) # "" -name_op('STORE_GLOBAL', 97) # "" -name_op('DELETE_GLOBAL', 98) # "" -def_op('SWAP', 99) -def_op('LOAD_CONST', 100) # Index in const list -hasconst.append(100) -name_op('LOAD_NAME', 101) # Index in name list -def_op('BUILD_TUPLE', 102) # Number of tuple items -def_op('BUILD_LIST', 103) # Number of list items -def_op('BUILD_SET', 104) # Number of set items -def_op('BUILD_MAP', 105) # Number of dict entries -name_op('LOAD_ATTR', 106) # Index in name list -def_op('COMPARE_OP', 107) # Comparison operator -hascompare.append(107) -name_op('IMPORT_NAME', 108) # Index in name list -name_op('IMPORT_FROM', 109) # Index in name list -jrel_op('JUMP_FORWARD', 110) # Number of words to skip -jrel_op('JUMP_IF_FALSE_OR_POP', 111) # Number of words to skip -jrel_op('JUMP_IF_TRUE_OR_POP', 112) # "" -jrel_op('POP_JUMP_IF_FALSE', 114) -jrel_op('POP_JUMP_IF_TRUE', 115) -name_op('LOAD_GLOBAL', 116) # Index in name list -def_op('IS_OP', 117) -def_op('CONTAINS_OP', 118) -def_op('RERAISE', 119) -def_op('COPY', 120) -def_op('BINARY_OP', 122) -jrel_op('SEND', 123) # Number of bytes to skip -def_op('LOAD_FAST', 124) # Local variable number, no null check -haslocal.append(124) -def_op('STORE_FAST', 125) # Local variable number -haslocal.append(125) -def_op('DELETE_FAST', 126) # Local variable number -haslocal.append(126) -def_op('LOAD_FAST_CHECK', 127) # Local variable number -haslocal.append(127) -jrel_op('POP_JUMP_IF_NOT_NONE', 128) -jrel_op('POP_JUMP_IF_NONE', 129) -def_op('RAISE_VARARGS', 130) # Number of raise arguments (1, 2, or 3) -def_op('GET_AWAITABLE', 131) -def_op('MAKE_FUNCTION', 132) # Flags -def_op('BUILD_SLICE', 133) # Number of items -jrel_op('JUMP_BACKWARD_NO_INTERRUPT', 134) # Number of words to skip (backwards) -def_op('MAKE_CELL', 135) -hasfree.append(135) -def_op('LOAD_CLOSURE', 136) -hasfree.append(136) -def_op('LOAD_DEREF', 137) -hasfree.append(137) -def_op('STORE_DEREF', 138) -hasfree.append(138) -def_op('DELETE_DEREF', 139) -hasfree.append(139) -jrel_op('JUMP_BACKWARD', 140) # Number of words to skip (backwards) - -def_op('CALL_FUNCTION_EX', 142) # Flags - -def_op('EXTENDED_ARG', 144) -EXTENDED_ARG = 144 -def_op('LIST_APPEND', 145) -def_op('SET_ADD', 146) -def_op('MAP_ADD', 147) -def_op('LOAD_CLASSDEREF', 148) -hasfree.append(148) -def_op('COPY_FREE_VARS', 149) -def_op('YIELD_VALUE', 150) -def_op('RESUME', 151) # This must be kept in sync with deepfreeze.py -def_op('MATCH_CLASS', 152) - -def_op('FORMAT_VALUE', 155) -def_op('BUILD_CONST_KEY_MAP', 156) -def_op('BUILD_STRING', 157) - -def_op('LIST_EXTEND', 162) -def_op('SET_UPDATE', 163) -def_op('DICT_MERGE', 164) -def_op('DICT_UPDATE', 165) - -def_op('CALL', 171) -def_op('KW_NAMES', 172) -hasconst.append(172) +from _opcode_metadata import (_specializations, _specialized_opmap, opmap, + HAVE_ARGUMENT, MIN_INSTRUMENTED_OPCODE) +EXTENDED_ARG = opmap['EXTENDED_ARG'] - -hasarg.extend([op for op in opmap.values() if op >= HAVE_ARGUMENT]) - -MIN_PSEUDO_OPCODE = 256 - -pseudo_op('SETUP_FINALLY', 256, ['NOP']) -hasexc.append(256) -pseudo_op('SETUP_CLEANUP', 257, ['NOP']) -hasexc.append(257) -pseudo_op('SETUP_WITH', 258, ['NOP']) -hasexc.append(258) -pseudo_op('POP_BLOCK', 259, ['NOP']) - -pseudo_op('JUMP', 260, ['JUMP_FORWARD', 'JUMP_BACKWARD']) -pseudo_op('JUMP_NO_INTERRUPT', 261, ['JUMP_FORWARD', 'JUMP_BACKWARD_NO_INTERRUPT']) - -pseudo_op('LOAD_METHOD', 262, ['LOAD_ATTR']) - -MAX_PSEUDO_OPCODE = MIN_PSEUDO_OPCODE + len(_pseudo_ops) - 1 - -del def_op, name_op, jrel_op, jabs_op, pseudo_op - -opname = ['<%r>' % (op,) for op in range(MAX_PSEUDO_OPCODE + 1)] +opname = ['<%r>' % (op,) for op in range(max(opmap.values()) + 1)] for op, i in opmap.items(): opname[i] = op +cmp_op = ('<', '<=', '==', '!=', '>', '>=') -_nb_ops = [ - ("NB_ADD", "+"), - ("NB_AND", "&"), - ("NB_FLOOR_DIVIDE", "//"), - ("NB_LSHIFT", "<<"), - ("NB_MATRIX_MULTIPLY", "@"), - ("NB_MULTIPLY", "*"), - ("NB_REMAINDER", "%"), - ("NB_OR", "|"), - ("NB_POWER", "**"), - ("NB_RSHIFT", ">>"), - ("NB_SUBTRACT", "-"), - ("NB_TRUE_DIVIDE", "/"), - ("NB_XOR", "^"), - ("NB_INPLACE_ADD", "+="), - ("NB_INPLACE_AND", "&="), - ("NB_INPLACE_FLOOR_DIVIDE", "//="), - ("NB_INPLACE_LSHIFT", "<<="), - ("NB_INPLACE_MATRIX_MULTIPLY", "@="), - ("NB_INPLACE_MULTIPLY", "*="), - ("NB_INPLACE_REMAINDER", "%="), - ("NB_INPLACE_OR", "|="), - ("NB_INPLACE_POWER", "**="), - ("NB_INPLACE_RSHIFT", ">>="), - ("NB_INPLACE_SUBTRACT", "-="), - ("NB_INPLACE_TRUE_DIVIDE", "/="), - ("NB_INPLACE_XOR", "^="), -] +# These lists are documented as part of the dis module's API +hasarg = [op for op in opmap.values() if _opcode.has_arg(op)] +hasconst = [op for op in opmap.values() if _opcode.has_const(op)] +hasname = [op for op in opmap.values() if _opcode.has_name(op)] +hasjump = [op for op in opmap.values() if _opcode.has_jump(op)] +hasjrel = hasjump # for backward compatibility +hasjabs = [] +hasfree = [op for op in opmap.values() if _opcode.has_free(op)] +haslocal = [op for op in opmap.values() if _opcode.has_local(op)] +hasexc = [op for op in opmap.values() if _opcode.has_exc(op)] -_specializations = { - "BINARY_OP": [ - "BINARY_OP_ADAPTIVE", - "BINARY_OP_ADD_FLOAT", - "BINARY_OP_ADD_INT", - "BINARY_OP_ADD_UNICODE", - "BINARY_OP_INPLACE_ADD_UNICODE", - "BINARY_OP_MULTIPLY_FLOAT", - "BINARY_OP_MULTIPLY_INT", - "BINARY_OP_SUBTRACT_FLOAT", - "BINARY_OP_SUBTRACT_INT", - ], - "BINARY_SUBSCR": [ - "BINARY_SUBSCR_ADAPTIVE", - "BINARY_SUBSCR_DICT", - "BINARY_SUBSCR_GETITEM", - "BINARY_SUBSCR_LIST_INT", - "BINARY_SUBSCR_TUPLE_INT", - ], - "CALL": [ - "CALL_ADAPTIVE", - "CALL_PY_EXACT_ARGS", - "CALL_PY_WITH_DEFAULTS", - "CALL_BOUND_METHOD_EXACT_ARGS", - "CALL_BUILTIN_CLASS", - "CALL_BUILTIN_FAST_WITH_KEYWORDS", - "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", - "CALL_NO_KW_BUILTIN_FAST", - "CALL_NO_KW_BUILTIN_O", - "CALL_NO_KW_ISINSTANCE", - "CALL_NO_KW_LEN", - "CALL_NO_KW_LIST_APPEND", - "CALL_NO_KW_METHOD_DESCRIPTOR_FAST", - "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", - "CALL_NO_KW_METHOD_DESCRIPTOR_O", - "CALL_NO_KW_STR_1", - "CALL_NO_KW_TUPLE_1", - "CALL_NO_KW_TYPE_1", - ], - "COMPARE_OP": [ - "COMPARE_OP_ADAPTIVE", - "COMPARE_OP_FLOAT_JUMP", - "COMPARE_OP_INT_JUMP", - "COMPARE_OP_STR_JUMP", - ], - "EXTENDED_ARG": [ - "EXTENDED_ARG_QUICK", - ], - "FOR_ITER": [ - "FOR_ITER_ADAPTIVE", - "FOR_ITER_LIST", - "FOR_ITER_RANGE", - ], - "JUMP_BACKWARD": [ - "JUMP_BACKWARD_QUICK", - ], - "LOAD_ATTR": [ - "LOAD_ATTR_ADAPTIVE", - # These potentially push [NULL, bound method] onto the stack. - "LOAD_ATTR_CLASS", - "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", - "LOAD_ATTR_INSTANCE_VALUE", - "LOAD_ATTR_MODULE", - "LOAD_ATTR_PROPERTY", - "LOAD_ATTR_SLOT", - "LOAD_ATTR_WITH_HINT", - # These will always push [unbound method, self] onto the stack. - "LOAD_ATTR_METHOD_LAZY_DICT", - "LOAD_ATTR_METHOD_NO_DICT", - "LOAD_ATTR_METHOD_WITH_DICT", - "LOAD_ATTR_METHOD_WITH_VALUES", - ], - "LOAD_CONST": [ - "LOAD_CONST__LOAD_FAST", - ], - "LOAD_FAST": [ - "LOAD_FAST__LOAD_CONST", - "LOAD_FAST__LOAD_FAST", - ], - "LOAD_GLOBAL": [ - "LOAD_GLOBAL_ADAPTIVE", - "LOAD_GLOBAL_BUILTIN", - "LOAD_GLOBAL_MODULE", - ], - "RESUME": [ - "RESUME_QUICK", - ], - "STORE_ATTR": [ - "STORE_ATTR_ADAPTIVE", - "STORE_ATTR_INSTANCE_VALUE", - "STORE_ATTR_SLOT", - "STORE_ATTR_WITH_HINT", - ], - "STORE_FAST": [ - "STORE_FAST__LOAD_FAST", - "STORE_FAST__STORE_FAST", - ], - "STORE_SUBSCR": [ - "STORE_SUBSCR_ADAPTIVE", - "STORE_SUBSCR_DICT", - "STORE_SUBSCR_LIST_INT", - ], - "UNPACK_SEQUENCE": [ - "UNPACK_SEQUENCE_ADAPTIVE", - "UNPACK_SEQUENCE_LIST", - "UNPACK_SEQUENCE_TUPLE", - "UNPACK_SEQUENCE_TWO_TUPLE", - ], -} -_specialized_instructions = [ - opcode for family in _specializations.values() for opcode in family -] -_specialization_stats = [ - "success", - "failure", - "hit", - "deferred", - "miss", - "deopt", -] + +_intrinsic_1_descs = _opcode.get_intrinsic1_descs() +_intrinsic_2_descs = _opcode.get_intrinsic2_descs() +_nb_ops = _opcode.get_nb_ops() + +hascompare = [opmap["COMPARE_OP"]] _cache_format = { "LOAD_GLOBAL": { "counter": 1, "index": 1, - "module_keys_version": 2, + "module_keys_version": 1, "builtin_keys_version": 1, }, "BINARY_OP": { @@ -412,16 +55,19 @@ def pseudo_op(name, op, real_ops): }, "COMPARE_OP": { "counter": 1, - "mask": 1, + }, + "CONTAINS_OP": { + "counter": 1, }, "BINARY_SUBSCR": { "counter": 1, - "type_version": 2, - "func_version": 1, }, "FOR_ITER": { "counter": 1, }, + "LOAD_SUPER_ATTR": { + "counter": 1, + }, "LOAD_ATTR": { "counter": 1, "version": 2, @@ -436,13 +82,34 @@ def pseudo_op(name, op, real_ops): "CALL": { "counter": 1, "func_version": 2, - "min_args": 1, }, "STORE_SUBSCR": { "counter": 1, }, + "SEND": { + "counter": 1, + }, + "JUMP_BACKWARD": { + "counter": 1, + }, + "TO_BOOL": { + "counter": 1, + "version": 2, + }, + "POP_JUMP_IF_TRUE": { + "counter": 1, + }, + "POP_JUMP_IF_FALSE": { + "counter": 1, + }, + "POP_JUMP_IF_NONE": { + "counter": 1, + }, + "POP_JUMP_IF_NOT_NONE": { + "counter": 1, + }, } -_inline_cache_entries = [ - sum(_cache_format.get(opname[opcode], {}).values()) for opcode in range(256) -] +_inline_cache_entries = { + name : sum(value.values()) for (name, value) in _cache_format.items() +} diff --git a/Lib/runpy.py b/Lib/runpy.py index c7d3d8caad..ef54d3282e 100644 --- a/Lib/runpy.py +++ b/Lib/runpy.py @@ -14,18 +14,20 @@ import importlib.machinery # importlib first so we can test #15386 via -m import importlib.util import io -import types import os __all__ = [ "run_module", "run_path", ] +# avoid 'import types' just for ModuleType +ModuleType = type(sys) + class _TempModule(object): """Temporarily replace a module in sys.modules with an empty namespace""" def __init__(self, mod_name): self.mod_name = mod_name - self.module = types.ModuleType(mod_name) + self.module = ModuleType(mod_name) self._saved_module = [] def __enter__(self): @@ -245,17 +247,17 @@ def _get_main_module_details(error=ImportError): sys.modules[main_name] = saved_main -def _get_code_from_file(run_name, fname): +def _get_code_from_file(fname): # Check for a compiled file first from pkgutil import read_code - decoded_path = os.path.abspath(os.fsdecode(fname)) - with io.open_code(decoded_path) as f: + code_path = os.path.abspath(fname) + with io.open_code(code_path) as f: code = read_code(f) if code is None: # That didn't work, so try it as normal source code - with io.open_code(decoded_path) as f: + with io.open_code(code_path) as f: code = compile(f.read(), fname, 'exec') - return code, fname + return code def run_path(path_name, init_globals=None, run_name=None): """Execute code located at the specified filesystem location. @@ -277,17 +279,13 @@ def run_path(path_name, init_globals=None, run_name=None): pkg_name = run_name.rpartition(".")[0] from pkgutil import get_importer importer = get_importer(path_name) - # Trying to avoid importing imp so as to not consume the deprecation warning. - is_NullImporter = False - if type(importer).__module__ == 'imp': - if type(importer).__name__ == 'NullImporter': - is_NullImporter = True - if isinstance(importer, type(None)) or is_NullImporter: + path_name = os.fsdecode(path_name) + if isinstance(importer, type(None)): # Not a valid sys.path entry, so run the code directly # execfile() doesn't help as we want to allow compiled files - code, fname = _get_code_from_file(run_name, path_name) + code = _get_code_from_file(path_name) return _run_module_code(code, init_globals, run_name, - pkg_name=pkg_name, script_name=fname) + pkg_name=pkg_name, script_name=path_name) else: # Finder is defined for path, so add it to # the start of sys.path diff --git a/Lib/site.py b/Lib/site.py index acc8481b13..2983ca7154 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -75,6 +75,7 @@ import _sitebuiltins import io import stat +import errno # Prefixes for site-packages; add additional prefixes like /usr/local here PREFIXES = [sys.prefix, sys.exec_prefix] @@ -179,35 +180,46 @@ def addpackage(sitedir, name, known_paths): return _trace(f"Processing .pth file: {fullname!r}") try: - # locale encoding is not ideal especially on Windows. But we have used - # it for a long time. setuptools uses the locale encoding too. - f = io.TextIOWrapper(io.open_code(fullname), encoding="locale") + with io.open_code(fullname) as f: + pth_content = f.read() except OSError: return - with f: - for n, line in enumerate(f): - if line.startswith("#"): - continue - if line.strip() == "": + + try: + # Accept BOM markers in .pth files as we do in source files + # (Windows PowerShell 5.1 makes it hard to emit UTF-8 files without a BOM) + pth_content = pth_content.decode("utf-8-sig") + except UnicodeDecodeError: + # Fallback to locale encoding for backward compatibility. + # We will deprecate this fallback in the future. + import locale + pth_content = pth_content.decode(locale.getencoding()) + _trace(f"Cannot read {fullname!r} as UTF-8. " + f"Using fallback encoding {locale.getencoding()!r}") + + for n, line in enumerate(pth_content.splitlines(), 1): + if line.startswith("#"): + continue + if line.strip() == "": + continue + try: + if line.startswith(("import ", "import\t")): + exec(line) continue - try: - if line.startswith(("import ", "import\t")): - exec(line) - continue - line = line.rstrip() - dir, dircase = makepath(sitedir, line) - if not dircase in known_paths and os.path.exists(dir): - sys.path.append(dir) - known_paths.add(dircase) - except Exception as exc: - print("Error processing line {:d} of {}:\n".format(n+1, fullname), - file=sys.stderr) - import traceback - for record in traceback.format_exception(exc): - for line in record.splitlines(): - print(' '+line, file=sys.stderr) - print("\nRemainder of file ignored", file=sys.stderr) - break + line = line.rstrip() + dir, dircase = makepath(sitedir, line) + if dircase not in known_paths and os.path.exists(dir): + sys.path.append(dir) + known_paths.add(dircase) + except Exception as exc: + print(f"Error processing line {n:d} of {fullname}:\n", + file=sys.stderr) + import traceback + for record in traceback.format_exception(exc): + for line in record.splitlines(): + print(' '+line, file=sys.stderr) + print("\nRemainder of file ignored", file=sys.stderr) + break if reset: known_paths = None return known_paths @@ -270,14 +282,18 @@ def check_enableusersite(): # # See https://bugs.python.org/issue29585 +# Copy of sysconfig._get_implementation() +def _get_implementation(): + return 'RustPython' # XXX: RustPython; for site-packages + # Copy of sysconfig._getuserbase() def _getuserbase(): env_base = os.environ.get("PYTHONUSERBASE", None) if env_base: return env_base - # Emscripten, VxWorks, and WASI have no home directories - if sys.platform in {"emscripten", "vxworks", "wasi"}: + # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories + if sys.platform in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}: return None def joinuser(*args): @@ -285,8 +301,7 @@ def joinuser(*args): if os.name == "nt": base = os.environ.get("APPDATA") or "~" - # XXX: RUSTPYTHON; please keep this change for site-packages - return joinuser(base, "RustPython") + return joinuser(base, _get_implementation()) if sys.platform == "darwin" and sys._framework: return joinuser("~", "Library", sys._framework, @@ -298,15 +313,22 @@ def joinuser(*args): # Same to sysconfig.get_path('purelib', os.name+'_user') def _get_path(userbase): version = sys.version_info + if hasattr(sys, 'abiflags') and 't' in sys.abiflags: + abi_thread = 't' + else: + abi_thread = '' + implementation = _get_implementation() + implementation_lower = implementation.lower() if os.name == 'nt': ver_nodot = sys.winver.replace('.', '') - return f'{userbase}\\RustPython{ver_nodot}\\site-packages' + return f'{userbase}\\{implementation}{ver_nodot}\\site-packages' if sys.platform == 'darwin' and sys._framework: - return f'{userbase}/lib/rustpython/site-packages' + return f'{userbase}/lib/{implementation_lower}/site-packages' - return f'{userbase}/lib/rustpython{version[0]}.{version[1]}/site-packages' + # XXX: RUSTPYTHON + return f'{userbase}/lib/rustpython{version[0]}.{version[1]}{abi_thread}/site-packages' def getuserbase(): @@ -372,6 +394,12 @@ def getsitepackages(prefixes=None): continue seen.add(prefix) + implementation = _get_implementation().lower() + ver = sys.version_info + if hasattr(sys, 'abiflags') and 't' in sys.abiflags: + abi_thread = 't' + else: + abi_thread = '' if os.sep == '/': libdirs = [sys.platlibdir] if sys.platlibdir != "lib": @@ -379,8 +407,7 @@ def getsitepackages(prefixes=None): for libdir in libdirs: path = os.path.join(prefix, libdir, - # XXX: RUSTPYTHON; please keep this change for site-packages - "rustpython%d.%d" % sys.version_info[:2], + f"{implementation}{ver[0]}.{ver[1]}{abi_thread}", "site-packages") sitepackages.append(path) else: @@ -417,8 +444,9 @@ def setcopyright(): """Set 'copyright' and 'credits' in builtins""" builtins.copyright = _sitebuiltins._Printer("copyright", sys.copyright) builtins.credits = _sitebuiltins._Printer("credits", """\ - Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands - for supporting Python development. See www.python.org for more information.""") + Thanks to CWI, CNRI, BeOpen, Zope Corporation, the Python Software + Foundation, and a cast of thousands for supporting Python + development. See www.python.org for more information.""") files, dirs = [], [] # Not all modules are required to have a __file__ attribute. See # PEP 420 for more details. @@ -437,27 +465,76 @@ def setcopyright(): def sethelper(): builtins.help = _sitebuiltins._Helper() + +def gethistoryfile(): + """Check if the PYTHON_HISTORY environment variable is set and define + it as the .python_history file. If PYTHON_HISTORY is not set, use the + default .python_history file. + """ + if not sys.flags.ignore_environment: + history = os.environ.get("PYTHON_HISTORY") + if history: + return history + return os.path.join(os.path.expanduser('~'), + '.python_history') + + def enablerlcompleter(): """Enable default readline configuration on interactive prompts, by registering a sys.__interactivehook__. + """ + sys.__interactivehook__ = register_readline + + +def register_readline(): + """Configure readline completion on interactive prompts. If the readline module can be imported, the hook will set the Tab key as completion key and register ~/.python_history as history file. This can be overridden in the sitecustomize or usercustomize module, or in a PYTHONSTARTUP file. """ - def register_readline(): - import atexit + if not sys.flags.ignore_environment: + PYTHON_BASIC_REPL = os.getenv("PYTHON_BASIC_REPL") + else: + PYTHON_BASIC_REPL = False + + import atexit + + try: try: import readline - import rlcompleter except ImportError: - return + readline = None + else: + import rlcompleter # noqa: F401 + except ImportError: + return + try: + if PYTHON_BASIC_REPL: + CAN_USE_PYREPL = False + else: + original_path = sys.path + sys.path = [p for p in original_path if p != ''] + try: + import _pyrepl.readline + if os.name == "nt": + import _pyrepl.windows_console + console_errors = (_pyrepl.windows_console._error,) + else: + import _pyrepl.unix_console + console_errors = _pyrepl.unix_console._error + from _pyrepl.main import CAN_USE_PYREPL + finally: + sys.path = original_path + except ImportError: + return + + if readline is not None: # Reading the initialization (config) file may not be enough to set a # completion key, so we set one first and then read the file. - readline_doc = getattr(readline, '__doc__', '') - if readline_doc is not None and 'libedit' in readline_doc: + if readline.backend == 'editline': readline.parse_and_bind('bind ^I rl_complete') else: readline.parse_and_bind('tab: complete') @@ -471,30 +548,44 @@ def register_readline(): # want to ignore the exception. pass - if readline.get_current_history_length() == 0: - # If no history was loaded, default to .python_history. - # The guard is necessary to avoid doubling history size at - # each interpreter exit when readline was already configured - # through a PYTHONSTARTUP hook, see: - # http://bugs.python.org/issue5845#msg198636 - history = os.path.join(os.path.expanduser('~'), - '.python_history') + if readline is None or readline.get_current_history_length() == 0: + # If no history was loaded, default to .python_history, + # or PYTHON_HISTORY. + # The guard is necessary to avoid doubling history size at + # each interpreter exit when readline was already configured + # through a PYTHONSTARTUP hook, see: + # http://bugs.python.org/issue5845#msg198636 + history = gethistoryfile() + + if CAN_USE_PYREPL: + readline_module = _pyrepl.readline + exceptions = (OSError, *console_errors) + else: + if readline is None: + return + readline_module = readline + exceptions = OSError + + try: + readline_module.read_history_file(history) + except exceptions: + pass + + def write_history(): try: - readline.read_history_file(history) - except OSError: + readline_module.write_history_file(history) + except (FileNotFoundError, PermissionError): + # home directory does not exist or is not writable + # https://bugs.python.org/issue19891 pass + except OSError: + if errno.EROFS: + pass # gh-128066: read-only file system + else: + raise - def write_history(): - try: - readline.write_history_file(history) - except OSError: - # bpo-19891, bpo-41193: Home directory does not exist - # or is not writable, or the filesystem is read-only. - pass - - atexit.register(write_history) + atexit.register(write_history) - sys.__interactivehook__ = register_readline def venv(known_paths): global PREFIXES, ENABLE_USER_SITE @@ -679,17 +770,5 @@ def exists(path): print(textwrap.dedent(help % (sys.argv[0], os.pathsep))) sys.exit(10) -def gethistoryfile(): - """Check if the PYTHON_HISTORY environment variable is set and define - it as the .python_history file. If PYTHON_HISTORY is not set, use the - default .python_history file. - """ - if not sys.flags.ignore_environment: - history = os.environ.get("PYTHON_HISTORY") - if history: - return history - return os.path.join(os.path.expanduser('~'), - '.python_history') - if __name__ == '__main__': _script() diff --git a/Lib/sysconfig.py b/Lib/sysconfig/__init__.py similarity index 53% rename from Lib/sysconfig.py rename to Lib/sysconfig/__init__.py index 9999d6bbd5..3ad9df603f 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig/__init__.py @@ -1,10 +1,9 @@ -# XXX: RUSTPYTHON; Trick to make sysconfig work as RustPython -exec(r''' """Access to Python's configuration information.""" import os import sys -from os.path import pardir, realpath +import threading +from os.path import realpath __all__ = [ 'get_config_h_filename', @@ -22,29 +21,30 @@ # Keys for get_config_var() that are never converted to Python integers. _ALWAYS_STR = { + 'IPHONEOS_DEPLOYMENT_TARGET', 'MACOSX_DEPLOYMENT_TARGET', } _INSTALL_SCHEMES = { 'posix_prefix': { - 'stdlib': '{installed_base}/{platlibdir}/python{py_version_short}', - 'platstdlib': '{platbase}/{platlibdir}/python{py_version_short}', - 'purelib': '{base}/lib/python{py_version_short}/site-packages', - 'platlib': '{platbase}/{platlibdir}/python{py_version_short}/site-packages', + 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'purelib': '{base}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}/site-packages', 'include': - '{installed_base}/include/python{py_version_short}{abiflags}', + '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}', 'platinclude': - '{installed_platbase}/include/python{py_version_short}{abiflags}', + '{installed_platbase}/include/{implementation_lower}{py_version_short}{abiflags}', 'scripts': '{base}/bin', 'data': '{base}', }, 'posix_home': { - 'stdlib': '{installed_base}/lib/python', - 'platstdlib': '{base}/lib/python', - 'purelib': '{base}/lib/python', - 'platlib': '{base}/lib/python', - 'include': '{installed_base}/include/python', - 'platinclude': '{installed_base}/include/python', + 'stdlib': '{installed_base}/lib/{implementation_lower}', + 'platstdlib': '{base}/lib/{implementation_lower}', + 'purelib': '{base}/lib/{implementation_lower}', + 'platlib': '{base}/lib/{implementation_lower}', + 'include': '{installed_base}/include/{implementation_lower}', + 'platinclude': '{installed_base}/include/{implementation_lower}', 'scripts': '{base}/bin', 'data': '{base}', }, @@ -58,6 +58,7 @@ 'scripts': '{base}/Scripts', 'data': '{base}', }, + # Downstream distributors can overwrite the default install scheme. # This is done to support downstream modifications where distributors change # the installation layout (eg. different site-packages directory). @@ -76,14 +77,14 @@ # Downstream distributors who patch posix_prefix/nt scheme are encouraged to # leave the following schemes unchanged 'posix_venv': { - 'stdlib': '{installed_base}/{platlibdir}/python{py_version_short}', - 'platstdlib': '{platbase}/{platlibdir}/python{py_version_short}', - 'purelib': '{base}/lib/python{py_version_short}/site-packages', - 'platlib': '{platbase}/{platlibdir}/python{py_version_short}/site-packages', + 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'purelib': '{base}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}/site-packages', 'include': - '{installed_base}/include/python{py_version_short}{abiflags}', + '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}', 'platinclude': - '{installed_platbase}/include/python{py_version_short}{abiflags}', + '{installed_platbase}/include/{implementation_lower}{py_version_short}{abiflags}', 'scripts': '{base}/bin', 'data': '{base}', }, @@ -105,6 +106,8 @@ else: _INSTALL_SCHEMES['venv'] = _INSTALL_SCHEMES['posix_venv'] +def _get_implementation(): + return 'RustPython' # XXX: For site-packages # NOTE: site.py has copy of this function. # Sync it when modify this function. @@ -113,8 +116,8 @@ def _getuserbase(): if env_base: return env_base - # Emscripten, VxWorks, and WASI have no home directories - if sys.platform in {"emscripten", "vxworks", "wasi"}: + # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories + if sys.platform in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}: return None def joinuser(*args): @@ -122,7 +125,7 @@ def joinuser(*args): if os.name == "nt": base = os.environ.get("APPDATA") or "~" - return joinuser(base, "Python") + return joinuser(base, _get_implementation()) if sys.platform == "darwin" and sys._framework: return joinuser("~", "Library", sys._framework, @@ -136,29 +139,29 @@ def joinuser(*args): _INSTALL_SCHEMES |= { # NOTE: When modifying "purelib" scheme, update site._get_path() too. 'nt_user': { - 'stdlib': '{userbase}/Python{py_version_nodot_plat}', - 'platstdlib': '{userbase}/Python{py_version_nodot_plat}', - 'purelib': '{userbase}/Python{py_version_nodot_plat}/site-packages', - 'platlib': '{userbase}/Python{py_version_nodot_plat}/site-packages', - 'include': '{userbase}/Python{py_version_nodot_plat}/Include', - 'scripts': '{userbase}/Python{py_version_nodot_plat}/Scripts', + 'stdlib': '{userbase}/{implementation}{py_version_nodot_plat}', + 'platstdlib': '{userbase}/{implementation}{py_version_nodot_plat}', + 'purelib': '{userbase}/{implementation}{py_version_nodot_plat}/site-packages', + 'platlib': '{userbase}/{implementation}{py_version_nodot_plat}/site-packages', + 'include': '{userbase}/{implementation}{py_version_nodot_plat}/Include', + 'scripts': '{userbase}/{implementation}{py_version_nodot_plat}/Scripts', 'data': '{userbase}', }, 'posix_user': { - 'stdlib': '{userbase}/{platlibdir}/python{py_version_short}', - 'platstdlib': '{userbase}/{platlibdir}/python{py_version_short}', - 'purelib': '{userbase}/lib/python{py_version_short}/site-packages', - 'platlib': '{userbase}/lib/python{py_version_short}/site-packages', - 'include': '{userbase}/include/python{py_version_short}', + 'stdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'platstdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'include': '{userbase}/include/{implementation_lower}{py_version_short}{abi_thread}', 'scripts': '{userbase}/bin', 'data': '{userbase}', }, 'osx_framework_user': { - 'stdlib': '{userbase}/lib/python', - 'platstdlib': '{userbase}/lib/python', - 'purelib': '{userbase}/lib/python/site-packages', - 'platlib': '{userbase}/lib/python/site-packages', - 'include': '{userbase}/include/python{py_version_short}', + 'stdlib': '{userbase}/lib/{implementation_lower}', + 'platstdlib': '{userbase}/lib/{implementation_lower}', + 'purelib': '{userbase}/lib/{implementation_lower}/site-packages', + 'platlib': '{userbase}/lib/{implementation_lower}/site-packages', + 'include': '{userbase}/include/{implementation_lower}{py_version_short}', 'scripts': '{userbase}/bin', 'data': '{userbase}', }, @@ -170,19 +173,15 @@ def joinuser(*args): _PY_VERSION = sys.version.split()[0] _PY_VERSION_SHORT = f'{sys.version_info[0]}.{sys.version_info[1]}' _PY_VERSION_SHORT_NO_DOT = f'{sys.version_info[0]}{sys.version_info[1]}' -_PREFIX = os.path.normpath(sys.prefix) _BASE_PREFIX = os.path.normpath(sys.base_prefix) -_EXEC_PREFIX = os.path.normpath(sys.exec_prefix) _BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) +# Mutex guarding initialization of _CONFIG_VARS. +_CONFIG_VARS_LOCK = threading.RLock() _CONFIG_VARS = None +# True iff _CONFIG_VARS has been fully initialized. +_CONFIG_VARS_INITIALIZED = False _USER_BASE = None -# Regexes needed for parsing Makefile (and similar syntaxes, -# like old-style Setup files). -_variable_rx = r"([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)" -_findvar1_rx = r"\$\(([A-Za-z][A-Za-z0-9_]*)\)" -_findvar2_rx = r"\${([A-Za-z][A-Za-z0-9_]*)}" - def _safe_realpath(path): try: @@ -221,8 +220,15 @@ def _safe_realpath(path): def is_python_build(check_home=None): if check_home is not None: import warnings - warnings.warn("check_home argument is deprecated and ignored.", - DeprecationWarning, stacklevel=2) + warnings.warn( + ( + 'The check_home argument of sysconfig.is_python_build is ' + 'deprecated and its value is ignored. ' + 'It will be removed in Python 3.15.' + ), + DeprecationWarning, + stacklevel=2, + ) for fn in ("Setup", "Setup.local"): if os.path.isfile(os.path.join(_PROJECT_BASE, "Modules", fn)): return True @@ -291,6 +297,7 @@ def _get_preferred_schemes(): 'home': 'posix_home', 'user': 'osx_framework_user', } + return { 'prefix': 'posix_prefix', 'home': 'posix_home', @@ -314,134 +321,6 @@ def get_default_scheme(): return get_preferred_scheme('prefix') -def _parse_makefile(filename, vars=None, keep_unresolved=True): - """Parse a Makefile-style file. - - A dictionary containing name/value pairs is returned. If an - optional dictionary is passed in as the second argument, it is - used instead of a new dictionary. - """ - import re - - if vars is None: - vars = {} - done = {} - notdone = {} - - with open(filename, encoding=sys.getfilesystemencoding(), - errors="surrogateescape") as f: - lines = f.readlines() - - for line in lines: - if line.startswith('#') or line.strip() == '': - continue - m = re.match(_variable_rx, line) - if m: - n, v = m.group(1, 2) - v = v.strip() - # `$$' is a literal `$' in make - tmpv = v.replace('$$', '') - - if "$" in tmpv: - notdone[n] = v - else: - try: - if n in _ALWAYS_STR: - raise ValueError - - v = int(v) - except ValueError: - # insert literal `$' - done[n] = v.replace('$$', '$') - else: - done[n] = v - - # do variable interpolation here - variables = list(notdone.keys()) - - # Variables with a 'PY_' prefix in the makefile. These need to - # be made available without that prefix through sysconfig. - # Special care is needed to ensure that variable expansion works, even - # if the expansion uses the name without a prefix. - renamed_variables = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS') - - while len(variables) > 0: - for name in tuple(variables): - value = notdone[name] - m1 = re.search(_findvar1_rx, value) - m2 = re.search(_findvar2_rx, value) - if m1 and m2: - m = m1 if m1.start() < m2.start() else m2 - else: - m = m1 if m1 else m2 - if m is not None: - n = m.group(1) - found = True - if n in done: - item = str(done[n]) - elif n in notdone: - # get it on a subsequent round - found = False - elif n in os.environ: - # do it like make: fall back to environment - item = os.environ[n] - - elif n in renamed_variables: - if (name.startswith('PY_') and - name[3:] in renamed_variables): - item = "" - - elif 'PY_' + n in notdone: - found = False - - else: - item = str(done['PY_' + n]) - - else: - done[n] = item = "" - - if found: - after = value[m.end():] - value = value[:m.start()] + item + after - if "$" in after: - notdone[name] = value - else: - try: - if name in _ALWAYS_STR: - raise ValueError - value = int(value) - except ValueError: - done[name] = value.strip() - else: - done[name] = value - variables.remove(name) - - if name.startswith('PY_') \ - and name[3:] in renamed_variables: - - name = name[3:] - if name not in done: - done[name] = value - - else: - # Adds unresolved variables to the done dict. - # This is disabled when called from distutils.sysconfig - if keep_unresolved: - done[name] = value - # bogus variable reference (e.g. "prefix=$/opt/python"); - # just drop it since we can't deal - variables.remove(name) - - # strip spurious spaces - for k, v in done.items(): - if isinstance(v, str): - done[k] = v.strip() - - # save the results in the global dictionary - vars.update(done) - return vars - - def get_makefile_filename(): """Return the path of the Makefile.""" if _PYTHON_BUILD: @@ -462,91 +341,44 @@ def _get_sysconfigdata_name(): f'_sysconfigdata_{sys.abiflags}_{sys.platform}_{multiarch}', ) - -def _generate_posix_vars(): - """Generate the Python module containing build-time variables.""" - import pprint - vars = {} - # load the installed Makefile: - makefile = get_makefile_filename() - try: - _parse_makefile(makefile, vars) - except OSError as e: - msg = f"invalid Python installation: unable to open {makefile}" - if hasattr(e, "strerror"): - msg = f"{msg} ({e.strerror})" - raise OSError(msg) - # load the installed pyconfig.h: - config_h = get_config_h_filename() - try: - with open(config_h, encoding="utf-8") as f: - parse_config_h(f, vars) - except OSError as e: - msg = f"invalid Python installation: unable to open {config_h}" - if hasattr(e, "strerror"): - msg = f"{msg} ({e.strerror})" - raise OSError(msg) - # On AIX, there are wrong paths to the linker scripts in the Makefile - # -- these paths are relative to the Python source, but when installed - # the scripts are in another directory. - if _PYTHON_BUILD: - vars['BLDSHARED'] = vars['LDSHARED'] - - # There's a chicken-and-egg situation on OS X with regards to the - # _sysconfigdata module after the changes introduced by #15298: - # get_config_vars() is called by get_platform() as part of the - # `make pybuilddir.txt` target -- which is a precursor to the - # _sysconfigdata.py module being constructed. Unfortunately, - # get_config_vars() eventually calls _init_posix(), which attempts - # to import _sysconfigdata, which we won't have built yet. In order - # for _init_posix() to work, if we're on Darwin, just mock up the - # _sysconfigdata module manually and populate it with the build vars. - # This is more than sufficient for ensuring the subsequent call to - # get_platform() succeeds. - name = _get_sysconfigdata_name() - if 'darwin' in sys.platform: - import types - module = types.ModuleType(name) - module.build_time_vars = vars - sys.modules[name] = module - - pybuilddir = f'build/lib.{get_platform()}-{_PY_VERSION_SHORT}' - if hasattr(sys, "gettotalrefcount"): - pybuilddir += '-pydebug' - os.makedirs(pybuilddir, exist_ok=True) - destfile = os.path.join(pybuilddir, name + '.py') - - with open(destfile, 'w', encoding='utf8') as f: - f.write('# system configuration generated and used by' - ' the sysconfig module\n') - f.write('build_time_vars = ') - pprint.pprint(vars, stream=f) - - # Create file used for sys.path fixup -- see Modules/getpath.c - with open('pybuilddir.txt', 'w', encoding='utf8') as f: - f.write(pybuilddir) - def _init_posix(vars): """Initialize the module as appropriate for POSIX systems.""" # _sysconfigdata is generated at build time, see _generate_posix_vars() name = _get_sysconfigdata_name() - _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) + + # For cross builds, the path to the target's sysconfigdata must be specified + # so it can be imported. It cannot be in PYTHONPATH, as foreign modules in + # sys.path can cause crashes when loaded by the host interpreter. + # Rely on truthiness as a valueless env variable is still an empty string. + # See OS X note in _generate_posix_vars re _sysconfigdata. + if (path := os.environ.get('_PYTHON_SYSCONFIGDATA_PATH')): + from importlib.machinery import FileFinder, SourceFileLoader, SOURCE_SUFFIXES + from importlib.util import module_from_spec + spec = FileFinder(path, (SourceFileLoader, SOURCE_SUFFIXES)).find_spec(name) + _temp = module_from_spec(spec) + spec.loader.exec_module(_temp) + else: + _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) build_time_vars = _temp.build_time_vars vars.update(build_time_vars) def _init_non_posix(vars): """Initialize the module as appropriate for NT""" # set basic install directories - import _imp + import _winapi + import _sysconfig vars['LIBDEST'] = get_path('stdlib') vars['BINLIBDEST'] = get_path('platstdlib') vars['INCLUDEPY'] = get_path('include') - try: - # GH-99201: _imp.extension_suffixes may be empty when - # HAVE_DYNAMIC_LOADING is not set. In this case, don't set EXT_SUFFIX. - vars['EXT_SUFFIX'] = _imp.extension_suffixes()[0] - except IndexError: - pass + + # Add EXT_SUFFIX, SOABI, and Py_GIL_DISABLED + vars.update(_sysconfig.config_vars()) + + vars['LIBDIR'] = _safe_realpath(os.path.join(get_config_var('installed_base'), 'libs')) + if hasattr(sys, 'dllhandle'): + dllhandle = _winapi.GetModuleFileName(sys.dllhandle) + vars['LIBRARY'] = os.path.basename(_safe_realpath(dllhandle)) + vars['LDLIBRARY'] = vars['LIBRARY'] vars['EXE'] = '.exe' vars['VERSION'] = _PY_VERSION_SHORT_NO_DOT vars['BINDIR'] = os.path.dirname(_safe_realpath(sys.executable)) @@ -595,7 +427,7 @@ def get_config_h_filename(): """Return the path of pyconfig.h.""" if _PYTHON_BUILD: if os.name == "nt": - inc_dir = os.path.join(_PROJECT_BASE, "PC") + inc_dir = os.path.dirname(sys._base_executable) else: inc_dir = _PROJECT_BASE else: @@ -633,6 +465,78 @@ def get_path(name, scheme=get_default_scheme(), vars=None, expand=True): return get_paths(scheme, vars, expand)[name] +def _init_config_vars(): + global _CONFIG_VARS + _CONFIG_VARS = {} + # Normalized versions of prefix and exec_prefix are handy to have; + # in fact, these are the standard versions used most places in the + # Distutils. + _PREFIX = os.path.normpath(sys.prefix) + _EXEC_PREFIX = os.path.normpath(sys.exec_prefix) + _CONFIG_VARS['prefix'] = _PREFIX # FIXME: This gets overwriten by _init_posix. + _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX # FIXME: This gets overwriten by _init_posix. + _CONFIG_VARS['py_version'] = _PY_VERSION + _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT + _CONFIG_VARS['py_version_nodot'] = _PY_VERSION_SHORT_NO_DOT + _CONFIG_VARS['installed_base'] = _BASE_PREFIX + _CONFIG_VARS['base'] = _PREFIX + _CONFIG_VARS['installed_platbase'] = _BASE_EXEC_PREFIX + _CONFIG_VARS['platbase'] = _EXEC_PREFIX + _CONFIG_VARS['projectbase'] = _PROJECT_BASE + _CONFIG_VARS['platlibdir'] = sys.platlibdir + _CONFIG_VARS['implementation'] = _get_implementation() + _CONFIG_VARS['implementation_lower'] = _get_implementation().lower() + try: + _CONFIG_VARS['abiflags'] = sys.abiflags + except AttributeError: + # sys.abiflags may not be defined on all platforms. + _CONFIG_VARS['abiflags'] = '' + try: + _CONFIG_VARS['py_version_nodot_plat'] = sys.winver.replace('.', '') + except AttributeError: + _CONFIG_VARS['py_version_nodot_plat'] = '' + + if os.name == 'nt': + _init_non_posix(_CONFIG_VARS) + _CONFIG_VARS['VPATH'] = sys._vpath + if os.name == 'posix': + _init_posix(_CONFIG_VARS) + if _HAS_USER_BASE: + # Setting 'userbase' is done below the call to the + # init function to enable using 'get_config_var' in + # the init-function. + _CONFIG_VARS['userbase'] = _getuserbase() + + # e.g., 't' for free-threaded or '' for default build + _CONFIG_VARS['abi_thread'] = 't' if _CONFIG_VARS.get('Py_GIL_DISABLED') else '' + + # Always convert srcdir to an absolute path + srcdir = _CONFIG_VARS.get('srcdir', _PROJECT_BASE) + if os.name == 'posix': + if _PYTHON_BUILD: + # If srcdir is a relative path (typically '.' or '..') + # then it should be interpreted relative to the directory + # containing Makefile. + base = os.path.dirname(get_makefile_filename()) + srcdir = os.path.join(base, srcdir) + else: + # srcdir is not meaningful since the installation is + # spread about the filesystem. We choose the + # directory containing the Makefile since we know it + # exists. + srcdir = os.path.dirname(get_makefile_filename()) + _CONFIG_VARS['srcdir'] = _safe_realpath(srcdir) + + # OS X platforms require special customization to handle + # multi-architecture, multi-os-version installers + if sys.platform == 'darwin': + import _osx_support + _osx_support.customize_config_vars(_CONFIG_VARS) + + global _CONFIG_VARS_INITIALIZED + _CONFIG_VARS_INITIALIZED = True + + def get_config_vars(*args): """With no arguments, return a dictionary of all configuration variables relevant for the current platform. @@ -643,66 +547,26 @@ def get_config_vars(*args): With arguments, return a list of values that result from looking up each argument in the configuration variable dictionary. """ - global _CONFIG_VARS - if _CONFIG_VARS is None: - _CONFIG_VARS = {} - # Normalized versions of prefix and exec_prefix are handy to have; - # in fact, these are the standard versions used most places in the - # Distutils. - _CONFIG_VARS['prefix'] = _PREFIX - _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX - _CONFIG_VARS['py_version'] = _PY_VERSION - _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT - _CONFIG_VARS['py_version_nodot'] = _PY_VERSION_SHORT_NO_DOT - _CONFIG_VARS['installed_base'] = _BASE_PREFIX - _CONFIG_VARS['base'] = _PREFIX - _CONFIG_VARS['installed_platbase'] = _BASE_EXEC_PREFIX - _CONFIG_VARS['platbase'] = _EXEC_PREFIX - _CONFIG_VARS['projectbase'] = _PROJECT_BASE - _CONFIG_VARS['platlibdir'] = sys.platlibdir - try: - _CONFIG_VARS['abiflags'] = sys.abiflags - except AttributeError: - # sys.abiflags may not be defined on all platforms. - _CONFIG_VARS['abiflags'] = '' - try: - _CONFIG_VARS['py_version_nodot_plat'] = sys.winver.replace('.', '') - except AttributeError: - _CONFIG_VARS['py_version_nodot_plat'] = '' - - if os.name == 'nt': - _init_non_posix(_CONFIG_VARS) - _CONFIG_VARS['VPATH'] = sys._vpath - if os.name == 'posix': - _init_posix(_CONFIG_VARS) - if _HAS_USER_BASE: - # Setting 'userbase' is done below the call to the - # init function to enable using 'get_config_var' in - # the init-function. - _CONFIG_VARS['userbase'] = _getuserbase() - - # Always convert srcdir to an absolute path - srcdir = _CONFIG_VARS.get('srcdir', _PROJECT_BASE) - if os.name == 'posix': - if _PYTHON_BUILD: - # If srcdir is a relative path (typically '.' or '..') - # then it should be interpreted relative to the directory - # containing Makefile. - base = os.path.dirname(get_makefile_filename()) - srcdir = os.path.join(base, srcdir) - else: - # srcdir is not meaningful since the installation is - # spread about the filesystem. We choose the - # directory containing the Makefile since we know it - # exists. - srcdir = os.path.dirname(get_makefile_filename()) - _CONFIG_VARS['srcdir'] = _safe_realpath(srcdir) - - # OS X platforms require special customization to handle - # multi-architecture, multi-os-version installers - if sys.platform == 'darwin': - import _osx_support - _osx_support.customize_config_vars(_CONFIG_VARS) + global _CONFIG_VARS_INITIALIZED + + # Avoid claiming the lock once initialization is complete. + if not _CONFIG_VARS_INITIALIZED: + with _CONFIG_VARS_LOCK: + # Test again with the lock held to avoid races. Note that + # we test _CONFIG_VARS here, not _CONFIG_VARS_INITIALIZED, + # to ensure that recursive calls to get_config_vars() + # don't re-enter init_config_vars(). + if _CONFIG_VARS is None: + _init_config_vars() + else: + # If the site module initialization happened after _CONFIG_VARS was + # initialized, a virtual environment might have been activated, resulting in + # variables like sys.prefix changing their value, so we need to re-init the + # config vars (see GH-126789). + if _CONFIG_VARS['base'] != os.path.normpath(sys.prefix): + with _CONFIG_VARS_LOCK: + _CONFIG_VARS_INITIALIZED = False + _init_config_vars() if args: vals = [] @@ -737,7 +601,8 @@ def get_platform(): solaris-2.6-sun4u Windows will return one of: - win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc) + win-amd64 (64-bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc) + win-arm64 (64-bit Windows on ARM64 (aka AArch64) win32 (all others - specifically, sys.platform is returned) For other non-POSIX platforms, currently just returns 'sys.platform'. @@ -770,10 +635,22 @@ def get_platform(): machine = machine.replace('/', '-') if osname[:5] == "linux": - # At least on Linux/Intel, 'machine' is the processor -- - # i386, etc. - # XXX what about Alpha, SPARC, etc? - return f"{osname}-{machine}" + if sys.platform == "android": + osname = "android" + release = get_config_var("ANDROID_API_LEVEL") + + # Wheel tags use the ABI names from Android's own tools. + machine = { + "x86_64": "x86_64", + "i686": "x86", + "aarch64": "arm64_v8a", + "armv7l": "armeabi_v7a", + }[machine] + else: + # At least on Linux/Intel, 'machine' is the processor -- + # i386, etc. + # XXX what about Alpha, SPARC, etc? + return f"{osname}-{machine}" elif osname[:5] == "sunos": if release[0] >= "5": # SunOS 5 == Solaris 2 osname = "solaris" @@ -795,10 +672,15 @@ def get_platform(): if m: release = m.group() elif osname[:6] == "darwin": - import _osx_support - osname, release, machine = _osx_support.get_platform_osx( - get_config_vars(), - osname, release, machine) + if sys.platform == "ios": + release = get_config_vars().get("IPHONEOS_DEPLOYMENT_TARGET", "13.0") + osname = sys.platform + machine = sys.implementation._multiarch + else: + import _osx_support + osname, release, machine = _osx_support.get_platform_osx( + get_config_vars(), + osname, release, machine) return f"{osname}-{release}-{machine}" @@ -807,6 +689,10 @@ def get_python_version(): return _PY_VERSION_SHORT +def _get_python_version_abi(): + return _PY_VERSION_SHORT + get_config_var("abi_thread") + + def expand_makefile_vars(s, vars): """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in 'string' according to 'vars' (a dictionary mapping variable names to @@ -817,6 +703,9 @@ def expand_makefile_vars(s, vars): """ import re + _findvar1_rx = r"\$\(([A-Za-z][A-Za-z0-9_]*)\)" + _findvar2_rx = r"\${([A-Za-z][A-Za-z0-9_]*)}" + # This algorithm does multiple expansion, so if vars['foo'] contains # "${bar}", it will expand ${foo} to ${bar}, and then expand # ${bar}... and so forth. This is fine as long as 'vars' comes from @@ -831,28 +720,3 @@ def expand_makefile_vars(s, vars): else: break return s - - -def _print_dict(title, data): - for index, (key, value) in enumerate(sorted(data.items())): - if index == 0: - print(f'{title}: ') - print(f'\t{key} = "{value}"') - - -def _main(): - """Display all information sysconfig detains.""" - if '--generate-posix-vars' in sys.argv: - _generate_posix_vars() - return - print(f'Platform: "{get_platform()}"') - print(f'Python version: "{get_python_version()}"') - print(f'Current installation scheme: "{get_default_scheme()}"') - print() - _print_dict('Paths', get_paths()) - print() - _print_dict('Variables', get_config_vars()) - -if __name__ == '__main__': - _main() -'''.replace("Python", "RustPython").replace("/python", "/rustpython")) diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py new file mode 100644 index 0000000000..d7257b9d2d --- /dev/null +++ b/Lib/sysconfig/__main__.py @@ -0,0 +1,248 @@ +import os +import sys +from sysconfig import ( + _ALWAYS_STR, + _PYTHON_BUILD, + _get_sysconfigdata_name, + get_config_h_filename, + get_config_vars, + get_default_scheme, + get_makefile_filename, + get_paths, + get_platform, + get_python_version, + parse_config_h, +) + + +# Regexes needed for parsing Makefile (and similar syntaxes, +# like old-style Setup files). +_variable_rx = r"([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)" +_findvar1_rx = r"\$\(([A-Za-z][A-Za-z0-9_]*)\)" +_findvar2_rx = r"\${([A-Za-z][A-Za-z0-9_]*)}" + + +def _parse_makefile(filename, vars=None, keep_unresolved=True): + """Parse a Makefile-style file. + + A dictionary containing name/value pairs is returned. If an + optional dictionary is passed in as the second argument, it is + used instead of a new dictionary. + """ + import re + + if vars is None: + vars = {} + done = {} + notdone = {} + + with open(filename, encoding=sys.getfilesystemencoding(), + errors="surrogateescape") as f: + lines = f.readlines() + + for line in lines: + if line.startswith('#') or line.strip() == '': + continue + m = re.match(_variable_rx, line) + if m: + n, v = m.group(1, 2) + v = v.strip() + # `$$' is a literal `$' in make + tmpv = v.replace('$$', '') + + if "$" in tmpv: + notdone[n] = v + else: + try: + if n in _ALWAYS_STR: + raise ValueError + + v = int(v) + except ValueError: + # insert literal `$' + done[n] = v.replace('$$', '$') + else: + done[n] = v + + # do variable interpolation here + variables = list(notdone.keys()) + + # Variables with a 'PY_' prefix in the makefile. These need to + # be made available without that prefix through sysconfig. + # Special care is needed to ensure that variable expansion works, even + # if the expansion uses the name without a prefix. + renamed_variables = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS') + + while len(variables) > 0: + for name in tuple(variables): + value = notdone[name] + m1 = re.search(_findvar1_rx, value) + m2 = re.search(_findvar2_rx, value) + if m1 and m2: + m = m1 if m1.start() < m2.start() else m2 + else: + m = m1 if m1 else m2 + if m is not None: + n = m.group(1) + found = True + if n in done: + item = str(done[n]) + elif n in notdone: + # get it on a subsequent round + found = False + elif n in os.environ: + # do it like make: fall back to environment + item = os.environ[n] + + elif n in renamed_variables: + if (name.startswith('PY_') and + name[3:] in renamed_variables): + item = "" + + elif 'PY_' + n in notdone: + found = False + + else: + item = str(done['PY_' + n]) + + else: + done[n] = item = "" + + if found: + after = value[m.end():] + value = value[:m.start()] + item + after + if "$" in after: + notdone[name] = value + else: + try: + if name in _ALWAYS_STR: + raise ValueError + value = int(value) + except ValueError: + done[name] = value.strip() + else: + done[name] = value + variables.remove(name) + + if name.startswith('PY_') \ + and name[3:] in renamed_variables: + + name = name[3:] + if name not in done: + done[name] = value + + else: + # Adds unresolved variables to the done dict. + # This is disabled when called from distutils.sysconfig + if keep_unresolved: + done[name] = value + # bogus variable reference (e.g. "prefix=$/opt/python"); + # just drop it since we can't deal + variables.remove(name) + + # strip spurious spaces + for k, v in done.items(): + if isinstance(v, str): + done[k] = v.strip() + + # save the results in the global dictionary + vars.update(done) + return vars + + +def _print_config_dict(d, stream): + print ("{", file=stream) + for k, v in sorted(d.items()): + print(f" {k!r}: {v!r},", file=stream) + print ("}", file=stream) + + +def _generate_posix_vars(): + """Generate the Python module containing build-time variables.""" + vars = {} + # load the installed Makefile: + makefile = get_makefile_filename() + try: + _parse_makefile(makefile, vars) + except OSError as e: + msg = f"invalid Python installation: unable to open {makefile}" + if hasattr(e, "strerror"): + msg = f"{msg} ({e.strerror})" + raise OSError(msg) + # load the installed pyconfig.h: + config_h = get_config_h_filename() + try: + with open(config_h, encoding="utf-8") as f: + parse_config_h(f, vars) + except OSError as e: + msg = f"invalid Python installation: unable to open {config_h}" + if hasattr(e, "strerror"): + msg = f"{msg} ({e.strerror})" + raise OSError(msg) + # On AIX, there are wrong paths to the linker scripts in the Makefile + # -- these paths are relative to the Python source, but when installed + # the scripts are in another directory. + if _PYTHON_BUILD: + vars['BLDSHARED'] = vars['LDSHARED'] + + # There's a chicken-and-egg situation on OS X with regards to the + # _sysconfigdata module after the changes introduced by #15298: + # get_config_vars() is called by get_platform() as part of the + # `make pybuilddir.txt` target -- which is a precursor to the + # _sysconfigdata.py module being constructed. Unfortunately, + # get_config_vars() eventually calls _init_posix(), which attempts + # to import _sysconfigdata, which we won't have built yet. In order + # for _init_posix() to work, if we're on Darwin, just mock up the + # _sysconfigdata module manually and populate it with the build vars. + # This is more than sufficient for ensuring the subsequent call to + # get_platform() succeeds. + name = _get_sysconfigdata_name() + if 'darwin' in sys.platform: + import types + module = types.ModuleType(name) + module.build_time_vars = vars + sys.modules[name] = module + + pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' + if hasattr(sys, "gettotalrefcount"): + pybuilddir += '-pydebug' + os.makedirs(pybuilddir, exist_ok=True) + destfile = os.path.join(pybuilddir, name + '.py') + + with open(destfile, 'w', encoding='utf8') as f: + f.write('# system configuration generated and used by' + ' the sysconfig module\n') + f.write('build_time_vars = ') + _print_config_dict(vars, stream=f) + + # Create file used for sys.path fixup -- see Modules/getpath.c + with open('pybuilddir.txt', 'w', encoding='utf8') as f: + f.write(pybuilddir) + + +def _print_dict(title, data): + for index, (key, value) in enumerate(sorted(data.items())): + if index == 0: + print(f'{title}: ') + print(f'\t{key} = "{value}"') + + +def _main(): + """Display all information sysconfig detains.""" + if '--generate-posix-vars' in sys.argv: + _generate_posix_vars() + return + print(f'Platform: "{get_platform()}"') + print(f'Python version: "{get_python_version()}"') + print(f'Current installation scheme: "{get_default_scheme()}"') + print() + _print_dict('Paths', get_paths()) + print() + _print_dict('Variables', get_config_vars()) + + +if __name__ == '__main__': + try: + _main() + except BrokenPipeError: + pass diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 652a8cd92b..88369e25c1 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -7,7 +7,7 @@ import dataclasses import functools import logging -# import _opcode # TODO: RUSTPYTHON +import _opcode import os import re import stat diff --git a/Lib/test/support/bytecode_helper.py b/Lib/test/support/bytecode_helper.py index 388d126677..85bcd1f0f1 100644 --- a/Lib/test/support/bytecode_helper.py +++ b/Lib/test/support/bytecode_helper.py @@ -3,10 +3,26 @@ import unittest import dis import io -from _testinternalcapi import compiler_codegen, optimize_cfg, assemble_code_object +import opcode +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None _UNSPECIFIED = object() +def instructions_with_positions(instrs, co_positions): + # Return (instr, positions) pairs from the instrs list and co_positions + # iterator. The latter contains items for cache lines and the former + # doesn't, so those need to be skipped. + + co_positions = co_positions or iter(()) + for instr in instrs: + yield instr, next(co_positions, ()) + for _, size, _ in (instr.cache_info or ()): + for i in range(size): + next(co_positions, ()) + class BytecodeTestCase(unittest.TestCase): """Custom assertion methods for inspecting bytecode.""" @@ -53,16 +69,14 @@ class CompilationStepTestCase(unittest.TestCase): class Label: pass - def assertInstructionsMatch(self, actual_, expected_): - # get two lists where each entry is a label or - # an instruction tuple. Normalize the labels to the - # instruction count of the target, and compare the lists. - - self.assertIsInstance(actual_, list) - self.assertIsInstance(expected_, list) + def assertInstructionsMatch(self, actual_seq, expected): + # get an InstructionSequence and an expected list, where each + # entry is a label or an instruction tuple. Construct an expcted + # instruction sequence and compare with the one given. - actual = self.normalize_insts(actual_) - expected = self.normalize_insts(expected_) + self.assertIsInstance(expected, list) + actual = actual_seq.get_instructions() + expected = self.seq_from_insts(expected).get_instructions() self.assertEqual(len(actual), len(expected)) # compare instructions @@ -72,10 +86,8 @@ def assertInstructionsMatch(self, actual_, expected_): continue self.assertIsInstance(exp, tuple) self.assertIsInstance(act, tuple) - # crop comparison to the provided expected values - if len(act) > len(exp): - act = act[:len(exp)] - self.assertEqual(exp, act) + idx = max([p[0] for p in enumerate(exp) if p[1] != -1]) + self.assertEqual(exp[:idx], act[:idx]) def resolveAndRemoveLabels(self, insts): idx = 0 @@ -90,54 +102,57 @@ def resolveAndRemoveLabels(self, insts): return res - def normalize_insts(self, insts): - """ Map labels to instruction index. - Map opcodes to opnames. - """ - insts = self.resolveAndRemoveLabels(insts) - res = [] - for item in insts: - assert isinstance(item, tuple) - opcode, oparg, *loc = item - opcode = dis.opmap.get(opcode, opcode) - if isinstance(oparg, self.Label): - arg = oparg.value - else: - arg = oparg if opcode in self.HAS_ARG else None - opcode = dis.opname[opcode] - res.append((opcode, arg, *loc)) - return res + def seq_from_insts(self, insts): + labels = {item for item in insts if isinstance(item, self.Label)} + for i, lbl in enumerate(labels): + lbl.value = i - def complete_insts_info(self, insts): - # fill in omitted fields in location, and oparg 0 for ops with no arg. - res = [] + seq = _testinternalcapi.new_instruction_sequence() for item in insts: - assert isinstance(item, tuple) - inst = list(item) - opcode = dis.opmap[inst[0]] - oparg = inst[1] - loc = inst[2:] + [-1] * (6 - len(inst)) - res.append((opcode, oparg, *loc)) - return res + if isinstance(item, self.Label): + seq.use_label(item.value) + else: + op = item[0] + if isinstance(op, str): + op = opcode.opmap[op] + arg, *loc = item[1:] + if isinstance(arg, self.Label): + arg = arg.value + loc = loc + [-1] * (4 - len(loc)) + seq.addop(op, arg or 0, *loc) + return seq + + def check_instructions(self, insts): + for inst in insts: + if isinstance(inst, self.Label): + continue + op, arg, *loc = inst + if isinstance(op, str): + op = opcode.opmap[op] + self.assertEqual(op in opcode.hasarg, + arg is not None, + f"{opcode.opname[op]=} {arg=}") + self.assertTrue(all(isinstance(l, int) for l in loc)) +@unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") class CodegenTestCase(CompilationStepTestCase): def generate_code(self, ast): - insts, _ = compiler_codegen(ast, "my_file.py", 0) + insts, _ = _testinternalcapi.compiler_codegen(ast, "my_file.py", 0) return insts +@unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") class CfgOptimizationTestCase(CompilationStepTestCase): - def get_optimized(self, insts, consts, nlocals=0): - insts = self.normalize_insts(insts) - insts = self.complete_insts_info(insts) - insts = optimize_cfg(insts, consts, nlocals) + def get_optimized(self, seq, consts, nlocals=0): + insts = _testinternalcapi.optimize_cfg(seq, consts, nlocals) return insts, consts +@unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") class AssemblerTestCase(CompilationStepTestCase): def get_code_object(self, filename, insts, metadata): - co = assemble_code_object(filename, insts, metadata) + co = _testinternalcapi.assemble_code_object(filename, insts, metadata) return co diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py new file mode 100644 index 0000000000..b1e38b43dc --- /dev/null +++ b/Lib/test/test__opcode.py @@ -0,0 +1,143 @@ +import dis +from test.support.import_helper import import_module +import unittest +import opcode + +_opcode = import_module("_opcode") +from _opcode import stack_effect + + +class OpListTests(unittest.TestCase): + def check_bool_function_result(self, func, ops, expected): + for op in ops: + if isinstance(op, str): + op = dis.opmap[op] + with self.subTest(opcode=op, func=func): + self.assertIsInstance(func(op), bool) + self.assertEqual(func(op), expected) + + def test_invalid_opcodes(self): + invalid = [-100, -1, 255, 512, 513, 1000] + self.check_bool_function_result(_opcode.is_valid, invalid, False) + self.check_bool_function_result(_opcode.has_arg, invalid, False) + self.check_bool_function_result(_opcode.has_const, invalid, False) + self.check_bool_function_result(_opcode.has_name, invalid, False) + self.check_bool_function_result(_opcode.has_jump, invalid, False) + self.check_bool_function_result(_opcode.has_free, invalid, False) + self.check_bool_function_result(_opcode.has_local, invalid, False) + self.check_bool_function_result(_opcode.has_exc, invalid, False) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'dis' has no attribute 'opmap' + def test_is_valid(self): + names = [ + 'CACHE', + 'POP_TOP', + 'IMPORT_NAME', + 'JUMP', + 'INSTRUMENTED_RETURN_VALUE', + ] + opcodes = [dis.opmap[opname] for opname in names] + self.check_bool_function_result(_opcode.is_valid, opcodes, True) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'dis' has no attribute 'hasarg' + def test_oplists(self): + def check_function(self, func, expected): + for op in [-10, 520]: + with self.subTest(opcode=op, func=func): + res = func(op) + self.assertIsInstance(res, bool) + self.assertEqual(res, op in expected) + + check_function(self, _opcode.has_arg, dis.hasarg) + check_function(self, _opcode.has_const, dis.hasconst) + check_function(self, _opcode.has_name, dis.hasname) + check_function(self, _opcode.has_jump, dis.hasjump) + check_function(self, _opcode.has_free, dis.hasfree) + check_function(self, _opcode.has_local, dis.haslocal) + check_function(self, _opcode.has_exc, dis.hasexc) + + +class StackEffectTests(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_stack_effect(self): + self.assertEqual(stack_effect(dis.opmap['POP_TOP']), -1) + self.assertEqual(stack_effect(dis.opmap['BUILD_SLICE'], 0), -1) + self.assertEqual(stack_effect(dis.opmap['BUILD_SLICE'], 1), -1) + self.assertEqual(stack_effect(dis.opmap['BUILD_SLICE'], 3), -2) + self.assertRaises(ValueError, stack_effect, 30000) + # All defined opcodes + has_arg = dis.hasarg + for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()): + if code >= opcode.MIN_INSTRUMENTED_OPCODE: + continue + with self.subTest(opname=name): + stack_effect(code) + stack_effect(code, 0) + # All not defined opcodes + for code in set(range(256)) - set(dis.opmap.values()): + with self.subTest(opcode=code): + self.assertRaises(ValueError, stack_effect, code) + self.assertRaises(ValueError, stack_effect, code, 0) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_stack_effect_jump(self): + FOR_ITER = dis.opmap['FOR_ITER'] + self.assertEqual(stack_effect(FOR_ITER, 0), 1) + self.assertEqual(stack_effect(FOR_ITER, 0, jump=True), 1) + self.assertEqual(stack_effect(FOR_ITER, 0, jump=False), 1) + JUMP_FORWARD = dis.opmap['JUMP_FORWARD'] + self.assertEqual(stack_effect(JUMP_FORWARD, 0), 0) + self.assertEqual(stack_effect(JUMP_FORWARD, 0, jump=True), 0) + self.assertEqual(stack_effect(JUMP_FORWARD, 0, jump=False), 0) + # All defined opcodes + has_arg = dis.hasarg + has_exc = dis.hasexc + has_jump = dis.hasjabs + dis.hasjrel + for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()): + if code >= opcode.MIN_INSTRUMENTED_OPCODE: + continue + with self.subTest(opname=name): + if code not in has_arg: + common = stack_effect(code) + jump = stack_effect(code, jump=True) + nojump = stack_effect(code, jump=False) + else: + common = stack_effect(code, 0) + jump = stack_effect(code, 0, jump=True) + nojump = stack_effect(code, 0, jump=False) + if code in has_jump or code in has_exc: + self.assertEqual(common, max(jump, nojump)) + else: + self.assertEqual(jump, common) + self.assertEqual(nojump, common) + + +class SpecializationStatsTests(unittest.TestCase): + def test_specialization_stats(self): + stat_names = ["success", "failure", "hit", "deferred", "miss", "deopt"] + specialized_opcodes = [ + op.lower() + for op in opcode._specializations + if opcode._inline_cache_entries.get(op, 0) + ] + self.assertIn('load_attr', specialized_opcodes) + self.assertIn('binary_subscr', specialized_opcodes) + + stats = _opcode.get_specialization_stats() + if stats is not None: + self.assertIsInstance(stats, dict) + self.assertCountEqual(stats.keys(), specialized_opcodes) + self.assertCountEqual( + stats['load_attr'].keys(), + stat_names + ['failure_kinds']) + for sn in stat_names: + self.assertIsInstance(stats['load_attr'][sn], int) + self.assertIsInstance( + stats['load_attr']['failure_kinds'], + tuple) + for v in stats['load_attr']['failure_kinds']: + self.assertIsInstance(v, int) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 10319e36fa..09d9444d5d 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -72,8 +72,7 @@ def test_AST_objects(self): # "ast.AST constructor takes 0 positional arguments" ast.AST(2) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_AST_fields_NULL_check(self): # See: https://github.com/python/cpython/issues/126105 old_value = ast.AST._fields @@ -91,8 +90,7 @@ def cleanup(): with self.assertRaisesRegex(AttributeError, msg): ast.AST() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_AST_garbage_collection(self): class X: pass @@ -105,8 +103,7 @@ class X: support.gc_collect() self.assertIsNone(ref()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_snippets(self): for input, output, kind in ( (exec_tests, exec_results, "exec"), @@ -121,8 +118,7 @@ def test_snippets(self): with self.subTest(action="compiling", input=i, kind=kind): compile(ast_tree, "?", kind) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_ast_validation(self): # compile() is the only function that calls PyAST_Validate snippets_to_validate = exec_tests + single_tests + eval_tests @@ -130,8 +126,7 @@ def test_ast_validation(self): tree = ast.parse(snippet) compile(tree, "", "exec") - # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags def test_optimization_levels__debug__(self): cases = [(-1, "__debug__"), (0, "__debug__"), (1, False), (2, False)] for optval, expected in cases: @@ -147,8 +142,7 @@ def test_optimization_levels__debug__(self): self.assertIsInstance(res.body[0].value, ast.Name) self.assertEqual(res.body[0].value.id, expected) - # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags def test_optimization_levels_const_folding(self): folded = ("Expr", (1, 0, 1, 5), ("Constant", (1, 0, 1, 5), 3, None)) not_folded = ( @@ -172,8 +166,7 @@ def test_optimization_levels_const_folding(self): res = to_tuple(tree.body[0]) self.assertEqual(res, expected) - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_invalid_position_information(self): invalid_linenos = [(10, 1), (-10, -11), (10, -11), (-5, -2), (-5, 1)] @@ -198,8 +191,7 @@ def test_invalid_position_information(self): with self.assertRaises(ValueError): compile(tree, "", "exec") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_compilation_of_ast_nodes_with_default_end_position_values(self): tree = ast.Module( body=[ @@ -220,8 +212,7 @@ def test_compilation_of_ast_nodes_with_default_end_position_values(self): # Check that compilation doesn't crash. Note: this may crash explicitly only on debug mode. compile(tree, "", "exec") - # TODO: RUSTPYTHON; TypeError: required field "end_lineno" missing from alias - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: required field "end_lineno" missing from alias def test_negative_locations_for_compile(self): # See https://github.com/python/cpython/issues/130775 alias = ast.alias(name='traceback', lineno=0, col_offset=0) @@ -328,8 +319,7 @@ def test_field_attr_existence_deprecated(self): if isinstance(x, ast.AST): self.assertIs(type(x._fields), tuple) - # TODO: RUSTPYTHON; type object 'Module' has no attribute '__annotations__' - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; type object 'Module' has no attribute '__annotations__' def test_field_attr_existence(self): for name, item in ast.__dict__.items(): # These emit DeprecationWarnings @@ -356,8 +346,7 @@ def _construct_ast_class(self, cls): kwargs[name] = self._construct_ast_class(typ) return cls(**kwargs) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_arguments(self): x = ast.arguments() self.assertEqual( @@ -406,8 +395,7 @@ def test_field_attr_writable(self): x._fields = 666 self.assertEqual(x._fields, 666) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_classattrs_deprecated(self): with warnings.catch_warnings(): warnings.filterwarnings("ignore", "", DeprecationWarning) @@ -499,8 +487,7 @@ def test_classattrs_deprecated(self): ], ) - # TODO: RUSTPYTHON; DeprecationWarning not triggered - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered def test_classattrs(self): with self.assertWarns(DeprecationWarning): x = ast.Constant() @@ -706,8 +693,7 @@ class S(str): with assertNumDeprecated(): self.assertNotIsInstance(Constant(S("42")), Num) - # TODO: RUSTPYTHON; will be removed in Python 3.14 - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; will be removed in Python 3.14 def test_constant_subclasses_deprecated(self): with warnings.catch_warnings(): warnings.filterwarnings("ignore", "", DeprecationWarning) @@ -774,8 +760,7 @@ def test_module(self): x = ast.Module(body, []) self.assertEqual(x.body, body) - # TODO: RUSTPYTHON; DeprecationWarning not triggered - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered def test_nodeclasses(self): # Zero arguments constructor explicitly allowed (but deprecated) with self.assertWarns(DeprecationWarning): @@ -827,8 +812,7 @@ def test_no_fields(self): x = ast.Sub() self.assertEqual(x._fields, ()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid_sum(self): pos = dict(lineno=2, col_offset=3) m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], []) @@ -836,8 +820,7 @@ def test_invalid_sum(self): compile(m, "", "exec") self.assertIn("but got ", "eval") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_empty_yield_from(self): # Issue 16546: yield from value is not optional. empty_yield_from = ast.parse("def f():\n yield from g()") @@ -910,8 +892,7 @@ def test_issue39579_dotted_name_end_col_offset(self): attr_b = tree.body[0].decorator_list[0].value self.assertEqual(attr_b.end_col_offset, 4) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_ast_asdl_signature(self): self.assertEqual( ast.withitem.__doc__, "withitem(expr context_expr, expr? optional_vars)" @@ -926,8 +907,7 @@ def test_ast_asdl_signature(self): expressions[0] = f"expr = {ast.expr.__subclasses__()[0].__doc__}" self.assertCountEqual(ast.expr.__doc__.split("\n"), expressions) - # TODO: RUSTPYTHON; SyntaxError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised def test_positional_only_feature_version(self): ast.parse("def foo(x, /): ...", feature_version=(3, 8)) ast.parse("def bar(x=1, /): ...", feature_version=(3, 8)) @@ -943,8 +923,7 @@ def test_positional_only_feature_version(self): with self.assertRaises(SyntaxError): ast.parse("lambda x=1, /: ...", feature_version=(3, 7)) - # TODO: RUSTPYTHON; SyntaxError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised def test_assignment_expression_feature_version(self): ast.parse("(x := 0)", feature_version=(3, 8)) with self.assertRaises(SyntaxError): @@ -954,8 +933,7 @@ def test_conditional_context_managers_parse_with_low_feature_version(self): # regression test for gh-115881 ast.parse("with (x() if y else z()): ...", feature_version=(3, 8)) - # TODO: RUSTPYTHON; SyntaxError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised def test_exception_groups_feature_version(self): code = dedent(""" try: ... @@ -965,8 +943,7 @@ def test_exception_groups_feature_version(self): with self.assertRaises(SyntaxError): ast.parse(code, feature_version=(3, 10)) - # TODO: RUSTPYTHON; SyntaxError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised def test_type_params_feature_version(self): samples = [ "type X = int", @@ -979,8 +956,7 @@ def test_type_params_feature_version(self): with self.assertRaises(SyntaxError): ast.parse(sample, feature_version=(3, 11)) - # TODO: RUSTPYTHON; SyntaxError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised def test_type_params_default_feature_version(self): samples = [ "type X[*Ts=int] = int", @@ -999,8 +975,7 @@ def test_invalid_major_feature_version(self): with self.assertRaises(ValueError): ast.parse("pass", feature_version=(4, 0)) - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_constant_as_name(self): for constant in "True", "False", "None": expr = ast.Expression(ast.Name(constant, ast.Load())) @@ -1010,8 +985,7 @@ def test_constant_as_name(self): ): compile(expr, "", "eval") - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_constant_as_unicode_name(self): constants = [ ("True", b"Tru\xe1\xb5\x89"), @@ -1023,8 +997,7 @@ def test_constant_as_unicode_name(self): f"identifier field can't represent '{constant[0]}' constant"): ast.parse(constant[1], mode="eval") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_precedence_enum(self): class _Precedence(enum.IntEnum): """Precedence table that originated from python grammar.""" @@ -1101,8 +1074,7 @@ def assert_none_check(self, node: type[ast.AST], attr: str, source: str) -> None with self.assertRaisesRegex(ValueError, f"^{e}$"): compile(tree, "", "exec") - # TODO: RUSTPYTHON; TypeError: expected some sort of expr, but got None - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected some sort of expr, but got None def test_none_checks(self) -> None: tests = [ (ast.alias, "name", "import spam as SPAM"), @@ -1120,8 +1092,7 @@ def test_none_checks(self) -> None: class CopyTests(unittest.TestCase): """Test copying and pickling AST nodes.""" - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_pickling(self): import pickle @@ -1202,8 +1173,7 @@ def test_copy_with_parents(self): class ASTHelpers_Test(unittest.TestCase): maxDiff = None - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_parse(self): a = ast.parse("foo(1 + 1)") b = compile("foo(1 + 1)", "", "exec", ast.PyCF_ONLY_AST) @@ -1217,8 +1187,7 @@ def test_parse_in_error(self): ast.literal_eval(r"'\U'") self.assertIsNotNone(e.exception.__context__) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_dump(self): node = ast.parse('spam(eggs, "and cheese")') self.assertEqual( @@ -1242,8 +1211,7 @@ def test_dump(self): "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)])", ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_dump_indent(self): node = ast.parse('spam(eggs, "and cheese")') self.assertEqual( @@ -1308,8 +1276,7 @@ def test_dump_indent(self): end_col_offset=24)])""", ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_dump_incomplete(self): node = ast.Raise(lineno=3, col_offset=4) self.assertEqual(ast.dump(node), "Raise()") @@ -1377,8 +1344,7 @@ def test_dump_incomplete(self): "ClassDef('T', [], [keyword('a', Constant(None))], [], [Name('dataclass', Load())])", ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_dump_show_empty(self): def check_node(node, empty, full, **kwargs): with self.subTest(show_empty=False): @@ -1469,8 +1435,7 @@ def check_text(code, empty, full, **kwargs): full="Module(body=[Import(names=[alias(name='_ast', asname='ast')]), ImportFrom(module='module', names=[alias(name='sub')], level=0)], type_ignores=[])", ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_copy_location(self): src = ast.parse("1 + 1", mode="eval") src.body.right = ast.copy_location(ast.Constant(2), src.body.right) @@ -1491,8 +1456,7 @@ def test_copy_location(self): self.assertEqual(new.lineno, 1) self.assertEqual(new.col_offset, 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_fix_missing_locations(self): src = ast.parse('write("spam")') src.body.append( @@ -1514,8 +1478,7 @@ def test_fix_missing_locations(self): "end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)])", ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_increment_lineno(self): src = ast.parse("1 + 1", mode="eval") self.assertEqual(ast.increment_lineno(src, n=3), src) @@ -1542,8 +1505,7 @@ def test_increment_lineno(self): self.assertEqual(ast.increment_lineno(src).lineno, 2) self.assertIsNone(ast.increment_lineno(src).end_lineno) - # TODO: RUSTPYTHON; IndexError: index out of range - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; IndexError: index out of range def test_increment_lineno_on_module(self): src = ast.parse( dedent("""\ @@ -1565,8 +1527,7 @@ def test_iter_fields(self): self.assertEqual(d.pop("func").id, "foo") self.assertEqual(d, {"keywords": [], "args": []}) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_iter_child_nodes(self): node = ast.parse("spam(23, 42, eggs='leek')", mode="eval") self.assertEqual(len(list(ast.iter_child_nodes(node.body))), 4) @@ -1681,8 +1642,7 @@ def test_literal_eval(self): self.assertRaises(ValueError, ast.literal_eval, "+True") self.assertRaises(ValueError, ast.literal_eval, "2+3") - # TODO: RUSTPYTHON; SyntaxError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised def test_literal_eval_str_int_limit(self): with support.adjust_int_max_str_digits(4000): ast.literal_eval("3" * 4000) # no error @@ -1722,8 +1682,7 @@ def test_literal_eval_malformed_dict_nodes(self): ) self.assertRaises(ValueError, ast.literal_eval, malformed) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_literal_eval_trailing_ws(self): self.assertEqual(ast.literal_eval(" -1"), -1) self.assertEqual(ast.literal_eval("\t\t-1"), -1) @@ -1741,8 +1700,7 @@ def test_literal_eval_malformed_lineno(self): with self.assertRaisesRegex(ValueError, msg): ast.literal_eval(node) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_literal_eval_syntax_errors(self): with self.assertRaisesRegex(SyntaxError, "unexpected indent"): ast.literal_eval(r""" @@ -1750,8 +1708,7 @@ def test_literal_eval_syntax_errors(self): (\ \ """) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bad_integer(self): # issue13436: Bad error message with invalid numeric values body = [ @@ -1768,8 +1725,7 @@ def test_bad_integer(self): compile(mod, "test", "exec") self.assertIn("invalid integer value: None", str(cm.exception)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_level_as_none(self): body = [ ast.ImportFrom( @@ -1786,8 +1742,7 @@ def test_level_as_none(self): exec(code, ns) self.assertIn("sleep", ns) - # TODO: RUSTPYTHON - @unittest.skip("TODO: RUSTPYTHON; crash") + @unittest.skip('TODO: RUSTPYTHON; crash') def test_recursion_direct(self): e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) e.operand = e @@ -1795,8 +1750,7 @@ def test_recursion_direct(self): with support.infinite_recursion(): compile(ast.Expression(e), "", "eval") - # TODO: RUSTPYTHON - @unittest.skip("TODO: RUSTPYTHON; crash") + @unittest.skip('TODO: RUSTPYTHON; crash') def test_recursion_indirect(self): e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) f = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) @@ -1826,8 +1780,7 @@ def stmt(self, stmt, msg=None): mod = ast.Module([stmt], []) self.mod(mod, msg) - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_module(self): m = ast.Interactive([ast.Expr(ast.Name("x", ast.Store()))]) self.mod(m, "must have Load context", "single") @@ -1884,8 +1837,7 @@ def arguments( "must have Load context", ) - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_funcdef(self): a = ast.arguments([], [], None, [], [], None, []) f = ast.FunctionDef("x", a, [], [], None, None, []) @@ -1906,8 +1858,7 @@ def fac(args): self._check_arguments(fac, self.stmt) - # TODO: RUSTPYTHON; called `Result::unwrap()` on an `Err` value: StackUnderflow - ''' + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: class pattern defines no positional sub-patterns (__match_args__ missing) def test_funcdef_pattern_matching(self): # gh-104799: New fields on FunctionDef should be added at the end def matcher(node): @@ -1932,10 +1883,8 @@ def foo(bar) -> pacarana: funcdef = source.body[0] self.assertIsInstance(funcdef, ast.FunctionDef) self.assertTrue(matcher(funcdef)) - ''' - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_classdef(self): def cls( bases=None, keywords=None, body=None, decorator_list=None, type_params=None @@ -1965,15 +1914,13 @@ def cls( cls(decorator_list=[ast.Name("x", ast.Store())]), "must have Load context" ) - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_delete(self): self.stmt(ast.Delete([]), "empty targets on Delete") self.stmt(ast.Delete([None]), "None disallowed") self.stmt(ast.Delete([ast.Name("x", ast.Load())]), "must have Del context") - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_assign(self): self.stmt(ast.Assign([], ast.Constant(3)), "empty targets on Assign") self.stmt(ast.Assign([None], ast.Constant(3)), "None disallowed") @@ -1986,8 +1933,7 @@ def test_assign(self): "must have Load context", ) - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_augassign(self): aug = ast.AugAssign( ast.Name("x", ast.Load()), ast.Add(), ast.Name("y", ast.Load()) @@ -1998,8 +1944,7 @@ def test_augassign(self): ) self.stmt(aug, "must have Load context") - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_for(self): x = ast.Name("x", ast.Store()) y = ast.Name("y", ast.Load()) @@ -2015,8 +1960,7 @@ def test_for(self): self.stmt(ast.For(x, y, [e], []), "must have Load context") self.stmt(ast.For(x, y, [p], [e]), "must have Load context") - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_while(self): self.stmt(ast.While(ast.Constant(3), [], []), "empty body on While") self.stmt( @@ -2030,8 +1974,7 @@ def test_while(self): "must have Load context", ) - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_if(self): self.stmt(ast.If(ast.Constant(3), [], []), "empty body on If") i = ast.If(ast.Name("x", ast.Store()), [ast.Pass()], []) @@ -2043,8 +1986,7 @@ def test_if(self): ) self.stmt(i, "must have Load context") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_with(self): p = ast.Pass() self.stmt(ast.With([], [p]), "empty items on With") @@ -2055,8 +1997,7 @@ def test_with(self): i = ast.withitem(ast.Constant(3), ast.Name("x", ast.Load())) self.stmt(ast.With([i], [p]), "must have Store context") - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_raise(self): r = ast.Raise(None, ast.Constant(3)) self.stmt(r, "Raise with cause but no exception") @@ -2065,8 +2006,7 @@ def test_raise(self): r = ast.Raise(ast.Constant(4), ast.Name("x", ast.Store())) self.stmt(r, "must have Load context") - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_try(self): p = ast.Pass() t = ast.Try([], [], [], [p]) @@ -2087,8 +2027,7 @@ def test_try(self): t = ast.Try([p], e, [p], [ast.Expr(ast.Name("x", ast.Store()))]) self.stmt(t, "must have Load context") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_try_star(self): p = ast.Pass() t = ast.TryStar([], [], [], [p]) @@ -2109,8 +2048,7 @@ def test_try_star(self): t = ast.TryStar([p], e, [p], [ast.Expr(ast.Name("x", ast.Store()))]) self.stmt(t, "must have Load context") - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_assert(self): self.stmt( ast.Assert(ast.Name("x", ast.Store()), None), "must have Load context" @@ -2118,36 +2056,30 @@ def test_assert(self): assrt = ast.Assert(ast.Name("x", ast.Load()), ast.Name("y", ast.Store())) self.stmt(assrt, "must have Load context") - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_import(self): self.stmt(ast.Import([]), "empty names on Import") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_importfrom(self): imp = ast.ImportFrom(None, [ast.alias("x", None)], -42) self.stmt(imp, "Negative ImportFrom level") self.stmt(ast.ImportFrom(None, [], 0), "empty names on ImportFrom") - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_global(self): self.stmt(ast.Global([]), "empty names on Global") - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_nonlocal(self): self.stmt(ast.Nonlocal([]), "empty names on Nonlocal") - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_expr(self): e = ast.Expr(ast.Name("x", ast.Store())) self.stmt(e, "must have Load context") - # TODO: RUSTPYTHON - @unittest.skip("TODO: RUSTPYTHON; called `Option::unwrap()` on a `None` value") + @unittest.skip('TODO: RUSTPYTHON; called `Option::unwrap()` on a `None` value') def test_boolop(self): b = ast.BoolOp(ast.And(), []) self.expr(b, "less than 2 values") @@ -2158,14 +2090,12 @@ def test_boolop(self): b = ast.BoolOp(ast.And(), [ast.Constant(4), ast.Name("x", ast.Store())]) self.expr(b, "must have Load context") - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_unaryop(self): u = ast.UnaryOp(ast.Not(), ast.Name("x", ast.Store())) self.expr(u, "must have Load context") - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_lambda(self): a = ast.arguments([], [], None, [], [], None, []) self.expr(ast.Lambda(a, ast.Name("x", ast.Store())), "must have Load context") @@ -2175,24 +2105,21 @@ def fac(args): self._check_arguments(fac, self.expr) - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_ifexp(self): l = ast.Name("x", ast.Load()) s = ast.Name("y", ast.Store()) for args in (s, l, l), (l, s, l), (l, l, s): self.expr(ast.IfExp(*args), "must have Load context") - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_dict(self): d = ast.Dict([], [ast.Name("x", ast.Load())]) self.expr(d, "same number of keys as values") d = ast.Dict([ast.Name("x", ast.Load())], [None]) self.expr(d, "None disallowed") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_set(self): self.expr(ast.Set([None]), "None disallowed") s = ast.Set([ast.Name("x", ast.Store())]) @@ -2226,23 +2153,19 @@ def wrap(gens): self._check_comprehension(wrap) - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_listcomp(self): self._simple_comp(ast.ListComp) - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_setcomp(self): self._simple_comp(ast.SetComp) - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_generatorexp(self): self._simple_comp(ast.GeneratorExp) - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_dictcomp(self): g = ast.comprehension( ast.Name("y", ast.Store()), ast.Name("p", ast.Load()), [], 0 @@ -2259,13 +2182,11 @@ def factory(comps): self._check_comprehension(factory) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_yield(self): self.expr(ast.Yield(ast.Name("x", ast.Store())), "must have Load") self.expr(ast.YieldFrom(ast.Name("x", ast.Store())), "must have Load") - # TODO: RUSTPYTHON @unittest.skip("TODO: RUSTPYTHON; thread 'main' panicked") def test_compare(self): left = ast.Name("x", ast.Load()) @@ -2278,8 +2199,7 @@ def test_compare(self): comp = ast.Compare(left, [ast.In()], [ast.Constant("blah")]) self.expr(comp) - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_call(self): func = ast.Name("x", ast.Load()) args = [ast.Name("y", ast.Load())] @@ -2325,14 +2245,12 @@ class subcomplex(complex): ], ) - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_attribute(self): attr = ast.Attribute(ast.Name("x", ast.Store()), "y", ast.Load()) self.expr(attr, "must have Load context") - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_subscript(self): sub = ast.Subscript(ast.Name("x", ast.Store()), ast.Constant(3), ast.Load()) self.expr(sub, "must have Load context") @@ -2348,8 +2266,7 @@ def test_subscript(self): sl = ast.Tuple([s], ast.Load()) self.expr(ast.Subscript(x, sl, ast.Load()), "must have Load context") - # TODO: RUSTPYTHON; ValueError not raised - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_starred(self): left = ast.List( [ast.Starred(ast.Name("x", ast.Load()), ast.Store())], ast.Store() @@ -2363,13 +2280,11 @@ def _sequence(self, fac): fac([ast.Name("x", ast.Store())], ast.Load()), "must have Load context" ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_list(self): self._sequence(ast.List) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_tuple(self): self._sequence(ast.Tuple) @@ -2389,8 +2304,7 @@ def test_nameconstant(self): ], ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_resource("cpu") def test_stdlib_validates(self): stdlib = os.path.dirname(ast.__file__) @@ -2486,7 +2400,6 @@ def test_stdlib_validates(self): ast.MatchMapping([], [], rest="_"), ] - # TODO: RUSTPYTHON @unittest.skip("TODO: RUSTPYTHON; thread 'main' panicked") def test_match_validation_pattern(self): name_x = ast.Name("x", ast.Load()) @@ -2524,16 +2437,14 @@ def test_validation(self): self.compile_constant([1, 2, 3]) self.assertEqual(str(cm.exception), "got an invalid type in Constant: list") - # TODO: RUSTPYTHON; b'' is not b'' - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; b'' is not b'' def test_singletons(self): for const in (None, False, True, Ellipsis, b"", frozenset()): with self.subTest(const=const): value = self.compile_constant(const) self.assertIs(value, const) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_values(self): nested_tuple = (1,) nested_frozenset = frozenset({1}) @@ -2556,8 +2467,7 @@ def test_values(self): result = self.compile_constant(value) self.assertEqual(result, value) - # TODO: RUSTPYTHON; SyntaxError: cannot assign to literal - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError: cannot assign to literal def test_assign_to_constant(self): tree = ast.parse("x = 1") @@ -3079,8 +2989,7 @@ def assertASTTransformation(self, tranformer_class, initial_code, expected_code) self.assertASTEqual(result_ast, expected_ast) - # TODO: RUSTPYTHON; is not - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; is not def test_node_remove_single(self): code = "def func(arg) -> SomeType: ..." expected = "def func(arg): ..." @@ -3118,8 +3027,7 @@ def visit_Expr(self, node: ast.Expr): self.assertASTTransformation(YieldRemover, code, expected) - # TODO: RUSTPYTHON; is not - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; is not def test_node_return_list(self): code = """ class DSL(Base, kw1=True): ... @@ -3160,8 +3068,7 @@ def visit_Call(self, node: ast.Call): self.assertASTTransformation(PrintToLog, code, expected) - # TODO: RUSTPYTHON; is not - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; is not def test_node_replace(self): code = """ def func(arg): @@ -3193,8 +3100,7 @@ def visit_Call(self, node: ast.Call): class ASTConstructorTests(unittest.TestCase): """Test the autogenerated constructors for AST nodes.""" - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_FunctionDef(self): args = ast.arguments() self.assertEqual(args.args, []) @@ -3210,8 +3116,7 @@ def test_FunctionDef(self): self.assertEqual(node.name, "foo") self.assertEqual(node.decorator_list, []) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_expr_context(self): name = ast.Name("x") self.assertEqual(name.id, "x") @@ -3260,8 +3165,7 @@ class FieldsAndTypes(ast.AST): obj = FieldsAndTypes(a=1) self.assertEqual(obj.a, 1) - # TODO: RUSTPYTHON; DeprecationWarning not triggered - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered def test_custom_attributes(self): class MyAttrs(ast.AST): _attributes = ("a", "b") @@ -3276,8 +3180,7 @@ class MyAttrs(ast.AST): ): obj = MyAttrs(c=3) - # TODO: RUSTPYTHON; DeprecationWarning not triggered - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered def test_fields_and_types_no_default(self): class FieldsAndTypesNoDefault(ast.AST): _fields = ("a",) @@ -3293,8 +3196,7 @@ class FieldsAndTypesNoDefault(ast.AST): obj = FieldsAndTypesNoDefault(a=1) self.assertEqual(obj.a, 1) - # TODO: RUSTPYTHON; DeprecationWarning not triggered - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered def test_incomplete_field_types(self): class MoreFieldsThanTypes(ast.AST): _fields = ("a", "b") @@ -3314,8 +3216,7 @@ class MoreFieldsThanTypes(ast.AST): self.assertEqual(obj.a, 1) self.assertEqual(obj.b, 2) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_complete_field_types(self): class _AllFieldTypes(ast.AST): _fields = ("a", "b") @@ -3416,8 +3317,7 @@ def test_subinterpreter(self): class ASTMainTests(unittest.TestCase): # Tests `ast.main()` function. - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cli_file_input(self): code = "print(1, 2, 3)" expected = ast.dump(ast.parse(code), indent=3) @@ -3435,7 +3335,7 @@ def test_cli_file_input(self): def compare(left, right): return ast.dump(left) == ast.dump(right) -class ASTOptimiziationTests(unittest.TestCase): +class ASTOptimizationTests(unittest.TestCase): binop = { "+": ast.Add(), "-": ast.Sub(), @@ -3492,8 +3392,7 @@ def assert_ast(self, code, non_optimized_target, optimized_target): def create_binop(self, operand, left=ast.Constant(1), right=ast.Constant(1)): return ast.BinOp(left=left, op=self.binop[operand], right=right) - # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags def test_folding_binop(self): code = "1 %s 1" operators = self.binop.keys() @@ -3517,8 +3416,7 @@ def test_folding_binop(self): self.assert_ast(code, non_optimized_target, optimized_target) - # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags def test_folding_unaryop(self): code = "%s1" operators = self.unaryop.keys() @@ -3538,8 +3436,7 @@ def create_unaryop(operand): ): self.assert_ast(result_code, non_optimized_target, optimized_target) - # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags def test_folding_not(self): code = "not (1 %s (1,))" operators = { @@ -3572,8 +3469,7 @@ def create_notop(operand): ): self.assert_ast(result_code, non_optimized_target, optimized_target) - # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags def test_folding_format(self): code = "'%s' % (a,)" @@ -3594,8 +3490,7 @@ def test_folding_format(self): self.assert_ast(code, non_optimized_target, optimized_target) - # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags def test_folding_tuple(self): code = "(1,)" @@ -3604,8 +3499,7 @@ def test_folding_tuple(self): self.assert_ast(code, non_optimized_target, optimized_target) - # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags def test_folding_comparator(self): code = "1 %s %s1%s" operators = [("in", ast.In()), ("not in", ast.NotIn())] @@ -3625,8 +3519,7 @@ def test_folding_comparator(self): )) self.assert_ast(code % (op, left, right), non_optimized_target, optimized_target) - # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags def test_folding_iter(self): code = "for _ in %s1%s: pass" braces = [ @@ -3648,8 +3541,7 @@ def test_folding_iter(self): self.assert_ast(code % (left, right), non_optimized_target, optimized_target) - # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags def test_folding_subscript(self): code = "(1,)[0]" @@ -3660,8 +3552,7 @@ def test_folding_subscript(self): self.assert_ast(code, non_optimized_target, optimized_target) - # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags def test_folding_type_param_in_function_def(self): code = "def foo[%s = 1 + 1](): pass" @@ -3692,8 +3583,7 @@ def test_folding_type_param_in_function_def(self): ) self.assert_ast(result_code, non_optimized_target, optimized_target) - # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags def test_folding_type_param_in_class_def(self): code = "class foo[%s = 1 + 1]: pass" @@ -3722,8 +3612,7 @@ def test_folding_type_param_in_class_def(self): ) self.assert_ast(result_code, non_optimized_target, optimized_target) - # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags def test_folding_type_param_in_type_alias(self): code = "type foo[%s = 1 + 1] = 1" diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 3cb9659acb..86ba0aa4b6 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -1,14 +1,29 @@ -import datetime import unittest -from test.support import cpython_only +from test.support import (cpython_only, is_wasi, requires_limited_api, Py_DEBUG, + set_recursion_limit, skip_on_s390x, import_helper) try: import _testcapi except ImportError: _testcapi = None +try: + import _testlimitedcapi +except ImportError: + _testlimitedcapi = None import struct import collections import itertools import gc +import contextlib +import sys +import types + + +class BadStr(str): + def __eq__(self, other): + return True + def __hash__(self): + # Guaranteed different hash + return str.__hash__(self) ^ 3 class FunctionCalls(unittest.TestCase): @@ -26,124 +41,22 @@ def fn(**kw): self.assertIsInstance(res, dict) self.assertEqual(list(res.items()), expected) - -# The test cases here cover several paths through the function calling -# code. They depend on the METH_XXX flag that is used to define a C -# function, which can't be verified from Python. If the METH_XXX decl -# for a C function changes, these tests may not cover the right paths. - -class CFunctionCalls(unittest.TestCase): - - def test_varargs0(self): - self.assertRaises(TypeError, {}.__contains__) - - def test_varargs1(self): - {}.__contains__(0) - - def test_varargs2(self): - self.assertRaises(TypeError, {}.__contains__, 0, 1) - - def test_varargs0_ext(self): - try: - {}.__contains__(*()) - except TypeError: - pass - - def test_varargs1_ext(self): - {}.__contains__(*(0,)) - - def test_varargs2_ext(self): - try: - {}.__contains__(*(1, 2)) - except TypeError: - pass - else: - raise RuntimeError - - def test_varargs1_kw(self): - self.assertRaises(TypeError, {}.__contains__, x=2) - - def test_varargs2_kw(self): - self.assertRaises(TypeError, {}.__contains__, x=2, y=2) - - def test_oldargs0_0(self): - {}.keys() - - def test_oldargs0_1(self): - self.assertRaises(TypeError, {}.keys, 0) - - def test_oldargs0_2(self): - self.assertRaises(TypeError, {}.keys, 0, 1) - - def test_oldargs0_0_ext(self): - {}.keys(*()) - - def test_oldargs0_1_ext(self): - try: - {}.keys(*(0,)) - except TypeError: + def test_frames_are_popped_after_failed_calls(self): + # GH-93252: stuff blows up if we don't pop the new frame after + # recovering from failed calls: + def f(): pass - else: - raise RuntimeError - - def test_oldargs0_2_ext(self): - try: - {}.keys(*(1, 2)) - except TypeError: - pass - else: - raise RuntimeError - - def test_oldargs0_0_kw(self): - try: - {}.keys(x=2) - except TypeError: - pass - else: - raise RuntimeError - - def test_oldargs0_1_kw(self): - self.assertRaises(TypeError, {}.keys, x=2) - - def test_oldargs0_2_kw(self): - self.assertRaises(TypeError, {}.keys, x=2, y=2) - - def test_oldargs1_0(self): - self.assertRaises(TypeError, [].count) - - def test_oldargs1_1(self): - [].count(1) - - def test_oldargs1_2(self): - self.assertRaises(TypeError, [].count, 1, 2) - - def test_oldargs1_0_ext(self): - try: - [].count(*()) - except TypeError: - pass - else: - raise RuntimeError - - def test_oldargs1_1_ext(self): - [].count(*(1,)) - - def test_oldargs1_2_ext(self): - try: - [].count(*(1, 2)) - except TypeError: - pass - else: - raise RuntimeError - - def test_oldargs1_0_kw(self): - self.assertRaises(TypeError, [].count, x=2) - - def test_oldargs1_1_kw(self): - self.assertRaises(TypeError, [].count, {}, x=2) - - def test_oldargs1_2_kw(self): - self.assertRaises(TypeError, [].count, x=2, y=2) + class C: + def m(self): + pass + callables = [f, C.m, [].__len__] + for c in callables: + for _ in range(1000): + try: + c(None) + except TypeError: + pass + # BOOM! @cpython_only @@ -158,11 +71,12 @@ def test_varargs2(self): self.assertRaisesRegex(TypeError, msg, {}.__contains__, 0, 1) def test_varargs3(self): - msg = r"^from_bytes\(\) takes exactly 2 positional arguments \(3 given\)" + msg = r"^from_bytes\(\) takes at most 2 positional arguments \(3 given\)" self.assertRaisesRegex(TypeError, msg, int.from_bytes, b'a', 'little', False) def test_varargs1min(self): - msg = r"get expected at least 1 argument, got 0" + msg = (r"get\(\) takes at least 1 argument \(0 given\)|" + r"get expected at least 1 argument, got 0") self.assertRaisesRegex(TypeError, msg, {}.get) msg = r"expected 1 argument, got 0" @@ -173,11 +87,13 @@ def test_varargs2min(self): self.assertRaisesRegex(TypeError, msg, getattr) def test_varargs1max(self): - msg = r"input expected at most 1 argument, got 2" + msg = (r"input\(\) takes at most 1 argument \(2 given\)|" + r"input expected at most 1 argument, got 2") self.assertRaisesRegex(TypeError, msg, input, 1, 2) def test_varargs2max(self): - msg = r"get expected at most 2 arguments, got 3" + msg = (r"get\(\) takes at most 2 arguments \(3 given\)|" + r"get expected at most 2 arguments, got 3") self.assertRaisesRegex(TypeError, msg, {}.get, 1, 2, 3) def test_varargs1_kw(self): @@ -193,7 +109,7 @@ def test_varargs3_kw(self): self.assertRaisesRegex(TypeError, msg, bool, x=2) def test_varargs4_kw(self): - msg = r"^index\(\) takes no keyword arguments$" + msg = r"^(list[.])?index\(\) takes no keyword arguments$" self.assertRaisesRegex(TypeError, msg, [].index, x=2) def test_varargs5_kw(self): @@ -209,19 +125,19 @@ def test_varargs7_kw(self): self.assertRaisesRegex(TypeError, msg, next, x=2) def test_varargs8_kw(self): - msg = r"^pack\(\) takes no keyword arguments$" + msg = r"^_struct[.]pack\(\) takes no keyword arguments$" self.assertRaisesRegex(TypeError, msg, struct.pack, x=2) def test_varargs9_kw(self): - msg = r"^pack_into\(\) takes no keyword arguments$" + msg = r"^_struct[.]pack_into\(\) takes no keyword arguments$" self.assertRaisesRegex(TypeError, msg, struct.pack_into, x=2) def test_varargs10_kw(self): - msg = r"^index\(\) takes no keyword arguments$" + msg = r"^deque[.]index\(\) takes no keyword arguments$" self.assertRaisesRegex(TypeError, msg, collections.deque().index, x=2) def test_varargs11_kw(self): - msg = r"^pack\(\) takes no keyword arguments$" + msg = r"^Struct[.]pack\(\) takes no keyword arguments$" self.assertRaisesRegex(TypeError, msg, struct.Struct.pack, struct.Struct(""), x=2) def test_varargs12_kw(self): @@ -238,9 +154,9 @@ def test_varargs14_kw(self): itertools.product, 0, repeat=1, foo=2) def test_varargs15_kw(self): - msg = r"^ImportError\(\) takes at most 2 keyword arguments \(3 given\)$" + msg = r"^ImportError\(\) takes at most 3 keyword arguments \(4 given\)$" self.assertRaisesRegex(TypeError, msg, - ImportError, 0, name=1, path=2, foo=3) + ImportError, 0, name=1, path=2, name_from=3, foo=3) def test_varargs16_kw(self): msg = r"^min\(\) takes at most 2 keyword arguments \(3 given\)$" @@ -248,10 +164,22 @@ def test_varargs16_kw(self): min, 0, default=1, key=2, foo=3) def test_varargs17_kw(self): - msg = r"^print\(\) takes at most 4 keyword arguments \(5 given\)$" + msg = r"print\(\) got an unexpected keyword argument 'foo'$" self.assertRaisesRegex(TypeError, msg, print, 0, sep=1, end=2, file=3, flush=4, foo=5) + def test_varargs18_kw(self): + # _PyArg_UnpackKeywordsWithVararg() + msg = r"invalid keyword argument for print\(\)$" + with self.assertRaisesRegex(TypeError, msg): + print(0, 1, **{BadStr('foo'): ','}) + + def test_varargs19_kw(self): + # _PyArg_UnpackKeywords() + msg = r"invalid keyword argument for round\(\)$" + with self.assertRaisesRegex(TypeError, msg): + round(1.75, **{BadStr('foo'): 1}) + def test_oldargs0_1(self): msg = r"keys\(\) takes no arguments \(1 given\)" self.assertRaisesRegex(TypeError, msg, {}.keys, 0) @@ -288,6 +216,208 @@ def test_oldargs1_2_kw(self): msg = r"count\(\) takes no keyword arguments" self.assertRaisesRegex(TypeError, msg, [].count, x=2, y=2) + def test_object_not_callable(self): + msg = r"^'object' object is not callable$" + self.assertRaisesRegex(TypeError, msg, object()) + + def test_module_not_callable_no_suggestion_0(self): + msg = r"^'module' object is not callable$" + self.assertRaisesRegex(TypeError, msg, types.ModuleType("mod")) + + def test_module_not_callable_no_suggestion_1(self): + msg = r"^'module' object is not callable$" + mod = types.ModuleType("mod") + mod.mod = 42 + self.assertRaisesRegex(TypeError, msg, mod) + + def test_module_not_callable_no_suggestion_2(self): + msg = r"^'module' object is not callable$" + mod = types.ModuleType("mod") + del mod.__name__ + self.assertRaisesRegex(TypeError, msg, mod) + + def test_module_not_callable_no_suggestion_3(self): + msg = r"^'module' object is not callable$" + mod = types.ModuleType("mod") + mod.__name__ = 42 + self.assertRaisesRegex(TypeError, msg, mod) + + def test_module_not_callable_suggestion(self): + msg = r"^'module' object is not callable\. Did you mean: 'mod\.mod\(\.\.\.\)'\?$" + mod = types.ModuleType("mod") + mod.mod = lambda: ... + self.assertRaisesRegex(TypeError, msg, mod) + + +@unittest.skipIf(_testcapi is None, "requires _testcapi") +class TestCallingConventions(unittest.TestCase): + """Test calling using various C calling conventions (METH_*) from Python + + Subclasses test several kinds of functions (module-level, methods, + class methods static methods) using these attributes: + obj: the object that contains tested functions (as attributes) + expected_self: expected "self" argument to the C function + + The base class tests module-level functions. + """ + + def setUp(self): + self.obj = self.expected_self = _testcapi + + def test_varargs(self): + self.assertEqual( + self.obj.meth_varargs(1, 2, 3), + (self.expected_self, (1, 2, 3)), + ) + + def test_varargs_ext(self): + self.assertEqual( + self.obj.meth_varargs(*(1, 2, 3)), + (self.expected_self, (1, 2, 3)), + ) + + def test_varargs_error_kw(self): + msg = r"meth_varargs\(\) takes no keyword arguments" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_varargs(k=1), + ) + + def test_varargs_keywords(self): + self.assertEqual( + self.obj.meth_varargs_keywords(1, 2, a=3, b=4), + (self.expected_self, (1, 2), {'a': 3, 'b': 4}) + ) + + def test_varargs_keywords_ext(self): + self.assertEqual( + self.obj.meth_varargs_keywords(*[1, 2], **{'a': 3, 'b': 4}), + (self.expected_self, (1, 2), {'a': 3, 'b': 4}) + ) + + def test_o(self): + self.assertEqual(self.obj.meth_o(1), (self.expected_self, 1)) + + def test_o_ext(self): + self.assertEqual(self.obj.meth_o(*[1]), (self.expected_self, 1)) + + def test_o_error_no_arg(self): + msg = r"meth_o\(\) takes exactly one argument \(0 given\)" + self.assertRaisesRegex(TypeError, msg, self.obj.meth_o) + + def test_o_error_two_args(self): + msg = r"meth_o\(\) takes exactly one argument \(2 given\)" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_o(1, 2), + ) + + def test_o_error_ext(self): + msg = r"meth_o\(\) takes exactly one argument \(3 given\)" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_o(*(1, 2, 3)), + ) + + def test_o_error_kw(self): + msg = r"meth_o\(\) takes no keyword arguments" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_o(k=1), + ) + + def test_o_error_arg_kw(self): + msg = r"meth_o\(\) takes no keyword arguments" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_o(k=1), + ) + + def test_noargs(self): + self.assertEqual(self.obj.meth_noargs(), self.expected_self) + + def test_noargs_ext(self): + self.assertEqual(self.obj.meth_noargs(*[]), self.expected_self) + + def test_noargs_error_arg(self): + msg = r"meth_noargs\(\) takes no arguments \(1 given\)" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_noargs(1), + ) + + def test_noargs_error_arg2(self): + msg = r"meth_noargs\(\) takes no arguments \(2 given\)" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_noargs(1, 2), + ) + + def test_noargs_error_ext(self): + msg = r"meth_noargs\(\) takes no arguments \(3 given\)" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_noargs(*(1, 2, 3)), + ) + + def test_noargs_error_kw(self): + msg = r"meth_noargs\(\) takes no keyword arguments" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_noargs(k=1), + ) + + def test_fastcall(self): + self.assertEqual( + self.obj.meth_fastcall(1, 2, 3), + (self.expected_self, (1, 2, 3)), + ) + + def test_fastcall_ext(self): + self.assertEqual( + self.obj.meth_fastcall(*(1, 2, 3)), + (self.expected_self, (1, 2, 3)), + ) + + def test_fastcall_error_kw(self): + msg = r"meth_fastcall\(\) takes no keyword arguments" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_fastcall(k=1), + ) + + def test_fastcall_keywords(self): + self.assertEqual( + self.obj.meth_fastcall_keywords(1, 2, a=3, b=4), + (self.expected_self, (1, 2), {'a': 3, 'b': 4}) + ) + + def test_fastcall_keywords_ext(self): + self.assertEqual( + self.obj.meth_fastcall_keywords(*(1, 2), **{'a': 3, 'b': 4}), + (self.expected_self, (1, 2), {'a': 3, 'b': 4}) + ) + + +class TestCallingConventionsInstance(TestCallingConventions): + """Test calling instance methods using various calling conventions""" + + def setUp(self): + self.obj = self.expected_self = _testcapi.MethInstance() + + +class TestCallingConventionsClass(TestCallingConventions): + """Test calling class methods using various calling conventions""" + + def setUp(self): + self.obj = self.expected_self = _testcapi.MethClass + + +class TestCallingConventionsClassInstance(TestCallingConventions): + """Test calling class methods on instance""" + + def setUp(self): + self.obj = _testcapi.MethClass() + self.expected_self = _testcapi.MethClass + + +class TestCallingConventionsStatic(TestCallingConventions): + """Test calling static methods using various calling conventions""" + + def setUp(self): + self.obj = _testcapi.MethStatic() + self.expected_self = None + def pyfunc(arg1, arg2): return [arg1, arg2] @@ -315,14 +445,15 @@ def static_method(): PYTHON_INSTANCE = PythonClass() - -IGNORE_RESULT = object() +NULL_OR_EMPTY = object() -@cpython_only class FastCallTests(unittest.TestCase): + """Test calling using various callables from C + """ + # Test calls with positional arguments - CALLS_POSARGS = ( + CALLS_POSARGS = [ # (func, args: tuple, result) # Python function with 2 arguments @@ -341,31 +472,11 @@ class FastCallTests(unittest.TestCase): (PYTHON_INSTANCE.class_method, (), "classmethod"), (PYTHON_INSTANCE.static_method, (), "staticmethod"), - # C function: METH_NOARGS - (globals, (), IGNORE_RESULT), - - # C function: METH_O - (id, ("hello",), IGNORE_RESULT), - - # C function: METH_VARARGS - (dir, (1,), IGNORE_RESULT), - - # C function: METH_VARARGS | METH_KEYWORDS - (min, (5, 9), 5), - - # C function: METH_FASTCALL - (divmod, (1000, 33), (30, 10)), - - # C type static method: METH_FASTCALL | METH_CLASS - (int.from_bytes, (b'\x01\x00', 'little'), 1), - - # bpo-30524: Test that calling a C type static method with no argument - # doesn't crash (ignore the result): METH_FASTCALL | METH_CLASS - (datetime.datetime.now, (), IGNORE_RESULT), - ) + # C callables are added later + ] # Test calls with positional and keyword arguments - CALLS_KWARGS = ( + CALLS_KWARGS = [ # (func, args: tuple, kwargs: dict, result) # Python function with 2 arguments @@ -376,34 +487,57 @@ class FastCallTests(unittest.TestCase): (PYTHON_INSTANCE.method, (1,), {'arg2': 2}, [1, 2]), (PYTHON_INSTANCE.method, (), {'arg1': 1, 'arg2': 2}, [1, 2]), - # C function: METH_VARARGS | METH_KEYWORDS - (max, ([],), {'default': 9}, 9), - - # C type static method: METH_FASTCALL | METH_CLASS - (int.from_bytes, (b'\x01\x00',), {'byteorder': 'little'}, 1), - (int.from_bytes, (), {'bytes': b'\x01\x00', 'byteorder': 'little'}, 1), - ) + # C callables are added later + ] + + # Add all the calling conventions and variants of C callables + if _testcapi: + _instance = _testcapi.MethInstance() + for obj, expected_self in ( + (_testcapi, _testcapi), # module-level function + (_instance, _instance), # bound method + (_testcapi.MethClass, _testcapi.MethClass), # class method on class + (_testcapi.MethClass(), _testcapi.MethClass), # class method on inst. + (_testcapi.MethStatic, None), # static method + ): + CALLS_POSARGS.extend([ + (obj.meth_varargs, (1, 2), (expected_self, (1, 2))), + (obj.meth_varargs_keywords, + (1, 2), (expected_self, (1, 2), NULL_OR_EMPTY)), + (obj.meth_fastcall, (1, 2), (expected_self, (1, 2))), + (obj.meth_fastcall, (), (expected_self, ())), + (obj.meth_fastcall_keywords, + (1, 2), (expected_self, (1, 2), NULL_OR_EMPTY)), + (obj.meth_fastcall_keywords, + (), (expected_self, (), NULL_OR_EMPTY)), + (obj.meth_noargs, (), expected_self), + (obj.meth_o, (123, ), (expected_self, 123)), + ]) + + CALLS_KWARGS.extend([ + (obj.meth_varargs_keywords, + (1, 2), {'x': 'y'}, (expected_self, (1, 2), {'x': 'y'})), + (obj.meth_varargs_keywords, + (), {'x': 'y'}, (expected_self, (), {'x': 'y'})), + (obj.meth_varargs_keywords, + (1, 2), {}, (expected_self, (1, 2), NULL_OR_EMPTY)), + (obj.meth_fastcall_keywords, + (1, 2), {'x': 'y'}, (expected_self, (1, 2), {'x': 'y'})), + (obj.meth_fastcall_keywords, + (), {'x': 'y'}, (expected_self, (), {'x': 'y'})), + (obj.meth_fastcall_keywords, + (1, 2), {}, (expected_self, (1, 2), NULL_OR_EMPTY)), + ]) def check_result(self, result, expected): - if expected is IGNORE_RESULT: - return + if isinstance(expected, tuple) and expected[-1] is NULL_OR_EMPTY: + if result[-1] in ({}, None): + expected = (*expected[:-1], result[-1]) self.assertEqual(result, expected) - def test_fastcall(self): - # Test _PyObject_FastCall() - - for func, args, expected in self.CALLS_POSARGS: - with self.subTest(func=func, args=args): - result = _testcapi.pyobject_fastcall(func, args) - self.check_result(result, expected) - - if not args: - # args=NULL, nargs=0 - result = _testcapi.pyobject_fastcall(func, None) - self.check_result(result, expected) - + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_vectorcall_dict(self): - # Test _PyObject_FastCallDict() + # Test PyObject_VectorcallDict() for func, args, expected in self.CALLS_POSARGS: with self.subTest(func=func, args=args): @@ -411,26 +545,19 @@ def test_vectorcall_dict(self): result = _testcapi.pyobject_fastcalldict(func, args, None) self.check_result(result, expected) - # kwargs={} - result = _testcapi.pyobject_fastcalldict(func, args, {}) - self.check_result(result, expected) - if not args: # args=NULL, nargs=0, kwargs=NULL result = _testcapi.pyobject_fastcalldict(func, None, None) self.check_result(result, expected) - # args=NULL, nargs=0, kwargs={} - result = _testcapi.pyobject_fastcalldict(func, None, {}) - self.check_result(result, expected) - for func, args, kwargs, expected in self.CALLS_KWARGS: with self.subTest(func=func, args=args, kwargs=kwargs): result = _testcapi.pyobject_fastcalldict(func, args, kwargs) self.check_result(result, expected) + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_vectorcall(self): - # Test _PyObject_Vectorcall() + # Test PyObject_Vectorcall() for func, args, expected in self.CALLS_POSARGS: with self.subTest(func=func, args=args): @@ -458,6 +585,7 @@ def test_vectorcall(self): result = _testcapi.pyobject_vectorcall(func, args, kwnames) self.check_result(result, expected) + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'IntWithDict' found. def test_fastcall_clearing_dict(self): # Test bpo-36907: the point of the test is just checking that this # does not crash. @@ -469,7 +597,7 @@ def __index__(self): self.kwargs.clear() gc.collect() return 0 - x = IntWithDict(dont_inherit=IntWithDict()) + x = IntWithDict(optimize=IntWithDict()) # We test the argument handling of "compile" here, the compilation # itself is not relevant. When we pass flags=x below, x.__index__() is # called, which changes the keywords dict. @@ -490,10 +618,12 @@ def testfunction_kw(self, *, kw): return self +ADAPTIVE_WARMUP_DELAY = 2 + + +@unittest.skipIf(_testcapi is None, "requires _testcapi") class TestPEP590(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_method_descriptor_flag(self): import functools cached = functools.lru_cache(1)(testfunction) @@ -508,26 +638,32 @@ def test_method_descriptor_flag(self): self.assertTrue(_testcapi.MethodDescriptorDerived.__flags__ & Py_TPFLAGS_METHOD_DESCRIPTOR) self.assertFalse(_testcapi.MethodDescriptorNopGet.__flags__ & Py_TPFLAGS_METHOD_DESCRIPTOR) - # Heap type should not inherit Py_TPFLAGS_METHOD_DESCRIPTOR + # Mutable heap types should not inherit Py_TPFLAGS_METHOD_DESCRIPTOR class MethodDescriptorHeap(_testcapi.MethodDescriptorBase): pass self.assertFalse(MethodDescriptorHeap.__flags__ & Py_TPFLAGS_METHOD_DESCRIPTOR) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_vectorcall_flag(self): self.assertTrue(_testcapi.MethodDescriptorBase.__flags__ & Py_TPFLAGS_HAVE_VECTORCALL) self.assertTrue(_testcapi.MethodDescriptorDerived.__flags__ & Py_TPFLAGS_HAVE_VECTORCALL) self.assertFalse(_testcapi.MethodDescriptorNopGet.__flags__ & Py_TPFLAGS_HAVE_VECTORCALL) self.assertTrue(_testcapi.MethodDescriptor2.__flags__ & Py_TPFLAGS_HAVE_VECTORCALL) - # Heap type should not inherit Py_TPFLAGS_HAVE_VECTORCALL + # Mutable heap types should inherit Py_TPFLAGS_HAVE_VECTORCALL, + # but should lose it when __call__ is overridden class MethodDescriptorHeap(_testcapi.MethodDescriptorBase): pass + self.assertTrue(MethodDescriptorHeap.__flags__ & Py_TPFLAGS_HAVE_VECTORCALL) + MethodDescriptorHeap.__call__ = print + self.assertFalse(MethodDescriptorHeap.__flags__ & Py_TPFLAGS_HAVE_VECTORCALL) + + # Mutable heap types should not inherit Py_TPFLAGS_HAVE_VECTORCALL if + # they define __call__ directly + class MethodDescriptorHeap(_testcapi.MethodDescriptorBase): + def __call__(self): + pass self.assertFalse(MethodDescriptorHeap.__flags__ & Py_TPFLAGS_HAVE_VECTORCALL) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_vectorcall_override(self): # Check that tp_call can correctly override vectorcall. # MethodDescriptorNopGet implements tp_call but it inherits from @@ -538,14 +674,64 @@ def test_vectorcall_override(self): f = _testcapi.MethodDescriptorNopGet() self.assertIs(f(*args), args) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_vectorcall_override_on_mutable_class(self): + """Setting __call__ should disable vectorcall""" + TestType = _testcapi.make_vectorcall_class() + instance = TestType() + self.assertEqual(instance(), "tp_call") + instance.set_vectorcall(TestType) + self.assertEqual(instance(), "vectorcall") # assume vectorcall is used + TestType.__call__ = lambda self: "custom" + self.assertEqual(instance(), "custom") + + def test_vectorcall_override_with_subclass(self): + """Setting __call__ on a superclass should disable vectorcall""" + SuperType = _testcapi.make_vectorcall_class() + class DerivedType(SuperType): + pass + + instance = DerivedType() + + # Derived types with its own vectorcall should be unaffected + UnaffectedType1 = _testcapi.make_vectorcall_class(DerivedType) + UnaffectedType2 = _testcapi.make_vectorcall_class(SuperType) + + # Aside: Quickly check that the C helper actually made derived types + self.assertTrue(issubclass(UnaffectedType1, DerivedType)) + self.assertTrue(issubclass(UnaffectedType2, SuperType)) + + # Initial state: tp_call + self.assertEqual(instance(), "tp_call") + self.assertEqual(_testcapi.has_vectorcall_flag(SuperType), True) + self.assertEqual(_testcapi.has_vectorcall_flag(DerivedType), True) + self.assertEqual(_testcapi.has_vectorcall_flag(UnaffectedType1), True) + self.assertEqual(_testcapi.has_vectorcall_flag(UnaffectedType2), True) + + # Setting the vectorcall function + instance.set_vectorcall(SuperType) + + self.assertEqual(instance(), "vectorcall") + self.assertEqual(_testcapi.has_vectorcall_flag(SuperType), True) + self.assertEqual(_testcapi.has_vectorcall_flag(DerivedType), True) + self.assertEqual(_testcapi.has_vectorcall_flag(UnaffectedType1), True) + self.assertEqual(_testcapi.has_vectorcall_flag(UnaffectedType2), True) + + # Setting __call__ should remove vectorcall from all subclasses + SuperType.__call__ = lambda self: "custom" + + self.assertEqual(instance(), "custom") + self.assertEqual(_testcapi.has_vectorcall_flag(SuperType), False) + self.assertEqual(_testcapi.has_vectorcall_flag(DerivedType), False) + self.assertEqual(_testcapi.has_vectorcall_flag(UnaffectedType1), True) + self.assertEqual(_testcapi.has_vectorcall_flag(UnaffectedType2), True) + + def test_vectorcall(self): # Test a bunch of different ways to call objects: # 1. vectorcall using PyVectorcall_Call() # (only for objects that support vectorcall directly) # 2. normal call - # 3. vectorcall using _PyObject_Vectorcall() + # 3. vectorcall using PyObject_Vectorcall() # 4. call as bound method # 5. call using functools.partial @@ -616,6 +802,300 @@ def __call__(self, *args): self.assertEqual(expected, meth(*args1, **kwargs)) self.assertEqual(expected, wrapped(*args, **kwargs)) + def test_setvectorcall(self): + from _testcapi import function_setvectorcall + def f(num): return num + 1 + assert_equal = self.assertEqual + num = 10 + assert_equal(11, f(num)) + function_setvectorcall(f) + # make sure specializer is triggered by running > 50 times + for _ in range(10 * ADAPTIVE_WARMUP_DELAY): + assert_equal("overridden", f(num)) + + def test_setvectorcall_load_attr_specialization_skip(self): + from _testcapi import function_setvectorcall + + class X: + def __getattribute__(self, attr): + return attr + + assert_equal = self.assertEqual + x = X() + assert_equal("a", x.a) + function_setvectorcall(X.__getattribute__) + # make sure specialization doesn't trigger + # when vectorcall is overridden + for _ in range(ADAPTIVE_WARMUP_DELAY): + assert_equal("overridden", x.a) + + def test_setvectorcall_load_attr_specialization_deopt(self): + from _testcapi import function_setvectorcall + + class X: + def __getattribute__(self, attr): + return attr + + def get_a(x): + return x.a + + assert_equal = self.assertEqual + x = X() + # trigger LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN specialization + for _ in range(ADAPTIVE_WARMUP_DELAY): + assert_equal("a", get_a(x)) + function_setvectorcall(X.__getattribute__) + # make sure specialized LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN + # gets deopted due to overridden vectorcall + for _ in range(ADAPTIVE_WARMUP_DELAY): + assert_equal("overridden", get_a(x)) + + @requires_limited_api + def test_vectorcall_limited_incoming(self): + from _testcapi import pyobject_vectorcall + obj = _testlimitedcapi.LimitedVectorCallClass() + self.assertEqual(pyobject_vectorcall(obj, (), ()), "vectorcall called") + + @requires_limited_api + def test_vectorcall_limited_outgoing(self): + from _testlimitedcapi import call_vectorcall + + args_captured = [] + kwargs_captured = [] + + def f(*args, **kwargs): + args_captured.append(args) + kwargs_captured.append(kwargs) + return "success" + + self.assertEqual(call_vectorcall(f), "success") + self.assertEqual(args_captured, [("foo",)]) + self.assertEqual(kwargs_captured, [{"baz": "bar"}]) + + @requires_limited_api + def test_vectorcall_limited_outgoing_method(self): + from _testlimitedcapi import call_vectorcall_method + + args_captured = [] + kwargs_captured = [] + + class TestInstance: + def f(self, *args, **kwargs): + args_captured.append(args) + kwargs_captured.append(kwargs) + return "success" + + self.assertEqual(call_vectorcall_method(TestInstance()), "success") + self.assertEqual(args_captured, [("foo",)]) + self.assertEqual(kwargs_captured, [{"baz": "bar"}]) + +class A: + def method_two_args(self, x, y): + pass + + @staticmethod + def static_no_args(): + pass + + @staticmethod + def positional_only(arg, /): + pass + +@cpython_only +class TestErrorMessagesUseQualifiedName(unittest.TestCase): + + @contextlib.contextmanager + def check_raises_type_error(self, message): + with self.assertRaises(TypeError) as cm: + yield + self.assertEqual(str(cm.exception), message) + + def test_missing_arguments(self): + msg = "A.method_two_args() missing 1 required positional argument: 'y'" + with self.check_raises_type_error(msg): + A().method_two_args("x") + + def test_too_many_positional(self): + msg = "A.static_no_args() takes 0 positional arguments but 1 was given" + with self.check_raises_type_error(msg): + A.static_no_args("oops it's an arg") + + def test_positional_only_passed_as_keyword(self): + msg = "A.positional_only() got some positional-only arguments passed as keyword arguments: 'arg'" + with self.check_raises_type_error(msg): + A.positional_only(arg="x") + + def test_unexpected_keyword(self): + msg = "A.method_two_args() got an unexpected keyword argument 'bad'" + with self.check_raises_type_error(msg): + A().method_two_args(bad="x") + + def test_multiple_values(self): + msg = "A.method_two_args() got multiple values for argument 'x'" + with self.check_raises_type_error(msg): + A().method_two_args("x", "y", x="oops") + +@cpython_only +class TestErrorMessagesSuggestions(unittest.TestCase): + @contextlib.contextmanager + def check_suggestion_includes(self, message): + with self.assertRaises(TypeError) as cm: + yield + self.assertIn(f"Did you mean '{message}'?", str(cm.exception)) + + @contextlib.contextmanager + def check_suggestion_not_present(self): + with self.assertRaises(TypeError) as cm: + yield + self.assertNotIn("Did you mean", str(cm.exception)) + + def test_unexpected_keyword_suggestion_valid_positions(self): + def foo(blech=None, /, aaa=None, *args, late1=None): + pass + + cases = [ + ("blach", None), + ("aa", "aaa"), + ("orgs", None), + ("late11", "late1"), + ] + + for keyword, suggestion in cases: + with self.subTest(keyword): + ctx = self.check_suggestion_includes(suggestion) if suggestion else self.check_suggestion_not_present() + with ctx: + foo(**{keyword:None}) + + def test_unexpected_keyword_suggestion_kinds(self): + + def substitution(noise=None, more_noise=None, a = None, blech = None): + pass + + def elimination(noise = None, more_noise = None, a = None, blch = None): + pass + + def addition(noise = None, more_noise = None, a = None, bluchin = None): + pass + + def substitution_over_elimination(blach = None, bluc = None): + pass + + def substitution_over_addition(blach = None, bluchi = None): + pass + + def elimination_over_addition(bluc = None, blucha = None): + pass + + def case_change_over_substitution(BLuch=None, Luch = None, fluch = None): + pass + + for func, suggestion in [ + (addition, "bluchin"), + (substitution, "blech"), + (elimination, "blch"), + (addition, "bluchin"), + (substitution_over_elimination, "blach"), + (substitution_over_addition, "blach"), + (elimination_over_addition, "bluc"), + (case_change_over_substitution, "BLuch"), + ]: + with self.subTest(suggestion): + with self.check_suggestion_includes(suggestion): + func(bluch=None) + + def test_unexpected_keyword_suggestion_via_getargs(self): + with self.check_suggestion_includes("maxsplit"): + "foo".split(maxsplt=1) + + self.assertRaisesRegex( + TypeError, r"split\(\) got an unexpected keyword argument 'blech'$", + "foo".split, blech=1 + ) + with self.check_suggestion_not_present(): + "foo".split(blech=1) + with self.check_suggestion_not_present(): + "foo".split(more_noise=1, maxsplt=1) + + # Also test the vgetargskeywords path + with self.check_suggestion_includes("name"): + ImportError(namez="oops") + + self.assertRaisesRegex( + TypeError, r"ImportError\(\) got an unexpected keyword argument 'blech'$", + ImportError, blech=1 + ) + with self.check_suggestion_not_present(): + ImportError(blech=1) + with self.check_suggestion_not_present(): + ImportError(blech=1, namez="oops") + +@cpython_only +class TestRecursion(unittest.TestCase): + + @skip_on_s390x + @unittest.skipIf(is_wasi and Py_DEBUG, "requires deep stack") + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_super_deep(self): + + def recurse(n): + if n: + recurse(n-1) + + def py_recurse(n, m): + if n: + py_recurse(n-1, m) + else: + c_py_recurse(m-1) + + def c_recurse(n): + if n: + _testcapi.pyobject_vectorcall(c_recurse, (n-1,), ()) + + def c_py_recurse(m): + if m: + _testcapi.pyobject_vectorcall(py_recurse, (1000, m), ()) + + with set_recursion_limit(100_000): + recurse(90_000) + with self.assertRaises(RecursionError): + recurse(101_000) + c_recurse(100) + with self.assertRaises(RecursionError): + c_recurse(90_000) + c_py_recurse(90) + with self.assertRaises(RecursionError): + c_py_recurse(100_000) + + +class TestFunctionWithManyArgs(unittest.TestCase): + def test_function_with_many_args(self): + for N in (10, 500, 1000): + with self.subTest(N=N): + args = ",".join([f"a{i}" for i in range(N)]) + src = f"def f({args}) : return a{N//2}" + l = {} + exec(src, {}, l) + self.assertEqual(l['f'](*range(N)), N//2) + + +@unittest.skipIf(_testcapi is None, 'need _testcapi') +class TestCAPI(unittest.TestCase): + def test_cfunction_call(self): + def func(*args, **kwargs): + return (args, kwargs) + + # PyCFunction_Call() was removed in Python 3.13 API, but was kept in + # the stable ABI. + def PyCFunction_Call(func, *args, **kwargs): + if kwargs: + return _testcapi.pycfunction_call(func, args, kwargs) + else: + return _testcapi.pycfunction_call(func, args) + + self.assertEqual(PyCFunction_Call(func), ((), {})) + self.assertEqual(PyCFunction_Call(func, 1, 2, 3), ((1, 2, 3), {})) + self.assertEqual(PyCFunction_Call(func, "arg", num=5), (("arg",), {'num': 5})) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 6b0dc09e28..b7e5784b48 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -347,8 +347,6 @@ def func(arg): newcode = code.replace(co_name="func") # Should not raise SystemError self.assertEqual(code, newcode) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_empty_linetable(self): def func(): pass @@ -468,8 +466,6 @@ def f(): # co_positions behavior when info is missing. - # TODO: RUSTPYTHON - @unittest.expectedFailure # @requires_debug_ranges() def test_co_positions_empty_linetable(self): def func(): @@ -480,8 +476,6 @@ def func(): self.assertIsNone(line) self.assertEqual(end_line, new_code.co_firstlineno + 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_code_equality(self): def f(): try: @@ -522,8 +516,6 @@ def test_code_hash_uses_order(self): self.assertNotEqual(c, swapped) self.assertNotEqual(hash(c), hash(swapped)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_code_hash_uses_bytecode(self): c = (lambda x, y: x + y).__code__ d = (lambda x, y: x * y).__code__ @@ -735,8 +727,6 @@ def check_positions(self, func): self.assertEqual(l1, l2) self.assertEqual(len(pos1), len(pos2)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_positions(self): self.check_positions(parse_location_table) self.check_positions(misshappen) @@ -751,8 +741,6 @@ def check_lines(self, func): self.assertEqual(l1, l2) self.assertEqual(len(lines1), len(lines2)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_lines(self): self.check_lines(parse_location_table) self.check_lines(misshappen) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index a12e5893dc..0d3d8c9e2d 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -2,7 +2,6 @@ import contextlib import copy import io -import locale import pickle import sys import unittest @@ -13,9 +12,9 @@ from test.support import os_helper try: - import _testcapi + import _testlimitedcapi except ImportError: - _testcapi = None + _testlimitedcapi = None try: import _testinternalcapi except ImportError: @@ -389,8 +388,7 @@ def test_bug1098990_b(self): ill_formed_sequence_replace = "\ufffd" - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_lone_surrogates(self): self.assertRaises(UnicodeEncodeError, "\ud800".encode, self.encoding) self.assertEqual("[\uDC80]".encode(self.encoding, "backslashreplace"), @@ -466,14 +464,7 @@ class UTF32Test(ReadTest, unittest.TestCase): b'\x00\x00\x00s\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m' b'\x00\x00\x00s\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m') - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_readline(self): # TODO: RUSTPYTHON, remove when this passes - super().test_readline() # TODO: RUSTPYTHON, remove when this passes - - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_only_one_bom(self): _,_,reader,writer = codecs.lookup(self.encoding) # encode some stream @@ -489,19 +480,17 @@ def test_only_one_bom(self): f = reader(s) self.assertEqual(f.read(), "spamspam") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_badbom(self): s = io.BytesIO(4*b"\xff") f = codecs.getreader(self.encoding)(s) - self.assertRaises(UnicodeError, f.read) + self.assertRaises(UnicodeDecodeError, f.read) s = io.BytesIO(8*b"\xff") f = codecs.getreader(self.encoding)(s) - self.assertRaises(UnicodeError, f.read) + self.assertRaises(UnicodeDecodeError, f.read) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -533,30 +522,26 @@ def test_partial(self): ] ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_handlers(self): self.assertEqual(('\ufffd', 1), codecs.utf_32_decode(b'\x01', 'replace', True)) self.assertEqual(('', 1), codecs.utf_32_decode(b'\x01', 'ignore', True)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_errors(self): self.assertRaises(UnicodeDecodeError, codecs.utf_32_decode, b"\xff", "strict", True) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_decoder_state(self): self.check_state_handling_decode(self.encoding, "spamspam", self.spamle) self.check_state_handling_decode(self.encoding, "spamspam", self.spambe) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_issue8941(self): # Issue #8941: insufficient result allocation when decoding into # surrogate pairs on UCS-2 builds. @@ -567,48 +552,40 @@ def test_issue8941(self): self.assertEqual('\U00010000' * 1024, codecs.utf_32_decode(encoded_be)[0]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug1098990_a(self): - super().test_bug1098990_a() + return super().test_bug1098990_a() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug1098990_b(self): - super().test_bug1098990_b() + return super().test_bug1098990_b() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug1175396(self): - super().test_bug1175396() + return super().test_bug1175396() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_incremental_surrogatepass(self): - super().test_incremental_surrogatepass() + return super().test_incremental_surrogatepass() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_mixed_readline_and_read(self): - super().test_mixed_readline_and_read() + return super().test_mixed_readline_and_read() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_readline(self): + return super().test_readline() + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_readlinequeue(self): - super().test_readlinequeue() + return super().test_readlinequeue() class UTF32LETest(ReadTest, unittest.TestCase): encoding = "utf-32-le" ill_formed_sequence = b"\x80\xdc\x00\x00" - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_readline(self): # TODO: RUSTPYTHON, remove when this passes - super().test_readline() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -636,19 +613,16 @@ def test_partial(self): ] ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_simple(self): self.assertEqual("\U00010203".encode(self.encoding), b"\x03\x02\x01\x00") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_errors(self): self.assertRaises(UnicodeDecodeError, codecs.utf_32_le_decode, b"\xff", "strict", True) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_issue8941(self): # Issue #8941: insufficient result allocation when decoding into # surrogate pairs on UCS-2 builds. @@ -656,48 +630,40 @@ def test_issue8941(self): self.assertEqual('\U00010000' * 1024, codecs.utf_32_le_decode(encoded)[0]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug1098990_a(self): - super().test_bug1098990_a() + return super().test_bug1098990_a() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug1098990_b(self): - super().test_bug1098990_b() + return super().test_bug1098990_b() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug1175396(self): - super().test_bug1175396() + return super().test_bug1175396() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_incremental_surrogatepass(self): - super().test_incremental_surrogatepass() + return super().test_incremental_surrogatepass() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_mixed_readline_and_read(self): - super().test_mixed_readline_and_read() + return super().test_mixed_readline_and_read() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_readline(self): + return super().test_readline() + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_readlinequeue(self): - super().test_readlinequeue() + return super().test_readlinequeue() class UTF32BETest(ReadTest, unittest.TestCase): encoding = "utf-32-be" ill_formed_sequence = b"\x00\x00\xdc\x80" - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_readline(self): # TODO: RUSTPYTHON, remove when this passes - super().test_readline() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -725,19 +691,16 @@ def test_partial(self): ] ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_simple(self): self.assertEqual("\U00010203".encode(self.encoding), b"\x00\x01\x02\x03") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_errors(self): self.assertRaises(UnicodeDecodeError, codecs.utf_32_be_decode, b"\xff", "strict", True) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_issue8941(self): # Issue #8941: insufficient result allocation when decoding into # surrogate pairs on UCS-2 builds. @@ -745,35 +708,33 @@ def test_issue8941(self): self.assertEqual('\U00010000' * 1024, codecs.utf_32_be_decode(encoded)[0]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug1098990_a(self): - super().test_bug1098990_a() + return super().test_bug1098990_a() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug1098990_b(self): - super().test_bug1098990_b() + return super().test_bug1098990_b() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug1175396(self): - super().test_bug1175396() + return super().test_bug1175396() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_incremental_surrogatepass(self): - super().test_incremental_surrogatepass() + return super().test_incremental_surrogatepass() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_mixed_readline_and_read(self): - super().test_mixed_readline_and_read() + return super().test_mixed_readline_and_read() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_readline(self): + return super().test_readline() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_readlinequeue(self): - super().test_readlinequeue() + return super().test_readlinequeue() class UTF16Test(ReadTest, unittest.TestCase): @@ -804,14 +765,13 @@ def test_only_one_bom(self): def test_badbom(self): s = io.BytesIO(b"\xff\xff") f = codecs.getreader(self.encoding)(s) - self.assertRaises(UnicodeError, f.read) + self.assertRaises(UnicodeDecodeError, f.read) s = io.BytesIO(b"\xff\xff\xff\xff") f = codecs.getreader(self.encoding)(s) - self.assertRaises(UnicodeError, f.read) + self.assertRaises(UnicodeDecodeError, f.read) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -833,8 +793,7 @@ def test_partial(self): ] ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_handlers(self): self.assertEqual(('\ufffd', 1), codecs.utf_16_decode(b'\x01', 'replace', True)) @@ -850,9 +809,7 @@ def test_decoder_state(self): "spamspam", self.spamle) self.check_state_handling_decode(self.encoding, "spamspam", self.spambe) - - # TODO: RUSTPYTHON - ValueError: invalid mode 'Ub' - @unittest.expectedFailure + def test_bug691291(self): # If encoding is not None, then # files are always opened in binary mode, even if no binary mode was @@ -864,22 +821,32 @@ def test_bug691291(self): self.addCleanup(os_helper.unlink, os_helper.TESTFN) with open(os_helper.TESTFN, 'wb') as fp: fp.write(s) - with warnings_helper.check_warnings(('', DeprecationWarning)): - reader = codecs.open(os_helper.TESTFN, 'U', encoding=self.encoding) - with reader: + with codecs.open(os_helper.TESTFN, 'r', + encoding=self.encoding) as reader: self.assertEqual(reader.read(), s1) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_invalid_modes(self): + for mode in ('U', 'rU', 'r+U'): + with self.assertRaises(ValueError) as cm: + codecs.open(os_helper.TESTFN, mode, encoding=self.encoding) + self.assertIn('invalid mode', str(cm.exception)) + + for mode in ('rt', 'wt', 'at', 'r+t'): + with self.assertRaises(ValueError) as cm: + codecs.open(os_helper.TESTFN, mode, encoding=self.encoding) + self.assertIn("can't have text and binary mode at once", + str(cm.exception)) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_incremental_surrogatepass(self): - super().test_incremental_surrogatepass() + return super().test_incremental_surrogatepass() + class UTF16LETest(ReadTest, unittest.TestCase): encoding = "utf-16-le" ill_formed_sequence = b"\x80\xdc" - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -899,8 +866,7 @@ def test_partial(self): ] ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_errors(self): tests = [ (b'\xff', '\ufffd'), @@ -922,17 +888,15 @@ def test_nonbmp(self): self.assertEqual(b'\x00\xd8\x03\xde'.decode(self.encoding), "\U00010203") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_incremental_surrogatepass(self): - super().test_incremental_surrogatepass() + return super().test_incremental_surrogatepass() class UTF16BETest(ReadTest, unittest.TestCase): encoding = "utf-16-be" ill_formed_sequence = b"\xdc\x80" - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -952,8 +916,7 @@ def test_partial(self): ] ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_errors(self): tests = [ (b'\xff', '\ufffd'), @@ -975,10 +938,9 @@ def test_nonbmp(self): self.assertEqual(b'\xd8\x00\xde\x03'.decode(self.encoding), "\U00010203") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_incremental_surrogatepass(self): - super().test_incremental_surrogatepass() + return super().test_incremental_surrogatepass() class UTF8Test(ReadTest, unittest.TestCase): encoding = "utf-8" @@ -1025,8 +987,7 @@ def test_decode_error(self): self.assertEqual(data.decode(self.encoding, error_handler), expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_lone_surrogates(self): super().test_lone_surrogates() # not sure if this is making sense for @@ -1079,13 +1040,7 @@ def test_incremental_errors(self): class UTF7Test(ReadTest, unittest.TestCase): encoding = "utf-7" - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_readline(self): # TODO: RUSTPYTHON, remove when this passes - super().test_readline() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_ascii(self): # Set D (directly encoded characters) set_d = ('ABCDEFGHIJKLMNOPQRSTUVWXYZ' @@ -1112,8 +1067,7 @@ def test_ascii(self): b'+AAAAAQACAAMABAAFAAYABwAIAAsADAAOAA8AEAARABIAEwAU' b'ABUAFgAXABgAGQAaABsAHAAdAB4AHwBcAH4Afw-') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( 'a+-b\x00c\x80d\u0100e\U00010000f', @@ -1153,8 +1107,7 @@ def test_partial(self): ] ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_errors(self): tests = [ (b'\xffb', '\ufffdb'), @@ -1185,8 +1138,7 @@ def test_errors(self): raw, 'strict', True) self.assertEqual(raw.decode('utf-7', 'replace'), expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_nonbmp(self): self.assertEqual('\U000104A0'.encode(self.encoding), b'+2AHcoA-') self.assertEqual('\ud801\udca0'.encode(self.encoding), b'+2AHcoA-') @@ -1202,8 +1154,7 @@ def test_nonbmp(self): self.assertEqual(b'+IKwgrNgB3KA'.decode(self.encoding), '\u20ac\u20ac\U000104A0') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_lone_surrogates(self): tests = [ (b'a+2AE-b', 'a\ud801b'), @@ -1224,15 +1175,17 @@ def test_lone_surrogates(self): with self.subTest(raw=raw): self.assertEqual(raw.decode('utf-7', 'replace'), expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug1175396(self): - super().test_bug1175396() + return super().test_bug1175396() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_incremental_surrogatepass(self): - super().test_incremental_surrogatepass() + return super().test_incremental_surrogatepass() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_readline(self): + return super().test_readline() class UTF16ExTest(unittest.TestCase): @@ -1357,8 +1310,7 @@ def test_raw(self): if b != b'\\': self.assertEqual(decode(b + b'0'), (b + b'0', 2)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_escape(self): decode = codecs.escape_decode check = coding_checker(self, decode) @@ -1379,28 +1331,43 @@ def test_escape(self): check(br"[\418]", b"[!8]") check(br"[\101]", b"[A]") check(br"[\1010]", b"[A0]") - check(br"[\501]", b"[A]") check(br"[\x41]", b"[A]") check(br"[\x410]", b"[A0]") + + @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered + def test_warnings(self): + decode = codecs.escape_decode + check = coding_checker(self, decode) for i in range(97, 123): b = bytes([i]) if b not in b'abfnrtvx': - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r"invalid escape sequence '\\%c'" % i): check(b"\\" + b, b"\\" + b) - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r"invalid escape sequence '\\%c'" % (i-32)): check(b"\\" + b.upper(), b"\\" + b.upper()) - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r"invalid escape sequence '\\8'"): check(br"\8", b"\\8") with self.assertWarns(DeprecationWarning): check(br"\9", b"\\9") - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r"invalid escape sequence '\\\xfa'") as cm: check(b"\\\xfa", b"\\\xfa") for i in range(0o400, 0o1000): - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r"invalid octal escape sequence '\\%o'" % i): check(rb'\%o' % i, bytes([i & 0o377])) - - # TODO: RUSTPYTHON - ValueError: not raised by escape_decode - @unittest.expectedFailure + + with self.assertWarnsRegex(DeprecationWarning, + r"invalid escape sequence '\\z'"): + self.assertEqual(decode(br'\x\z', 'ignore'), (b'\\z', 4)) + with self.assertWarnsRegex(DeprecationWarning, + r"invalid octal escape sequence '\\501'"): + self.assertEqual(decode(br'\x\501', 'ignore'), (b'A', 6)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: not raised by escape_decode def test_errors(self): decode = codecs.escape_decode self.assertRaises(ValueError, decode, br"\x") @@ -1543,13 +1510,29 @@ def test_decode(self): def test_decode_invalid(self): testcases = [ - (b"xn--w&", "strict", UnicodeError()), + (b"xn--w&", "strict", UnicodeDecodeError("punycode", b"", 5, 6, "")), + (b"&egbpdaj6bu4bxfgehfvwxn", "strict", UnicodeDecodeError("punycode", b"", 0, 1, "")), + (b"egbpdaj6bu&4bx&fgehfvwxn", "strict", UnicodeDecodeError("punycode", b"", 10, 11, "")), + (b"egbpdaj6bu4bxfgehfvwxn&", "strict", UnicodeDecodeError("punycode", b"", 22, 23, "")), + (b"\xFFProprostnemluvesky-uyb24dma41a", "strict", UnicodeDecodeError("ascii", b"", 0, 1, "")), + (b"Pro\xFFprostnemluvesky-uyb24dma41a", "strict", UnicodeDecodeError("ascii", b"", 3, 4, "")), + (b"Proprost&nemluvesky-uyb24&dma41a", "strict", UnicodeDecodeError("punycode", b"", 25, 26, "")), + (b"Proprostnemluvesky&-&uyb24dma41a", "strict", UnicodeDecodeError("punycode", b"", 20, 21, "")), + (b"Proprostnemluvesky-&uyb24dma41a", "strict", UnicodeDecodeError("punycode", b"", 19, 20, "")), + (b"Proprostnemluvesky-uyb24d&ma41a", "strict", UnicodeDecodeError("punycode", b"", 25, 26, "")), + (b"Proprostnemluvesky-uyb24dma41a&", "strict", UnicodeDecodeError("punycode", b"", 30, 31, "")), (b"xn--w&", "ignore", "xn-"), ] for puny, errors, expected in testcases: with self.subTest(puny=puny, errors=errors): if isinstance(expected, Exception): - self.assertRaises(UnicodeError, puny.decode, "punycode", errors) + with self.assertRaises(UnicodeDecodeError) as cm: + puny.decode("punycode", errors) + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, puny) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) else: self.assertEqual(puny.decode("punycode", errors), expected) @@ -1719,7 +1702,7 @@ def test_nameprep(self): orig = str(orig, "utf-8", "surrogatepass") if prepped is None: # Input contains prohibited characters - self.assertRaises(UnicodeError, nameprep, orig) + self.assertRaises(UnicodeEncodeError, nameprep, orig) else: prepped = str(prepped, "utf-8", "surrogatepass") try: @@ -1729,11 +1712,46 @@ def test_nameprep(self): class IDNACodecTest(unittest.TestCase): + + invalid_decode_testcases = [ + (b"\xFFpython.org", UnicodeDecodeError("idna", b"\xFFpython.org", 0, 1, "")), + (b"pyt\xFFhon.org", UnicodeDecodeError("idna", b"pyt\xFFhon.org", 3, 4, "")), + (b"python\xFF.org", UnicodeDecodeError("idna", b"python\xFF.org", 6, 7, "")), + (b"python.\xFForg", UnicodeDecodeError("idna", b"python.\xFForg", 7, 8, "")), + (b"python.o\xFFrg", UnicodeDecodeError("idna", b"python.o\xFFrg", 8, 9, "")), + (b"python.org\xFF", UnicodeDecodeError("idna", b"python.org\xFF", 10, 11, "")), + (b"xn--pythn-&mua.org", UnicodeDecodeError("idna", b"xn--pythn-&mua.org", 10, 11, "")), + (b"xn--pythn-m&ua.org", UnicodeDecodeError("idna", b"xn--pythn-m&ua.org", 11, 12, "")), + (b"xn--pythn-mua&.org", UnicodeDecodeError("idna", b"xn--pythn-mua&.org", 13, 14, "")), + ] + invalid_encode_testcases = [ + (f"foo.{'\xff'*60}", UnicodeEncodeError("idna", f"foo.{'\xff'*60}", 4, 64, "")), + ("あさ.\u034f", UnicodeEncodeError("idna", "あさ.\u034f", 3, 4, "")), + ] + def test_builtin_decode(self): self.assertEqual(str(b"python.org", "idna"), "python.org") self.assertEqual(str(b"python.org.", "idna"), "python.org.") self.assertEqual(str(b"xn--pythn-mua.org", "idna"), "pyth\xf6n.org") self.assertEqual(str(b"xn--pythn-mua.org.", "idna"), "pyth\xf6n.org.") + self.assertEqual(str(b"XN--pythn-mua.org.", "idna"), "pyth\xf6n.org.") + self.assertEqual(str(b"xN--pythn-mua.org.", "idna"), "pyth\xf6n.org.") + self.assertEqual(str(b"Xn--pythn-mua.org.", "idna"), "pyth\xf6n.org.") + self.assertEqual(str(b"bugs.xn--pythn-mua.org.", "idna"), + "bugs.pyth\xf6n.org.") + self.assertEqual(str(b"bugs.XN--pythn-mua.org.", "idna"), + "bugs.pyth\xf6n.org.") + + def test_builtin_decode_invalid(self): + for case, expected in self.invalid_decode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeDecodeError) as cm: + case.decode("idna") + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start, msg=f'reason: {exc.reason}') + self.assertEqual(exc.end, expected.end) def test_builtin_encode(self): self.assertEqual("python.org".encode("idna"), b"python.org") @@ -1741,10 +1759,21 @@ def test_builtin_encode(self): self.assertEqual("pyth\xf6n.org".encode("idna"), b"xn--pythn-mua.org") self.assertEqual("pyth\xf6n.org.".encode("idna"), b"xn--pythn-mua.org.") + def test_builtin_encode_invalid(self): + for case, expected in self.invalid_encode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeEncodeError) as cm: + case.encode("idna") + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) + def test_builtin_decode_length_limit(self): - with self.assertRaisesRegex(UnicodeError, "way too long"): + with self.assertRaisesRegex(UnicodeDecodeError, "way too long"): (b"xn--016c"+b"a"*1100).decode("idna") - with self.assertRaisesRegex(UnicodeError, "too long"): + with self.assertRaisesRegex(UnicodeDecodeError, "too long"): (b"xn--016c"+b"a"*70).decode("idna") def test_stream(self): @@ -1782,6 +1811,39 @@ def test_incremental_decode(self): self.assertEqual(decoder.decode(b"rg."), "org.") self.assertEqual(decoder.decode(b"", True), "") + def test_incremental_decode_invalid(self): + iterdecode_testcases = [ + (b"\xFFpython.org", UnicodeDecodeError("idna", b"\xFF", 0, 1, "")), + (b"pyt\xFFhon.org", UnicodeDecodeError("idna", b"pyt\xFF", 3, 4, "")), + (b"python\xFF.org", UnicodeDecodeError("idna", b"python\xFF", 6, 7, "")), + (b"python.\xFForg", UnicodeDecodeError("idna", b"\xFF", 0, 1, "")), + (b"python.o\xFFrg", UnicodeDecodeError("idna", b"o\xFF", 1, 2, "")), + (b"python.org\xFF", UnicodeDecodeError("idna", b"org\xFF", 3, 4, "")), + (b"xn--pythn-&mua.org", UnicodeDecodeError("idna", b"xn--pythn-&mua.", 10, 11, "")), + (b"xn--pythn-m&ua.org", UnicodeDecodeError("idna", b"xn--pythn-m&ua.", 11, 12, "")), + (b"xn--pythn-mua&.org", UnicodeDecodeError("idna", b"xn--pythn-mua&.", 13, 14, "")), + ] + for case, expected in iterdecode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeDecodeError) as cm: + list(codecs.iterdecode((bytes([c]) for c in case), "idna")) + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) + + decoder = codecs.getincrementaldecoder("idna")() + for case, expected in self.invalid_decode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeDecodeError) as cm: + decoder.decode(case) + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) + def test_incremental_encode(self): self.assertEqual( b"".join(codecs.iterencode("python.org", "idna")), @@ -1810,6 +1872,23 @@ def test_incremental_encode(self): self.assertEqual(encoder.encode("ample.org."), b"xn--xample-9ta.org.") self.assertEqual(encoder.encode("", True), b"") + def test_incremental_encode_invalid(self): + iterencode_testcases = [ + (f"foo.{'\xff'*60}", UnicodeEncodeError("idna", f"{'\xff'*60}", 0, 60, "")), + ("あさ.\u034f", UnicodeEncodeError("idna", "\u034f", 0, 1, "")), + ] + for case, expected in iterencode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeEncodeError) as cm: + list(codecs.iterencode(case, "idna")) + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) + + # codecs.getincrementalencoder.encode() does not throw an error + def test_errors(self): """Only supports "strict" error handler""" "python.org".encode("idna", "strict") @@ -1887,16 +1966,10 @@ def test_getwriter(self): self.assertRaises(TypeError, codecs.getwriter) self.assertRaises(LookupError, codecs.getwriter, "__spam__") + @support.run_with_locale('LC_CTYPE', 'tr_TR') def test_lookup_issue1813(self): # Issue #1813: under Turkish locales, lookup of some codecs failed # because 'I' is lowercased as "ı" (dotless i) - oldlocale = locale.setlocale(locale.LC_CTYPE) - self.addCleanup(locale.setlocale, locale.LC_CTYPE, oldlocale) - try: - locale.setlocale(locale.LC_CTYPE, 'tr_TR') - except locale.Error: - # Unsupported locale on this system - self.skipTest('test needs Turkish locale') c = codecs.lookup('ASCII') self.assertEqual(c.name, 'ascii') @@ -1949,6 +2022,76 @@ def test_file_closes_if_lookup_error_raised(self): file().close.assert_called() + def test_copy(self): + orig = codecs.lookup('utf-8') + dup = copy.copy(orig) + self.assertIsNot(dup, orig) + self.assertEqual(dup, orig) + self.assertTrue(orig._is_text_encoding) + self.assertEqual(dup.encode, orig.encode) + self.assertEqual(dup.name, orig.name) + self.assertEqual(dup.incrementalencoder, orig.incrementalencoder) + + # Test a CodecInfo with _is_text_encoding equal to false. + orig = codecs.lookup("base64") + dup = copy.copy(orig) + self.assertIsNot(dup, orig) + self.assertEqual(dup, orig) + self.assertFalse(orig._is_text_encoding) + self.assertEqual(dup.encode, orig.encode) + self.assertEqual(dup.name, orig.name) + self.assertEqual(dup.incrementalencoder, orig.incrementalencoder) + + def test_deepcopy(self): + orig = codecs.lookup('utf-8') + dup = copy.deepcopy(orig) + self.assertIsNot(dup, orig) + self.assertEqual(dup, orig) + self.assertTrue(orig._is_text_encoding) + self.assertEqual(dup.encode, orig.encode) + self.assertEqual(dup.name, orig.name) + self.assertEqual(dup.incrementalencoder, orig.incrementalencoder) + + # Test a CodecInfo with _is_text_encoding equal to false. + orig = codecs.lookup("base64") + dup = copy.deepcopy(orig) + self.assertIsNot(dup, orig) + self.assertEqual(dup, orig) + self.assertFalse(orig._is_text_encoding) + self.assertEqual(dup.encode, orig.encode) + self.assertEqual(dup.name, orig.name) + self.assertEqual(dup.incrementalencoder, orig.incrementalencoder) + + def test_pickle(self): + codec_info = codecs.lookup('utf-8') + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + pickled_codec_info = pickle.dumps(codec_info) + unpickled_codec_info = pickle.loads(pickled_codec_info) + self.assertIsNot(codec_info, unpickled_codec_info) + self.assertEqual(codec_info, unpickled_codec_info) + self.assertEqual(codec_info.name, unpickled_codec_info.name) + self.assertEqual( + codec_info.incrementalencoder, + unpickled_codec_info.incrementalencoder + ) + self.assertTrue(unpickled_codec_info._is_text_encoding) + + # Test a CodecInfo with _is_text_encoding equal to false. + codec_info = codecs.lookup('base64') + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + pickled_codec_info = pickle.dumps(codec_info) + unpickled_codec_info = pickle.loads(pickled_codec_info) + self.assertIsNot(codec_info, unpickled_codec_info) + self.assertEqual(codec_info, unpickled_codec_info) + self.assertEqual(codec_info.name, unpickled_codec_info.name) + self.assertEqual( + codec_info.incrementalencoder, + unpickled_codec_info.incrementalencoder + ) + self.assertFalse(unpickled_codec_info._is_text_encoding) + class StreamReaderTest(unittest.TestCase): @@ -1979,14 +2122,14 @@ class StreamWriterTest(unittest.TestCase): def setUp(self): self.writer = codecs.getwriter('utf-8') - + def test_copy(self): f = self.writer(Queue(b'')) with self.assertRaisesRegex(TypeError, 'StreamWriter'): copy.copy(f) with self.assertRaisesRegex(TypeError, 'StreamWriter'): copy.deepcopy(f) - + def test_pickle(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.subTest(protocol=proto): @@ -2007,7 +2150,7 @@ def test_copy(self): copy.copy(f) with self.assertRaisesRegex(TypeError, 'StreamReaderWriter'): copy.deepcopy(f) - + def test_pickle(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.subTest(protocol=proto): @@ -2148,8 +2291,7 @@ def test_basic(self): class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_basics(self): s = "abc123" # all codecs should be able to encode these for encoding in all_unicode_encodings: @@ -2231,14 +2373,14 @@ def test_basics(self): "encoding=%r" % encoding) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_basics_capi(self): s = "abc123" # all codecs should be able to encode these for encoding in all_unicode_encodings: if encoding not in broken_unicode_with_stateful: # check incremental decoder/encoder (fetched via the C API) try: - cencoder = _testcapi.codec_incrementalencoder(encoding) + cencoder = _testlimitedcapi.codec_incrementalencoder(encoding) except LookupError: # no IncrementalEncoder pass else: @@ -2247,7 +2389,7 @@ def test_basics_capi(self): for c in s: encodedresult += cencoder.encode(c) encodedresult += cencoder.encode("", True) - cdecoder = _testcapi.codec_incrementaldecoder(encoding) + cdecoder = _testlimitedcapi.codec_incrementaldecoder(encoding) decodedresult = "" for c in encodedresult: decodedresult += cdecoder.decode(bytes([c])) @@ -2258,19 +2400,18 @@ def test_basics_capi(self): if encoding not in ("idna", "mbcs"): # check incremental decoder/encoder with errors argument try: - cencoder = _testcapi.codec_incrementalencoder(encoding, "ignore") + cencoder = _testlimitedcapi.codec_incrementalencoder(encoding, "ignore") except LookupError: # no IncrementalEncoder pass else: encodedresult = b"".join(cencoder.encode(c) for c in s) - cdecoder = _testcapi.codec_incrementaldecoder(encoding, "ignore") + cdecoder = _testlimitedcapi.codec_incrementaldecoder(encoding, "ignore") decodedresult = "".join(cdecoder.decode(bytes([c])) for c in encodedresult) self.assertEqual(decodedresult, s, "encoding=%r" % encoding) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_seek(self): # all codecs should be able to encode these s = "%s\n%s\n" % (100*"abc123", 100*"def456") @@ -2286,8 +2427,7 @@ def test_seek(self): data = reader.read() self.assertEqual(s, data) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bad_decode_args(self): for encoding in all_unicode_encodings: decoder = codecs.getdecoder(encoding) @@ -2295,8 +2435,7 @@ def test_bad_decode_args(self): if encoding not in ("idna", "punycode"): self.assertRaises(TypeError, decoder, 42) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bad_encode_args(self): for encoding in all_unicode_encodings: encoder = codecs.getencoder(encoding) @@ -2308,8 +2447,7 @@ def test_encoding_map_type_initialized(self): table_type = type(cp1140.encoding_table) self.assertEqual(table_type, table_type) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_decoder_state(self): # Check that getstate() and setstate() handle the state properly u = "abc123" @@ -2320,8 +2458,7 @@ def test_decoder_state(self): class CharmapTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_decode_with_string_map(self): self.assertEqual( codecs.charmap_decode(b"\x00\x01\x02", "strict", "abc"), @@ -2377,8 +2514,7 @@ def test_decode_with_string_map(self): ("", len(allbytes)) ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_decode_with_int2str_map(self): self.assertEqual( codecs.charmap_decode(b"\x00\x01\x02", "strict", @@ -2495,8 +2631,7 @@ def test_decode_with_int2str_map(self): b"\x00\x01\x02", "strict", {0: "A", 1: 'Bb', 2: 999999999} ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_decode_with_int2int_map(self): a = ord('a') b = ord('b') @@ -2589,8 +2724,7 @@ def test_streamreaderwriter(self): class TypesTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_decode_unicode(self): # Most decoders don't accept unicode input decoders = [ @@ -2636,16 +2770,6 @@ class UnicodeEscapeTest(ReadTest, unittest.TestCase): test_lone_surrogates = None - # TODO: RUSTPYTHON, TypeError: Expected type 'str', not 'bytes' - @unittest.expectedFailure - def test_incremental_surrogatepass(self): # TODO: RUSTPYTHON, remove when this passes - super().test_incremental_surrogatepass() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_readline(self): # TODO: RUSTPYTHON, remove when this passes - super().test_readline() # TODO: RUSTPYTHON, remove when this passes - def test_empty(self): self.assertEqual(codecs.unicode_escape_encode(""), (b"", 0)) self.assertEqual(codecs.unicode_escape_decode(b""), ("", 0)) @@ -2677,8 +2801,6 @@ def test_escape_encode(self): check('\u20ac', br'\u20ac') check('\U0001d120', br'\U0001d120') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_escape_decode(self): decode = codecs.unicode_escape_decode check = coding_checker(self, decode) @@ -2703,24 +2825,41 @@ def test_escape_decode(self): check(br"[\x410]", "[A0]") check(br"\u20ac", "\u20ac") check(br"\U0001d120", "\U0001d120") + + @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered + def test_decode_warnings(self): + decode = codecs.unicode_escape_decode + check = coding_checker(self, decode) for i in range(97, 123): b = bytes([i]) if b not in b'abfnrtuvx': - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r"invalid escape sequence '\\%c'" % i): check(b"\\" + b, "\\" + chr(i)) if b.upper() not in b'UN': - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r"invalid escape sequence '\\%c'" % (i-32)): check(b"\\" + b.upper(), "\\" + chr(i-32)) - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r"invalid escape sequence '\\8'"): check(br"\8", "\\8") with self.assertWarns(DeprecationWarning): check(br"\9", "\\9") - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r"invalid escape sequence '\\\xfa'") as cm: check(b"\\\xfa", "\\\xfa") for i in range(0o400, 0o1000): - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r"invalid octal escape sequence '\\%o'" % i): check(rb'\%o' % i, chr(i)) + with self.assertWarnsRegex(DeprecationWarning, + r"invalid escape sequence '\\z'"): + self.assertEqual(decode(br'\x\z', 'ignore'), ('\\z', 4)) + with self.assertWarnsRegex(DeprecationWarning, + r"invalid octal escape sequence '\\501'"): + self.assertEqual(decode(br'\x\501', 'ignore'), ('\u0141', 6)) + def test_decode_errors(self): decode = codecs.unicode_escape_decode for c, d in (b'x', 2), (b'u', 4), (b'U', 4): @@ -2737,8 +2876,7 @@ def test_decode_errors(self): self.assertEqual(decode(br"\U00110000", "ignore"), ("", 10)) self.assertEqual(decode(br"\U00110000", "replace"), ("\ufffd", 10)) - # TODO: RUSTPYTHON, UnicodeDecodeError: ('unicodeescape', b'\\', 0, 1, '\\ at end of string') - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: '\x00\t\n\r\\' != '\x00\t\n\r' def test_partial(self): self.check_partial( "\x00\t\n\r\\\xff\uffff\U00010000", @@ -2778,21 +2916,19 @@ def test_partial(self): ] ) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_incremental_surrogatepass(self): + return super().test_incremental_surrogatepass() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_readline(self): + return super().test_readline() + class RawUnicodeEscapeTest(ReadTest, unittest.TestCase): encoding = "raw-unicode-escape" test_lone_surrogates = None - # TODO: RUSTPYTHON, AssertionError: '\\' != '' - @unittest.expectedFailure - def test_incremental_surrogatepass(self): # TODO: RUSTPYTHON, remove when this passes - super().test_incremental_surrogatepass() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_readline(self): # TODO: RUSTPYTHON, remove when this passes - super().test_readline() # TODO: RUSTPYTHON, remove when this passes - def test_empty(self): self.assertEqual(codecs.raw_unicode_escape_encode(""), (b"", 0)) self.assertEqual(codecs.raw_unicode_escape_decode(b""), ("", 0)) @@ -2841,8 +2977,7 @@ def test_decode_errors(self): self.assertEqual(decode(br"\U00110000", "ignore"), ("", 10)) self.assertEqual(decode(br"\U00110000", "replace"), ("\ufffd", 10)) - # TODO: RUSTPYTHON, AssertionError: '\x00\t\n\r\\' != '\x00\t\n\r' - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( "\x00\t\n\r\\\xff\uffff\U00010000", @@ -2872,6 +3007,14 @@ def test_partial(self): ] ) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_incremental_surrogatepass(self): + return super().test_incremental_surrogatepass() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_readline(self): + return super().test_readline() + class EscapeEncodeTest(unittest.TestCase): @@ -2914,8 +3057,7 @@ def test_ascii(self): self.assertEqual("foo\udc80bar".encode("ascii", "surrogateescape"), b"foo\x80bar") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_charmap(self): # bad byte: \xa5 is unmapped in iso-8859-3 self.assertEqual(b"foo\xa5bar".decode("iso-8859-3", "surrogateescape"), @@ -2930,8 +3072,7 @@ def test_latin1(self): class BomTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_seek0(self): data = "1234567890" tests = ("utf-16", @@ -3048,8 +3189,7 @@ def test_readline(self): sout = reader.readline() self.assertEqual(sout, b"\x80") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_buffer_api_usage(self): # We check all the transform codecs accept memoryview input # for encoding and decoding @@ -3112,10 +3252,8 @@ def test_binary_to_text_denylists_text_transforms(self): bad_input.decode("rot_13") self.assertIsNone(failure.exception.__cause__) - - # @unittest.skipUnless(zlib, "Requires zlib support") - # TODO: RUSTPYTHON, ^ restore once test passes - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.skipUnless(zlib, "Requires zlib support") def test_custom_zlib_error_is_noted(self): # Check zlib codec gives a good error for malformed input msg = "decoding with 'zlib_codec' codec failed" @@ -3123,8 +3261,7 @@ def test_custom_zlib_error_is_noted(self): codecs.decode(b"hello", "zlib_codec") self.assertEqual(msg, failure.exception.__notes__[0]) - # TODO: RUSTPYTHON - AttributeError: 'Error' object has no attribute '__notes__' - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; - AttributeError: 'Error' object has no attribute '__notes__' def test_custom_hex_error_is_noted(self): # Check hex codec gives a good error for malformed input import binascii @@ -3204,9 +3341,8 @@ def assertNoted(self, operation, exc_type, msg): def raise_obj(self, *args, **kwds): # Helper to dynamically change the object raised by a test codec raise self.obj_to_raise - - # TODO: RUSTPYTHON - @unittest.expectedFailure + + @unittest.expectedFailure # TODO: RUSTPYTHON def check_note(self, obj_to_raise, msg, exc_type=RuntimeError): self.obj_to_raise = obj_to_raise self.set_codec(self.raise_obj, self.raise_obj) @@ -3218,65 +3354,56 @@ def check_note(self, obj_to_raise, msg, exc_type=RuntimeError): b"bytes input".decode(self.codec_name) with self.assertNoted("decoding", exc_type, msg): codecs.decode(b"bytes input", self.codec_name) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_raise_by_type(self): self.check_note(RuntimeError, "") - - # TODO: RUSTPYTHON - @unittest.expectedFailure + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_raise_by_value(self): msg = "This should be noted" self.check_note(RuntimeError(msg), msg) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_raise_grandchild_subclass_exact_size(self): msg = "This should be noted" class MyRuntimeError(RuntimeError): __slots__ = () self.check_note(MyRuntimeError(msg), msg, MyRuntimeError) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_raise_subclass_with_weakref_support(self): msg = "This should be noted" class MyRuntimeError(RuntimeError): pass self.check_note(MyRuntimeError(msg), msg, MyRuntimeError) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_init_override(self): class CustomInit(RuntimeError): def __init__(self): pass self.check_note(CustomInit, "") - - # TODO: RUSTPYTHON - @unittest.expectedFailure + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_new_override(self): class CustomNew(RuntimeError): def __new__(cls): return super().__new__(cls) self.check_note(CustomNew, "") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_instance_attribute(self): msg = "This should be noted" exc = RuntimeError(msg) exc.attr = 1 self.check_note(exc, "^{}$".format(msg)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_non_str_arg(self): self.check_note(RuntimeError(1), "1") - - # TODO: RUSTPYTHON - @unittest.expectedFailure + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_multiple_args(self): msg_re = r"^\('a', 'b', 'c'\)$" self.check_note(RuntimeError('a', 'b', 'c'), msg_re) @@ -3293,8 +3420,7 @@ def test_codec_lookup_failure(self): with self.assertRaisesRegex(LookupError, msg): codecs.decode(b"bytes input", self.codec_name) - - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; def test_unflagged_non_text_codec_handling(self): # The stdlib non-text codecs are now marked so they're # pre-emptively skipped by the text model related methods @@ -3330,22 +3456,20 @@ def decode_to_bytes(*args, **kwds): class CodePageTest(unittest.TestCase): CP_UTF8 = 65001 - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid_code_page(self): self.assertRaises(ValueError, codecs.code_page_encode, -1, 'a') self.assertRaises(ValueError, codecs.code_page_decode, -1, b'a') self.assertRaises(OSError, codecs.code_page_encode, 123, 'a') self.assertRaises(OSError, codecs.code_page_decode, 123, b'a') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_code_page_name(self): self.assertRaisesRegex(UnicodeEncodeError, 'cp932', codecs.code_page_encode, 932, '\xff') self.assertRaisesRegex(UnicodeDecodeError, 'cp932', codecs.code_page_decode, 932, b'\x81\x00', 'strict', True) - self.assertRaisesRegex(UnicodeDecodeError, 'CP_UTF8', + self.assertRaisesRegex(UnicodeDecodeError, 'cp65001', codecs.code_page_decode, self.CP_UTF8, b'\xff', 'strict', True) def check_decode(self, cp, tests): @@ -3382,8 +3506,7 @@ def check_encode(self, cp, tests): self.assertRaises(UnicodeEncodeError, codecs.code_page_encode, cp, text, errors) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cp932(self): self.check_encode(932, ( ('abc', 'strict', b'abc'), @@ -3417,8 +3540,7 @@ def test_cp932(self): (b'\x81\x00abc', 'backslashreplace', '\\x81\x00abc'), )) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cp1252(self): self.check_encode(1252, ( ('abc', 'strict', b'abc'), @@ -3437,8 +3559,7 @@ def test_cp1252(self): (b'\xff', 'strict', '\xff'), )) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cp_utf7(self): cp = 65000 self.check_encode(cp, ( @@ -3459,8 +3580,7 @@ def test_cp_utf7(self): (b'[\xff]', 'strict', '[\xff]'), )) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_multibyte_encoding(self): self.check_decode(932, ( (b'\x84\xe9\x80', 'ignore', '\u9a3e'), @@ -3475,8 +3595,7 @@ def test_multibyte_encoding(self): ('[\U0010ffff\uDC80]', 'replace', b'[\xf4\x8f\xbf\xbf?]'), )) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_code_page_decode_flags(self): # Issue #36312: For some code pages (e.g. UTF-7) flags for # MultiByteToWideChar() must be set to 0. @@ -3496,8 +3615,7 @@ def test_code_page_decode_flags(self): self.assertEqual(codecs.code_page_decode(42, b'abc'), ('\uf061\uf062\uf063', 3)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_incremental(self): decoded = codecs.code_page_decode(932, b'\x82', 'strict', False) self.assertEqual(decoded, ('', 0)) @@ -3682,7 +3800,7 @@ def test_seeking_write(self): self.assertEqual(sr.readline(), b'1\n') self.assertEqual(sr.readline(), b'abc\n') self.assertEqual(sr.readline(), b'789\n') - + def test_copy(self): bio = io.BytesIO() codec = codecs.lookup('ascii') @@ -3693,7 +3811,7 @@ def test_copy(self): copy.copy(sr) with self.assertRaisesRegex(TypeError, 'StreamRecoder'): copy.deepcopy(sr) - + def test_pickle(self): q = Queue(b'') codec = codecs.lookup('ascii') @@ -3860,8 +3978,7 @@ def test_rot13_func(self): class CodecNameNormalizationTest(unittest.TestCase): """Test codec name normalization""" - # TODO: RUSTPYTHON, AssertionError: Tuples differ: (1, 2, 3, 4) != (None, None, None, None) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: Tuples differ: (1, 2, 3, 4) != (None, None, None, None) def test_codecs_lookup(self): FOUND = (1, 2, 3, 4) NOT_FOUND = (None, None, None, None) diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 964bcc7288..7b11601fc4 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -262,8 +262,7 @@ def __contains__(self, key): d = c.new_child(b=20, c=30) self.assertEqual(d.maps, [{'b': 20, 'c': 30}, {'a': 1, 'b': 2}]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_union_operators(self): cm1 = ChainMap(dict(a=1, b=2), dict(c=3, d=4)) cm2 = ChainMap(dict(a=10, e=5), dict(b=20, d=4)) @@ -470,8 +469,7 @@ def test_module_parameter(self): NT = namedtuple('NT', ['x', 'y'], module=collections) self.assertEqual(NT.__module__, collections) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_instance(self): Point = namedtuple('Point', 'x y') p = Point(11, 22) @@ -740,7 +738,7 @@ def validate_abstract_methods(self, abc, *names): stubs = methodstubs.copy() del stubs[name] C = type('C', (abc,), stubs) - self.assertRaises(TypeError, C, name) + self.assertRaises(TypeError, C) def validate_isinstance(self, abc, name): stub = lambda s, *args: 0 @@ -790,8 +788,7 @@ def _test_gen(): class TestOneTrickPonyABCs(ABCTestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_Awaitable(self): def gen(): yield @@ -844,8 +841,7 @@ class CoroLike: pass CoroLike = None support.gc_collect() # Kill CoroLike to clean-up ABCMeta cache - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_Coroutine(self): def gen(): yield @@ -971,7 +967,7 @@ class AnextOnly: async def __anext__(self): raise StopAsyncIteration self.assertNotIsInstance(AnextOnly(), AsyncIterator) - self.validate_abstract_methods(AsyncIterator, '__anext__', '__aiter__') + self.validate_abstract_methods(AsyncIterator, '__anext__') def test_Iterable(self): # Check some non-iterables @@ -1168,7 +1164,7 @@ def test_Iterator(self): for x in samples: self.assertIsInstance(x, Iterator) self.assertTrue(issubclass(type(x), Iterator), repr(type(x))) - self.validate_abstract_methods(Iterator, '__next__', '__iter__') + self.validate_abstract_methods(Iterator, '__next__') # Issue 10565 class NextOnly: @@ -1852,8 +1848,7 @@ def test_Mapping(self): for sample in [dict]: self.assertIsInstance(sample(), Mapping) self.assertTrue(issubclass(sample, Mapping)) - self.validate_abstract_methods(Mapping, '__contains__', '__iter__', '__len__', - '__getitem__') + self.validate_abstract_methods(Mapping, '__iter__', '__len__', '__getitem__') class MyMapping(Mapping): def __len__(self): return 0 @@ -1868,7 +1863,7 @@ def test_MutableMapping(self): for sample in [dict]: self.assertIsInstance(sample(), MutableMapping) self.assertTrue(issubclass(sample, MutableMapping)) - self.validate_abstract_methods(MutableMapping, '__contains__', '__iter__', '__len__', + self.validate_abstract_methods(MutableMapping, '__iter__', '__len__', '__getitem__', '__setitem__', '__delitem__') def test_MutableMapping_subclass(self): @@ -1907,8 +1902,7 @@ def test_Sequence(self): self.assertIsInstance(memoryview(b""), Sequence) self.assertTrue(issubclass(memoryview, Sequence)) self.assertTrue(issubclass(str, Sequence)) - self.validate_abstract_methods(Sequence, '__contains__', '__iter__', '__len__', - '__getitem__') + self.validate_abstract_methods(Sequence, '__len__', '__getitem__') def test_Sequence_mixins(self): class SequenceSubclass(Sequence): @@ -1967,10 +1961,7 @@ class X(ByteString): pass # No metaclass conflict class Z(ByteString, Awaitable): pass - # TODO: RUSTPYTHON - # Need to implement __buffer__ and __release_buffer__ - # https://docs.python.org/3.13/reference/datamodel.html#emulating-buffer-types - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; Need to implement __buffer__ and __release_buffer__ (https://docs.python.org/3.13/reference/datamodel.html#emulating-buffer-types) def test_Buffer(self): for sample in [bytes, bytearray, memoryview]: self.assertIsInstance(sample(b"x"), Buffer) @@ -1989,8 +1980,8 @@ def test_MutableSequence(self): self.assertTrue(issubclass(sample, MutableSequence)) self.assertTrue(issubclass(array.array, MutableSequence)) self.assertFalse(issubclass(str, MutableSequence)) - self.validate_abstract_methods(MutableSequence, '__contains__', '__iter__', - '__len__', '__getitem__', '__setitem__', '__delitem__', 'insert') + self.validate_abstract_methods(MutableSequence, '__len__', '__getitem__', + '__setitem__', '__delitem__', 'insert') def test_MutableSequence_mixins(self): # Test the mixins of MutableSequence by creating a minimal concrete @@ -2042,8 +2033,7 @@ def insert(self, index, value): self.assertEqual(len(mss), len(mss2)) self.assertEqual(list(mss), list(mss2)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_illegal_patma_flags(self): with self.assertRaises(TypeError): class Both(Collection): diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 5b07b3c85b..27bbe0b64a 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -885,8 +885,6 @@ def foo(x): self.assertIn('LOAD_ATTR', instructions) self.assertIn('PRECALL', instructions) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_lineno_procedure_call(self): def call(): ( diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 95cf51bf08..b7f93d1bac 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -86,14 +86,12 @@ def _test_arg_valid(self, ctor, arg): self.assertRaises(ValueError, ctor, arg, quotechar='\x85', lineterminator='\x85') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_reader_arg_valid(self): self._test_arg_valid(csv.reader, []) self.assertRaises(OSError, csv.reader, BadIterable()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_writer_arg_valid(self): self._test_arg_valid(csv.writer, StringIO()) class BadWriter: @@ -214,8 +212,7 @@ def test_write_bigfield(self): self._write_test([bigstring,bigstring], '%s,%s' % \ (bigstring, bigstring)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_quoting(self): self._write_test(['a',1,'p,q'], 'a,1,"p,q"') self._write_error_test(csv.Error, ['a',1,'p,q'], @@ -233,8 +230,7 @@ def test_write_quoting(self): self._write_test(['a','',None,1], '"a","",,"1"', quoting = csv.QUOTE_NOTNULL) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_escape(self): self._write_test(['a',1,'p,q'], 'a,1,"p,q"', escapechar='\\') @@ -266,8 +262,7 @@ def test_write_escape(self): self._write_test(['C\\', '6', '7', 'X"'], 'C\\\\,6,7,"X"""', escapechar='\\', quoting=csv.QUOTE_MINIMAL) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_lineterminator(self): for lineterminator in '\r\n', '\n', '\r', '!@#', '\0': with self.subTest(lineterminator=lineterminator): @@ -281,8 +276,7 @@ def test_write_lineterminator(self): f'1,2{lineterminator}' f'"\r","\n"{lineterminator}') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_iterable(self): self._write_test(iter(['a', 1, 'p,q']), 'a,1,"p,q"') self._write_test(iter(['a', 1, None]), 'a,1,') @@ -325,8 +319,7 @@ def test_writerows_with_none(self): self.assertEqual(fileobj.read(), 'a\r\n""\r\n') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_empty_fields(self): self._write_test((), '') self._write_test([''], '""') @@ -340,8 +333,7 @@ def test_write_empty_fields(self): self._write_test(['', ''], ',') self._write_test([None, None], ',') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_empty_fields_space_delimiter(self): self._write_test([''], '""', delimiter=' ', skipinitialspace=False) self._write_test([''], '""', delimiter=' ', skipinitialspace=True) @@ -382,8 +374,7 @@ def _read_test(self, input, expect, **kwargs): result = list(reader) self.assertEqual(result, expect) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_oddinputs(self): self._read_test([], []) self._read_test([''], [[]]) @@ -394,8 +385,7 @@ def test_read_oddinputs(self): self.assertRaises(csv.Error, self._read_test, [b'abc'], None) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_eol(self): self._read_test(['a,b', 'c,d'], [['a','b'], ['c','d']]) self._read_test(['a,b\n', 'c,d\n'], [['a','b'], ['c','d']]) @@ -410,8 +400,7 @@ def test_read_eol(self): with self.assertRaisesRegex(csv.Error, errmsg): next(csv.reader(['a,b\r\nc,d'])) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_eof(self): self._read_test(['a,"'], [['a', '']]) self._read_test(['"a'], [['a']]) @@ -421,8 +410,7 @@ def test_read_eof(self): self.assertRaises(csv.Error, self._read_test, ['^'], [], escapechar='^', strict=True) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_nul(self): self._read_test(['\0'], [['\0']]) self._read_test(['a,\0b,c'], [['a', '\0b', 'c']]) @@ -435,8 +423,7 @@ def test_read_delimiter(self): self._read_test(['a;b;c'], [['a', 'b', 'c']], delimiter=';') self._read_test(['a\0b\0c'], [['a', 'b', 'c']], delimiter='\0') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_escape(self): self._read_test(['a,\\b,c'], [['a', 'b', 'c']], escapechar='\\') self._read_test(['a,b\\,c'], [['a', 'b,c']], escapechar='\\') @@ -449,8 +436,7 @@ def test_read_escape(self): self._read_test(['a,\\b,c'], [['a', '\\b', 'c']], escapechar=None) self._read_test(['a,\\b,c'], [['a', '\\b', 'c']]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_quoting(self): self._read_test(['1,",3,",5'], [['1', ',3,', '5']]) self._read_test(['1,",3,",5'], [['1', '"', '3', '"', '5']], @@ -487,8 +473,7 @@ def test_read_quoting(self): self._read_test(['1\\.5,\\.5,"\\.5"'], [[1.5, 0.5, ".5"]], quoting=csv.QUOTE_STRINGS, escapechar='\\') - # TODO: RUSTPYTHON; panic - @unittest.skip("TODO: RUSTPYTHON; slice index starts at 1 but ends at 0") + @unittest.skip('TODO: RUSTPYTHON; slice index starts at 1 but ends at 0') def test_read_skipinitialspace(self): self._read_test(['no space, space, spaces,\ttab'], [['no space', 'space', 'spaces', '\ttab']], @@ -503,8 +488,7 @@ def test_read_skipinitialspace(self): [[None, None, None]], skipinitialspace=True, quoting=csv.QUOTE_STRINGS) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_space_delimiter(self): self._read_test(['a b', ' a ', ' ', ''], [['a', '', '', 'b'], ['', '', 'a', '', ''], ['', '', ''], []], @@ -544,8 +528,7 @@ def test_read_linenum(self): self.assertRaises(StopIteration, next, r) self.assertEqual(r.line_num, 3) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_roundtrip_quoteed_newlines(self): rows = [ ['\na', 'b\nc', 'd\n'], @@ -564,8 +547,7 @@ def test_roundtrip_quoteed_newlines(self): for i, row in enumerate(csv.reader(fileobj)): self.assertEqual(row, rows[i]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_roundtrip_escaped_unquoted_newlines(self): rows = [ ['\na', 'b\nc', 'd\n'], @@ -680,8 +662,7 @@ def compare_dialect_123(self, expected, *writeargs, **kwwriteargs): fileobj.seek(0) self.assertEqual(fileobj.read(), expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_dialect_apply(self): class testA(csv.excel): delimiter = "\t" @@ -717,8 +698,7 @@ def test_copy(self): dialect = csv.get_dialect(name) self.assertRaises(TypeError, copy.copy, dialect) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_pickle(self): for name in csv.list_dialects(): dialect = csv.get_dialect(name) @@ -805,8 +785,7 @@ def test_quoted_quote(self): '"I see," said the blind man', 'as he picked up his hammer and saw']]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_quoted_nl(self): input = '''\ 1,2,3,"""I see,"" @@ -847,21 +826,18 @@ class EscapedExcel(csv.excel): class TestEscapedExcel(TestCsvBase): dialect = EscapedExcel() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_escape_fieldsep(self): self.writerAssertEqual([['abc,def']], 'abc\\,def\r\n') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_escape_fieldsep(self): self.readerAssertEqual('abc\\,def\r\n', [['abc,def']]) class TestDialectUnix(TestCsvBase): dialect = 'unix' - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_simple_writer(self): self.writerAssertEqual([[1, 'abc def', 'abc']], '"1","abc def","abc"\n') @@ -878,8 +854,7 @@ class TestQuotedEscapedExcel(TestCsvBase): def test_write_escape_fieldsep(self): self.writerAssertEqual([['abc,def']], '"abc,def"\r\n') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_escape_fieldsep(self): self.readerAssertEqual('"abc\\,def"\r\n', [['abc,def']]) @@ -1076,8 +1051,7 @@ def test_read_multi(self): "s1": 'abc', "s2": 'def'}) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_with_blanks(self): reader = csv.DictReader(["1,2,abc,4,5,6\r\n","\r\n", "1,2,abc,4,5,6\r\n"], @@ -1129,8 +1103,7 @@ def test_float_write(self): fileobj.seek(0) self.assertEqual(fileobj.read(), expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_char_write(self): import array, string a = array.array('w', string.ascii_letters) @@ -1278,8 +1251,7 @@ class mydialect(csv.Dialect): self.assertEqual(str(cm.exception), '"lineterminator" must be a string') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid_chars(self): def create_invalid(field_name, value, **kwargs): class mydialect(csv.Dialect): diff --git a/Lib/test/test_dtrace.py b/Lib/test/test_dtrace.py index e1adf8e974..a63978fd1b 100644 --- a/Lib/test/test_dtrace.py +++ b/Lib/test/test_dtrace.py @@ -159,11 +159,43 @@ class DTraceNormalTests(TraceTests, unittest.TestCase): backend = DTraceBackend() optimize_python = 0 + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON') + def test_function_entry_return(self): + return super().test_function_entry_return() + + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON') + def test_verify_call_opcodes(self): + return super().test_verify_call_opcodes() + + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON') + def test_gc(self): + return super().test_gc() + + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON') + def test_line(self): + return super().test_line() + class DTraceOptimizedTests(TraceTests, unittest.TestCase): backend = DTraceBackend() optimize_python = 2 + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON') + def test_function_entry_return(self): + return super().test_function_entry_return() + + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON') + def test_verify_call_opcodes(self): + return super().test_verify_call_opcodes() + + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON') + def test_gc(self): + return super().test_gc() + + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON') + def test_line(self): + return super().test_line() + class SystemTapNormalTests(TraceTests, unittest.TestCase): backend = SystemTapBackend() diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index d7e2c6a1de..1225db997e 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -979,6 +979,7 @@ def test_cancel_later_without_dump_traceback_later(self): self.assertEqual(output, []) self.assertEqual(exitcode, 0) + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AttributeError: module 'msvcrt' has no attribute 'GetErrorMode'") @threading_helper.requires_working_threading() @unittest.skipUnless(support.Py_GIL_DISABLED, "only meaningful if the GIL is disabled") def test_free_threaded_dump_traceback(self): diff --git a/Lib/test/test_file.py b/Lib/test/test_file.py index d998af936a..d64f3f797d 100644 --- a/Lib/test/test_file.py +++ b/Lib/test/test_file.py @@ -344,10 +344,9 @@ def testIteration(self): class COtherFileTests(OtherFileTests, unittest.TestCase): open = io.open - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def testSetBufferSize(self): - super().testSetBufferSize() + return super().testSetBufferSize() class PyOtherFileTests(OtherFileTests, unittest.TestCase): open = staticmethod(pyio.open) diff --git a/Lib/test/test_fnmatch.py b/Lib/test/test_fnmatch.py index 092cd56285..b977b8f8eb 100644 --- a/Lib/test/test_fnmatch.py +++ b/Lib/test/test_fnmatch.py @@ -71,7 +71,7 @@ def test_fnmatchcase(self): check('usr/bin', 'usr\\bin', False, fnmatchcase) check('usr\\bin', 'usr\\bin', True, fnmatchcase) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON') def test_bytes(self): self.check_match(b'test', b'te*') self.check_match(b'test\xff', b'te*\xff') @@ -239,17 +239,9 @@ def test_translate(self): self.assertEqual(translate('A*********?[?]?'), r'(?s:A.*.[?].)\Z') # fancy translation to prevent exponential-time match failure t = translate('**a*a****a') - digits = re.findall(r'\d+', t) - self.assertEqual(len(digits), 4) - self.assertEqual(digits[0], digits[1]) - self.assertEqual(digits[2], digits[3]) - g1 = f"g{digits[0]}" # e.g., group name "g4" - g2 = f"g{digits[2]}" # e.g., group name "g5" - self.assertEqual(t, - fr'(?s:(?=(?P<{g1}>.*?a))(?P={g1})(?=(?P<{g2}>.*?a))(?P={g2}).*a)\Z') + self.assertEqual(t, r'(?s:(?>.*?a)(?>.*?a).*a)\Z') # and try pasting multiple translate results - it's an undocumented - # feature that this works; all the pain of generating unique group - # names across calls exists to support this + # feature that this works r1 = translate('**a**a**a*') r2 = translate('**b**b**b*') r3 = translate('*c*c*c*') diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 4996eedc1c..229f16393f 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -9,6 +9,7 @@ import ast import datetime +import dis import os import re import types @@ -19,7 +20,7 @@ from test.support.os_helper import temp_cwd from test.support.script_helper import assert_python_failure, assert_python_ok -a_global = "global variable" +a_global = 'global variable' # You could argue that I'm too strict in looking for specific error # values with assertRaisesRegex, but without it it's way too easy to @@ -28,7 +29,6 @@ # worthwhile tradeoff. When I switched to this method, I found many # examples where I wasn't testing what I thought I was. - class TestCase(unittest.TestCase): def assertAllRaise(self, exception_type, regex, error_strings): for str in error_strings: @@ -40,45 +40,43 @@ def test__format__lookup(self): # Make sure __format__ is looked up on the type, not the instance. class X: def __format__(self, spec): - return "class" + return 'class' x = X() # Add a bound __format__ method to the 'y' instance, but not # the 'x' instance. y = X() - y.__format__ = types.MethodType(lambda self, spec: "instance", y) + y.__format__ = types.MethodType(lambda self, spec: 'instance', y) - self.assertEqual(f"{y}", format(y)) - self.assertEqual(f"{y}", "class") + self.assertEqual(f'{y}', format(y)) + self.assertEqual(f'{y}', 'class') self.assertEqual(format(x), format(y)) # __format__ is not called this way, but still make sure it # returns what we expect (so we can make sure we're bypassing # it). - self.assertEqual(x.__format__(""), "class") - self.assertEqual(y.__format__(""), "instance") + self.assertEqual(x.__format__(''), 'class') + self.assertEqual(y.__format__(''), 'instance') # This is how __format__ is actually called. - self.assertEqual(type(x).__format__(x, ""), "class") - self.assertEqual(type(y).__format__(y, ""), "class") + self.assertEqual(type(x).__format__(x, ''), 'class') + self.assertEqual(type(y).__format__(y, ''), 'class') def test_ast(self): # Inspired by http://bugs.python.org/issue24975 class X: def __init__(self): self.called = False - def __call__(self): self.called = True return 4 - x = X() expr = """ a = 10 f'{a * x()}'""" t = ast.parse(expr) - c = compile(t, "", "exec") + c = compile(t, '', 'exec') # Make sure x was not called. self.assertFalse(x.called) @@ -284,6 +282,7 @@ def test_ast_line_numbers_duplicate_expression(self): self.assertEqual(binop.right.col_offset, 27) def test_ast_numbers_fstring_with_formatting(self): + t = ast.parse('f"Here is that pesky {xxx:.3f} again"') self.assertEqual(len(t.body), 1) self.assertEqual(t.body[0].lineno, 1) @@ -384,8 +383,7 @@ def test_ast_line_numbers_multiline_fstring(self): self.assertEqual(t.body[0].value.values[1].value.col_offset, 11) self.assertEqual(t.body[0].value.values[1].value.end_col_offset, 16) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_ast_line_numbers_with_parentheses(self): expr = """ x = ( @@ -441,12 +439,24 @@ def test_ast_line_numbers_with_parentheses(self): x, y = t.body # Check the single quoted string offsets first. - offsets = [(elt.col_offset, elt.end_col_offset) for elt in x.value.elts] - self.assertTrue(all(offset == (4, 10) for offset in offsets)) + offsets = [ + (elt.col_offset, elt.end_col_offset) + for elt in x.value.elts + ] + self.assertTrue(all( + offset == (4, 10) + for offset in offsets + )) # Check the triple quoted string offsets. - offsets = [(elt.col_offset, elt.end_col_offset) for elt in y.value.elts] - self.assertTrue(all(offset == (4, 14) for offset in offsets)) + offsets = [ + (elt.col_offset, elt.end_col_offset) + for elt in y.value.elts + ] + self.assertTrue(all( + offset == (4, 14) + for offset in offsets + )) expr = """ x = ( @@ -507,612 +517,530 @@ def test_ast_fstring_empty_format_spec(self): self.assertEqual(type(format_spec), ast.JoinedStr) self.assertEqual(len(format_spec.values), 0) + def test_ast_fstring_format_spec(self): + expr = "f'{1:{name}}'" + + mod = ast.parse(expr) + self.assertEqual(type(mod), ast.Module) + self.assertEqual(len(mod.body), 1) + + fstring = mod.body[0].value + self.assertEqual(type(fstring), ast.JoinedStr) + self.assertEqual(len(fstring.values), 1) + + fv = fstring.values[0] + self.assertEqual(type(fv), ast.FormattedValue) + + format_spec = fv.format_spec + self.assertEqual(type(format_spec), ast.JoinedStr) + self.assertEqual(len(format_spec.values), 1) + + format_spec_value = format_spec.values[0] + self.assertEqual(type(format_spec_value), ast.FormattedValue) + self.assertEqual(format_spec_value.value.id, 'name') + + expr = "f'{1:{name1}{name2}}'" + + mod = ast.parse(expr) + self.assertEqual(type(mod), ast.Module) + self.assertEqual(len(mod.body), 1) + + fstring = mod.body[0].value + self.assertEqual(type(fstring), ast.JoinedStr) + self.assertEqual(len(fstring.values), 1) + + fv = fstring.values[0] + self.assertEqual(type(fv), ast.FormattedValue) + + format_spec = fv.format_spec + self.assertEqual(type(format_spec), ast.JoinedStr) + self.assertEqual(len(format_spec.values), 2) + + format_spec_value = format_spec.values[0] + self.assertEqual(type(format_spec_value), ast.FormattedValue) + self.assertEqual(format_spec_value.value.id, 'name1') + + format_spec_value = format_spec.values[1] + self.assertEqual(type(format_spec_value), ast.FormattedValue) + self.assertEqual(format_spec_value.value.id, 'name2') + + def test_docstring(self): def f(): - f"""Not a docstring""" - + f'''Not a docstring''' self.assertIsNone(f.__doc__) - def g(): - """Not a docstring""" f"" - + '''Not a docstring''' \ + f'' self.assertIsNone(g.__doc__) def test_literal_eval(self): - with self.assertRaisesRegex(ValueError, "malformed node or string"): + with self.assertRaisesRegex(ValueError, 'malformed node or string'): ast.literal_eval("f'x'") def test_ast_compile_time_concat(self): - x = [""] + x = [''] expr = """x[0] = 'foo' f'{3}'""" t = ast.parse(expr) - c = compile(t, "", "exec") + c = compile(t, '', 'exec') exec(c) - self.assertEqual(x[0], "foo3") + self.assertEqual(x[0], 'foo3') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_compile_time_concat_errors(self): - self.assertAllRaise( - SyntaxError, - "cannot mix bytes and nonbytes literals", - [ - r"""f'' b''""", - r"""b'' f''""", - ], - ) + self.assertAllRaise(SyntaxError, + 'cannot mix bytes and nonbytes literals', + [r"""f'' b''""", + r"""b'' f''""", + ]) def test_literal(self): - self.assertEqual(f"", "") - self.assertEqual(f"a", "a") - self.assertEqual(f" ", " ") + self.assertEqual(f'', '') + self.assertEqual(f'a', 'a') + self.assertEqual(f' ', ' ') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_unterminated_string(self): - self.assertAllRaise( - SyntaxError, - "unterminated string", - [ - r"""f'{"x'""", - r"""f'{"x}'""", - r"""f'{("x'""", - r"""f'{("x}'""", - ], - ) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + self.assertAllRaise(SyntaxError, 'unterminated string', + [r"""f'{"x'""", + r"""f'{"x}'""", + r"""f'{("x'""", + r"""f'{("x}'""", + ]) + + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_mismatched_parens(self): - self.assertAllRaise( - SyntaxError, - r"closing parenthesis '\}' " r"does not match opening parenthesis '\('", - [ - "f'{((}'", - ], - ) - self.assertAllRaise( - SyntaxError, - r"closing parenthesis '\)' " r"does not match opening parenthesis '\['", - [ - "f'{a[4)}'", - ], - ) - self.assertAllRaise( - SyntaxError, - r"closing parenthesis '\]' " r"does not match opening parenthesis '\('", - [ - "f'{a(4]}'", - ], - ) - self.assertAllRaise( - SyntaxError, - r"closing parenthesis '\}' " r"does not match opening parenthesis '\['", - [ - "f'{a[4}'", - ], - ) - self.assertAllRaise( - SyntaxError, - r"closing parenthesis '\}' " r"does not match opening parenthesis '\('", - [ - "f'{a(4}'", - ], - ) - self.assertRaises(SyntaxError, eval, "f'{" + "(" * 500 + "}'") - - # TODO: RUSTPYTHON - @unittest.expectedFailure + self.assertAllRaise(SyntaxError, r"closing parenthesis '\}' " + r"does not match opening parenthesis '\('", + ["f'{((}'", + ]) + self.assertAllRaise(SyntaxError, r"closing parenthesis '\)' " + r"does not match opening parenthesis '\['", + ["f'{a[4)}'", + ]) + self.assertAllRaise(SyntaxError, r"closing parenthesis '\]' " + r"does not match opening parenthesis '\('", + ["f'{a(4]}'", + ]) + self.assertAllRaise(SyntaxError, r"closing parenthesis '\}' " + r"does not match opening parenthesis '\['", + ["f'{a[4}'", + ]) + self.assertAllRaise(SyntaxError, r"closing parenthesis '\}' " + r"does not match opening parenthesis '\('", + ["f'{a(4}'", + ]) + self.assertRaises(SyntaxError, eval, "f'{" + "("*500 + "}'") + + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_fstring_nested_too_deeply(self): - self.assertAllRaise( - SyntaxError, - "f-string: expressions nested too deeply", - ['f"{1+2:{1+2:{1+1:{1}}}}"'], - ) + self.assertAllRaise(SyntaxError, + "f-string: expressions nested too deeply", + ['f"{1+2:{1+2:{1+1:{1}}}}"']) def create_nested_fstring(n): if n == 0: return "1+1" - prev = create_nested_fstring(n - 1) + prev = create_nested_fstring(n-1) return f'f"{{{prev}}}"' - self.assertAllRaise( - SyntaxError, "too many nested f-strings", [create_nested_fstring(160)] - ) + self.assertAllRaise(SyntaxError, + "too many nested f-strings", + [create_nested_fstring(160)]) def test_syntax_error_in_nested_fstring(self): # See gh-104016 for more information on this crash - self.assertAllRaise( - SyntaxError, "invalid syntax", ['f"{1 1:' + ('{f"1:' * 199)] - ) + self.assertAllRaise(SyntaxError, + "invalid syntax", + ['f"{1 1:' + ('{f"1:' * 199)]) def test_double_braces(self): - self.assertEqual(f"{{", "{") - self.assertEqual(f"a{{", "a{") - self.assertEqual(f"{{b", "{b") - self.assertEqual(f"a{{b", "a{b") - self.assertEqual(f"}}", "}") - self.assertEqual(f"a}}", "a}") - self.assertEqual(f"}}b", "}b") - self.assertEqual(f"a}}b", "a}b") - self.assertEqual(f"{{}}", "{}") - self.assertEqual(f"a{{}}", "a{}") - self.assertEqual(f"{{b}}", "{b}") - self.assertEqual(f"{{}}c", "{}c") - self.assertEqual(f"a{{b}}", "a{b}") - self.assertEqual(f"a{{}}c", "a{}c") - self.assertEqual(f"{{b}}c", "{b}c") - self.assertEqual(f"a{{b}}c", "a{b}c") - - self.assertEqual(f"{{{10}", "{10") - self.assertEqual(f"}}{10}", "}10") - self.assertEqual(f"}}{{{10}", "}{10") - self.assertEqual(f"}}a{{{10}", "}a{10") - - self.assertEqual(f"{10}{{", "10{") - self.assertEqual(f"{10}}}", "10}") - self.assertEqual(f"{10}}}{{", "10}{") - self.assertEqual(f"{10}}}a{{" "}", "10}a{}") + self.assertEqual(f'{{', '{') + self.assertEqual(f'a{{', 'a{') + self.assertEqual(f'{{b', '{b') + self.assertEqual(f'a{{b', 'a{b') + self.assertEqual(f'}}', '}') + self.assertEqual(f'a}}', 'a}') + self.assertEqual(f'}}b', '}b') + self.assertEqual(f'a}}b', 'a}b') + self.assertEqual(f'{{}}', '{}') + self.assertEqual(f'a{{}}', 'a{}') + self.assertEqual(f'{{b}}', '{b}') + self.assertEqual(f'{{}}c', '{}c') + self.assertEqual(f'a{{b}}', 'a{b}') + self.assertEqual(f'a{{}}c', 'a{}c') + self.assertEqual(f'{{b}}c', '{b}c') + self.assertEqual(f'a{{b}}c', 'a{b}c') + + self.assertEqual(f'{{{10}', '{10') + self.assertEqual(f'}}{10}', '}10') + self.assertEqual(f'}}{{{10}', '}{10') + self.assertEqual(f'}}a{{{10}', '}a{10') + + self.assertEqual(f'{10}{{', '10{') + self.assertEqual(f'{10}}}', '10}') + self.assertEqual(f'{10}}}{{', '10}{') + self.assertEqual(f'{10}}}a{{' '}', '10}a{}') # Inside of strings, don't interpret doubled brackets. - self.assertEqual(f'{"{{}}"}', "{{}}") - - self.assertAllRaise( - TypeError, - "unhashable type", - [ - "f'{ {{}} }'", # dict in a set - ], - ) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + self.assertEqual(f'{"{{}}"}', '{{}}') + + self.assertAllRaise(TypeError, 'unhashable type', + ["f'{ {{}} }'", # dict in a set + ]) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_compile_time_concat(self): - x = "def" - self.assertEqual("abc" f"## {x}ghi", "abc## defghi") - self.assertEqual("abc" f"{x}" "ghi", "abcdefghi") - self.assertEqual("abc" f"{x}" "gh" f"i{x:4}", "abcdefghidef ") - self.assertEqual("{x}" f"{x}", "{x}def") - self.assertEqual("{x" f"{x}", "{xdef") - self.assertEqual("{x}" f"{x}", "{x}def") - self.assertEqual("{{x}}" f"{x}", "{{x}}def") - self.assertEqual("{{x" f"{x}", "{{xdef") - self.assertEqual("x}}" f"{x}", "x}}def") - self.assertEqual(f"{x}" "x}}", "defx}}") - self.assertEqual(f"{x}" "", "def") - self.assertEqual("" f"{x}" "", "def") - self.assertEqual("" f"{x}", "def") - self.assertEqual(f"{x}" "2", "def2") - self.assertEqual("1" f"{x}" "2", "1def2") - self.assertEqual("1" f"{x}", "1def") - self.assertEqual(f"{x}" f"-{x}", "def-def") - self.assertEqual("" f"", "") - self.assertEqual("" f"" "", "") - self.assertEqual("" f"" "" f"", "") - self.assertEqual(f"", "") - self.assertEqual(f"" "", "") - self.assertEqual(f"" "" f"", "") - self.assertEqual(f"" "" f"" "", "") + x = 'def' + self.assertEqual('abc' f'## {x}ghi', 'abc## defghi') + self.assertEqual('abc' f'{x}' 'ghi', 'abcdefghi') + self.assertEqual('abc' f'{x}' 'gh' f'i{x:4}', 'abcdefghidef ') + self.assertEqual('{x}' f'{x}', '{x}def') + self.assertEqual('{x' f'{x}', '{xdef') + self.assertEqual('{x}' f'{x}', '{x}def') + self.assertEqual('{{x}}' f'{x}', '{{x}}def') + self.assertEqual('{{x' f'{x}', '{{xdef') + self.assertEqual('x}}' f'{x}', 'x}}def') + self.assertEqual(f'{x}' 'x}}', 'defx}}') + self.assertEqual(f'{x}' '', 'def') + self.assertEqual('' f'{x}' '', 'def') + self.assertEqual('' f'{x}', 'def') + self.assertEqual(f'{x}' '2', 'def2') + self.assertEqual('1' f'{x}' '2', '1def2') + self.assertEqual('1' f'{x}', '1def') + self.assertEqual(f'{x}' f'-{x}', 'def-def') + self.assertEqual('' f'', '') + self.assertEqual('' f'' '', '') + self.assertEqual('' f'' '' f'', '') + self.assertEqual(f'', '') + self.assertEqual(f'' '', '') + self.assertEqual(f'' '' f'', '') + self.assertEqual(f'' '' f'' '', '') # This is not really [f'{'] + [f'}'] since we treat the inside # of braces as a purely new context, so it is actually f'{ and # then eval(' f') (a valid expression) and then }' which would # constitute a valid f-string. - # TODO: RUSTPYTHON SyntaxError - # self.assertEqual(f'{' f'}', " f") - - self.assertAllRaise( - SyntaxError, - "expecting '}'", - [ - '''f'{3' f"}"''', # can't concat to get a valid f-string - ], - ) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + self.assertEqual(f'{' f'}', ' f') + + self.assertAllRaise(SyntaxError, "expecting '}'", + ['''f'{3' f"}"''', # can't concat to get a valid f-string + ]) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_comments(self): # These aren't comments, since they're in strings. - d = {"#": "hash"} - self.assertEqual(f'{"#"}', "#") - self.assertEqual(f'{d["#"]}', "hash") - - self.assertAllRaise( - SyntaxError, - "'{' was never closed", - [ - "f'{1#}'", # error because everything after '#' is a comment - "f'{#}'", - "f'one: {1#}'", - "f'{1# one} {2 this is a comment still#}'", - ], - ) - self.assertAllRaise( - SyntaxError, - r"f-string: unmatched '\)'", - [ - "f'{)#}'", # When wrapped in parens, this becomes - # '()#)'. Make sure that doesn't compile. - ], - ) - self.assertEqual( - f"""A complex trick: { + d = {'#': 'hash'} + self.assertEqual(f'{"#"}', '#') + self.assertEqual(f'{d["#"]}', 'hash') + + self.assertAllRaise(SyntaxError, "'{' was never closed", + ["f'{1#}'", # error because everything after '#' is a comment + "f'{#}'", + "f'one: {1#}'", + "f'{1# one} {2 this is a comment still#}'", + ]) + self.assertAllRaise(SyntaxError, r"f-string: unmatched '\)'", + ["f'{)#}'", # When wrapped in parens, this becomes + # '()#)'. Make sure that doesn't compile. + ]) + self.assertEqual(f'''A complex trick: { 2 # two -}""", - "A complex trick: 2", - ) - self.assertEqual( - f""" +}''', 'A complex trick: 2') + self.assertEqual(f''' { 40 # forty + # plus 2 # two -}""", - "\n42", - ) - self.assertEqual( - f""" +}''', '\n42') + self.assertEqual(f''' { 40 # forty + # plus 2 # two -}""", - "\n42", - ) -# TODO: RUSTPYTHON SyntaxError -# self.assertEqual( -# f""" -# # this is not a comment -# { # the following operation it's -# 3 # this is a number -# * 2}""", -# "\n# this is not a comment\n6", -# ) - self.assertEqual( - f""" +}''', '\n42') + + self.assertEqual(f''' +# this is not a comment +{ # the following operation it's +3 # this is a number +* 2}''', '\n# this is not a comment\n6') + self.assertEqual(f''' {# f'a {comment}' 86 # constant # nothing more -}""", - "\n86", - ) - - self.assertAllRaise( - SyntaxError, - r"f-string: valid expression required before '}'", - [ - """f''' +}''', '\n86') + + self.assertAllRaise(SyntaxError, r"f-string: valid expression required before '}'", + ["""f''' { # only a comment }''' -""", # this is equivalent to f'{}' - ], - ) +""", # this is equivalent to f'{}' + ]) def test_many_expressions(self): # Create a string with many expressions in it. Note that # because we have a space in here as a literal, we're actually # going to use twice as many ast nodes: one for each literal # plus one for each expression. - def build_fstr(n, extra=""): - return "f'" + ("{x} " * n) + extra + "'" + def build_fstr(n, extra=''): + return "f'" + ('{x} ' * n) + extra + "'" - x = "X" + x = 'X' width = 1 # Test around 256. for i in range(250, 260): - self.assertEqual(eval(build_fstr(i)), (x + " ") * i) + self.assertEqual(eval(build_fstr(i)), (x+' ')*i) # Test concatenating 2 largs fstrings. - self.assertEqual(eval(build_fstr(255) * 256), (x + " ") * (255 * 256)) + self.assertEqual(eval(build_fstr(255)*256), (x+' ')*(255*256)) - s = build_fstr(253, "{x:{width}} ") - self.assertEqual(eval(s), (x + " ") * 254) + s = build_fstr(253, '{x:{width}} ') + self.assertEqual(eval(s), (x+' ')*254) # Test lots of expressions and constants, concatenated. s = "f'{1}' 'x' 'y'" * 1024 - self.assertEqual(eval(s), "1xy" * 1024) + self.assertEqual(eval(s), '1xy' * 1024) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_format_specifier_expressions(self): width = 10 precision = 4 - value = decimal.Decimal("12.34567") - self.assertEqual(f"result: {value:{width}.{precision}}", "result: 12.35") - self.assertEqual(f"result: {value:{width!r}.{precision}}", "result: 12.35") - self.assertEqual( - f"result: {value:{width:0}.{precision:1}}", "result: 12.35" - ) - self.assertEqual( - f"result: {value:{1}{0:0}.{precision:1}}", "result: 12.35" - ) - self.assertEqual( - f"result: {value:{ 1}{ 0:0}.{ precision:1}}", "result: 12.35" - ) - self.assertEqual(f"{10:#{1}0x}", " 0xa") - self.assertEqual(f'{10:{"#"}1{0}{"x"}}', " 0xa") - self.assertEqual(f'{-10:-{"#"}1{0}x}', " -0xa") - self.assertEqual(f'{-10:{"-"}#{1}0{"x"}}', " -0xa") - self.assertEqual(f"{10:#{3 != {4:5} and width}x}", " 0xa") - - # TODO: RUSTPYTHON SyntaxError - # self.assertEqual( - # f"result: {value:{width:{0}}.{precision:1}}", "result: 12.35" - # ) - - - self.assertAllRaise( - SyntaxError, - "f-string: expecting ':' or '}'", - [ - """f'{"s"!r{":10"}}'""", - # This looks like a nested format spec. - ], - ) - - - self.assertAllRaise( - SyntaxError, - "f-string: expecting a valid expression after '{'", - [ # Invalid syntax inside a nested spec. - "f'{4:{/5}}'", - ], - ) - - self.assertAllRaise( - SyntaxError, - "f-string: invalid conversion character", - [ # No expansion inside conversion or for - # the : or ! itself. - """f'{"s"!{"r"}}'""", - ], - ) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + value = decimal.Decimal('12.34567') + self.assertEqual(f'result: {value:{width}.{precision}}', 'result: 12.35') + self.assertEqual(f'result: {value:{width!r}.{precision}}', 'result: 12.35') + self.assertEqual(f'result: {value:{width:0}.{precision:1}}', 'result: 12.35') + self.assertEqual(f'result: {value:{1}{0:0}.{precision:1}}', 'result: 12.35') + self.assertEqual(f'result: {value:{ 1}{ 0:0}.{ precision:1}}', 'result: 12.35') + self.assertEqual(f'{10:#{1}0x}', ' 0xa') + self.assertEqual(f'{10:{"#"}1{0}{"x"}}', ' 0xa') + self.assertEqual(f'{-10:-{"#"}1{0}x}', ' -0xa') + self.assertEqual(f'{-10:{"-"}#{1}0{"x"}}', ' -0xa') + self.assertEqual(f'{10:#{3 != {4:5} and width}x}', ' 0xa') + self.assertEqual(f'result: {value:{width:{0}}.{precision:1}}', 'result: 12.35') + + self.assertAllRaise(SyntaxError, "f-string: expecting ':' or '}'", + ["""f'{"s"!r{":10"}}'""", + # This looks like a nested format spec. + ]) + + self.assertAllRaise(SyntaxError, + "f-string: expecting a valid expression after '{'", + [# Invalid syntax inside a nested spec. + "f'{4:{/5}}'", + ]) + + self.assertAllRaise(SyntaxError, 'f-string: invalid conversion character', + [# No expansion inside conversion or for + # the : or ! itself. + """f'{"s"!{"r"}}'""", + ]) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_custom_format_specifier(self): class CustomFormat: def __format__(self, format_spec): return format_spec - self.assertEqual(f"{CustomFormat():\n}", "\n") - self.assertEqual(f"{CustomFormat():\u2603}", "☃") + self.assertEqual(f'{CustomFormat():\n}', '\n') + self.assertEqual(f'{CustomFormat():\u2603}', '☃') with self.assertWarns(SyntaxWarning): - exec(r'f"{F():¯\_(ツ)_/¯}"', {"F": CustomFormat}) + exec(r'f"{F():¯\_(ツ)_/¯}"', {'F': CustomFormat}) def test_side_effect_order(self): class X: def __init__(self): self.i = 0 - def __format__(self, spec): self.i += 1 return str(self.i) x = X() - self.assertEqual(f"{x} {x}", "1 2") + self.assertEqual(f'{x} {x}', '1 2') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_missing_expression(self): - self.assertAllRaise( - SyntaxError, - "f-string: valid expression required before '}'", - [ - "f'{}'", - "f'{ }'" "f' {} '", - "f'{10:{ }}'", - "f' { } '", - # The Python parser ignores also the following - # whitespace characters in additional to a space. - "f'''{\t\f\r\n}'''", - ], - ) - - self.assertAllRaise( - SyntaxError, - "f-string: valid expression required before '!'", - [ - "f'{!r}'", - "f'{ !r}'", - "f'{!}'", - "f'''{\t\f\r\n!a}'''", - # Catch empty expression before the - # missing closing brace. - "f'{!'", - "f'{!s:'", - # Catch empty expression before the - # invalid conversion. - "f'{!x}'", - "f'{ !xr}'", - "f'{!x:}'", - "f'{!x:a}'", - "f'{ !xr:}'", - "f'{ !xr:a}'", - ], - ) - - self.assertAllRaise( - SyntaxError, - "f-string: valid expression required before ':'", - [ - "f'{:}'", - "f'{ :!}'", - "f'{:2}'", - "f'''{\t\f\r\n:a}'''", - "f'{:'", - "F'{[F'{:'}[F'{:'}]]]", - ], - ) - - self.assertAllRaise( - SyntaxError, - "f-string: valid expression required before '='", - [ - "f'{=}'", - "f'{ =}'", - "f'{ =:}'", - "f'{ =!}'", - "f'''{\t\f\r\n=}'''", - "f'{='", - ], - ) + self.assertAllRaise(SyntaxError, + "f-string: valid expression required before '}'", + ["f'{}'", + "f'{ }'" + "f' {} '", + "f'{10:{ }}'", + "f' { } '", + + # The Python parser ignores also the following + # whitespace characters in additional to a space. + "f'''{\t\f\r\n}'''", + ]) + + self.assertAllRaise(SyntaxError, + "f-string: valid expression required before '!'", + ["f'{!r}'", + "f'{ !r}'", + "f'{!}'", + "f'''{\t\f\r\n!a}'''", + + # Catch empty expression before the + # missing closing brace. + "f'{!'", + "f'{!s:'", + + # Catch empty expression before the + # invalid conversion. + "f'{!x}'", + "f'{ !xr}'", + "f'{!x:}'", + "f'{!x:a}'", + "f'{ !xr:}'", + "f'{ !xr:a}'", + ]) + + self.assertAllRaise(SyntaxError, + "f-string: valid expression required before ':'", + ["f'{:}'", + "f'{ :!}'", + "f'{:2}'", + "f'''{\t\f\r\n:a}'''", + "f'{:'", + "F'{[F'{:'}[F'{:'}]]]", + ]) + + self.assertAllRaise(SyntaxError, + "f-string: valid expression required before '='", + ["f'{=}'", + "f'{ =}'", + "f'{ =:}'", + "f'{ =!}'", + "f'''{\t\f\r\n=}'''", + "f'{='", + ]) # Different error message is raised for other whitespace characters. - self.assertAllRaise( - SyntaxError, - r"invalid non-printable character U\+00A0", - [ - "f'''{\xa0}'''", - "\xa0", - ], - ) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + self.assertAllRaise(SyntaxError, r"invalid non-printable character U\+00A0", + ["f'''{\xa0}'''", + "\xa0", + ]) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_parens_in_expressions(self): - self.assertEqual(f"{3,}", "(3,)") - - self.assertAllRaise( - SyntaxError, - "f-string: expecting a valid expression after '{'", - [ - "f'{,}'", - ], - ) - - self.assertAllRaise( - SyntaxError, - r"f-string: unmatched '\)'", - [ - "f'{3)+(4}'", - ], - ) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + self.assertEqual(f'{3,}', '(3,)') + + self.assertAllRaise(SyntaxError, + "f-string: expecting a valid expression after '{'", + ["f'{,}'", + ]) + + self.assertAllRaise(SyntaxError, r"f-string: unmatched '\)'", + ["f'{3)+(4}'", + ]) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_newlines_before_syntax_error(self): - self.assertAllRaise( - SyntaxError, - "f-string: expecting a valid expression after '{'", - ["f'{.}'", "\nf'{.}'", "\n\nf'{.}'"], - ) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + self.assertAllRaise(SyntaxError, + "f-string: expecting a valid expression after '{'", + ["f'{.}'", "\nf'{.}'", "\n\nf'{.}'"]) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_backslashes_in_string_part(self): - self.assertEqual(f"\t", "\t") - self.assertEqual(r"\t", "\\t") - self.assertEqual(rf"\t", "\\t") - self.assertEqual(f"{2}\t", "2\t") - self.assertEqual(f"{2}\t{3}", "2\t3") - self.assertEqual(f"\t{3}", "\t3") - - self.assertEqual(f"\u0394", "\u0394") - self.assertEqual(r"\u0394", "\\u0394") - self.assertEqual(rf"\u0394", "\\u0394") - self.assertEqual(f"{2}\u0394", "2\u0394") - self.assertEqual(f"{2}\u0394{3}", "2\u03943") - self.assertEqual(f"\u0394{3}", "\u03943") - - self.assertEqual(f"\U00000394", "\u0394") - self.assertEqual(r"\U00000394", "\\U00000394") - self.assertEqual(rf"\U00000394", "\\U00000394") - self.assertEqual(f"{2}\U00000394", "2\u0394") - self.assertEqual(f"{2}\U00000394{3}", "2\u03943") - self.assertEqual(f"\U00000394{3}", "\u03943") - - self.assertEqual(f"\N{GREEK CAPITAL LETTER DELTA}", "\u0394") - self.assertEqual(f"{2}\N{GREEK CAPITAL LETTER DELTA}", "2\u0394") - self.assertEqual(f"{2}\N{GREEK CAPITAL LETTER DELTA}{3}", "2\u03943") - self.assertEqual(f"\N{GREEK CAPITAL LETTER DELTA}{3}", "\u03943") - self.assertEqual(f"2\N{GREEK CAPITAL LETTER DELTA}", "2\u0394") - self.assertEqual(f"2\N{GREEK CAPITAL LETTER DELTA}3", "2\u03943") - self.assertEqual(f"\N{GREEK CAPITAL LETTER DELTA}3", "\u03943") - - self.assertEqual(f"\x20", " ") - self.assertEqual(r"\x20", "\\x20") - self.assertEqual(rf"\x20", "\\x20") - self.assertEqual(f"{2}\x20", "2 ") - self.assertEqual(f"{2}\x20{3}", "2 3") - self.assertEqual(f"\x20{3}", " 3") - - self.assertEqual(f"2\x20", "2 ") - self.assertEqual(f"2\x203", "2 3") - self.assertEqual(f"\x203", " 3") + self.assertEqual(f'\t', '\t') + self.assertEqual(r'\t', '\\t') + self.assertEqual(rf'\t', '\\t') + self.assertEqual(f'{2}\t', '2\t') + self.assertEqual(f'{2}\t{3}', '2\t3') + self.assertEqual(f'\t{3}', '\t3') + + self.assertEqual(f'\u0394', '\u0394') + self.assertEqual(r'\u0394', '\\u0394') + self.assertEqual(rf'\u0394', '\\u0394') + self.assertEqual(f'{2}\u0394', '2\u0394') + self.assertEqual(f'{2}\u0394{3}', '2\u03943') + self.assertEqual(f'\u0394{3}', '\u03943') + + self.assertEqual(f'\U00000394', '\u0394') + self.assertEqual(r'\U00000394', '\\U00000394') + self.assertEqual(rf'\U00000394', '\\U00000394') + self.assertEqual(f'{2}\U00000394', '2\u0394') + self.assertEqual(f'{2}\U00000394{3}', '2\u03943') + self.assertEqual(f'\U00000394{3}', '\u03943') + + self.assertEqual(f'\N{GREEK CAPITAL LETTER DELTA}', '\u0394') + self.assertEqual(f'{2}\N{GREEK CAPITAL LETTER DELTA}', '2\u0394') + self.assertEqual(f'{2}\N{GREEK CAPITAL LETTER DELTA}{3}', '2\u03943') + self.assertEqual(f'\N{GREEK CAPITAL LETTER DELTA}{3}', '\u03943') + self.assertEqual(f'2\N{GREEK CAPITAL LETTER DELTA}', '2\u0394') + self.assertEqual(f'2\N{GREEK CAPITAL LETTER DELTA}3', '2\u03943') + self.assertEqual(f'\N{GREEK CAPITAL LETTER DELTA}3', '\u03943') + + self.assertEqual(f'\x20', ' ') + self.assertEqual(r'\x20', '\\x20') + self.assertEqual(rf'\x20', '\\x20') + self.assertEqual(f'{2}\x20', '2 ') + self.assertEqual(f'{2}\x20{3}', '2 3') + self.assertEqual(f'\x20{3}', ' 3') + + self.assertEqual(f'2\x20', '2 ') + self.assertEqual(f'2\x203', '2 3') + self.assertEqual(f'\x203', ' 3') with self.assertWarns(SyntaxWarning): # invalid escape sequence value = eval(r"f'\{6*7}'") - self.assertEqual(value, "\\42") + self.assertEqual(value, '\\42') with self.assertWarns(SyntaxWarning): # invalid escape sequence value = eval(r"f'\g'") - self.assertEqual(value, "\\g") - self.assertEqual(f"\\{6*7}", "\\42") - self.assertEqual(rf"\{6*7}", "\\42") + self.assertEqual(value, '\\g') + self.assertEqual(f'\\{6*7}', '\\42') + self.assertEqual(fr'\{6*7}', '\\42') - AMPERSAND = "spam" + AMPERSAND = 'spam' # Get the right unicode character (&), or pick up local variable # depending on the number of backslashes. - self.assertEqual(f"\N{AMPERSAND}", "&") - self.assertEqual(f"\\N{AMPERSAND}", "\\Nspam") - self.assertEqual(rf"\N{AMPERSAND}", "\\Nspam") - self.assertEqual(f"\\\N{AMPERSAND}", "\\&") + self.assertEqual(f'\N{AMPERSAND}', '&') + self.assertEqual(f'\\N{AMPERSAND}', '\\Nspam') + self.assertEqual(fr'\N{AMPERSAND}', '\\Nspam') + self.assertEqual(f'\\\N{AMPERSAND}', '\\&') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_misformed_unicode_character_name(self): # These test are needed because unicode names are parsed # differently inside f-strings. - self.assertAllRaise( - SyntaxError, - r"\(unicode error\) 'unicodeescape' codec can't decode bytes in position .*: malformed \\N character escape", - [ - r"f'\N'", - r"f'\N '", - r"f'\N '", # See bpo-46503. - r"f'\N{'", - r"f'\N{GREEK CAPITAL LETTER DELTA'", - # Here are the non-f-string versions, - # which should give the same errors. - r"'\N'", - r"'\N '", - r"'\N '", - r"'\N{'", - r"'\N{GREEK CAPITAL LETTER DELTA'", - ], - ) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + self.assertAllRaise(SyntaxError, r"\(unicode error\) 'unicodeescape' codec can't decode bytes in position .*: malformed \\N character escape", + [r"f'\N'", + r"f'\N '", + r"f'\N '", # See bpo-46503. + r"f'\N{'", + r"f'\N{GREEK CAPITAL LETTER DELTA'", + + # Here are the non-f-string versions, + # which should give the same errors. + r"'\N'", + r"'\N '", + r"'\N '", + r"'\N{'", + r"'\N{GREEK CAPITAL LETTER DELTA'", + ]) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_backslashes_in_expression_part(self): - # TODO: RUSTPYTHON SyntaxError - # self.assertEqual( - # f"{( - # 1 + - # 2 - # )}", - # "3", - # ) - - self.assertEqual("\N{LEFT CURLY BRACKET}", "{") - self.assertEqual(f'{"\N{LEFT CURLY BRACKET}"}', "{") - self.assertEqual(rf'{"\N{LEFT CURLY BRACKET}"}', "{") - - self.assertAllRaise( - SyntaxError, - "f-string: valid expression required before '}'", - [ - "f'{\n}'", - ], - ) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + self.assertEqual(f"{( + 1 + + 2 + )}", "3") + + self.assertEqual("\N{LEFT CURLY BRACKET}", '{') + self.assertEqual(f'{"\N{LEFT CURLY BRACKET}"}', '{') + self.assertEqual(rf'{"\N{LEFT CURLY BRACKET}"}', '{') + + self.assertAllRaise(SyntaxError, + "f-string: valid expression required before '}'", + ["f'{\n}'", + ]) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid_backslashes_inside_fstring_context(self): # All of these variations are invalid python syntax, # so they are also invalid in f-strings as well. @@ -1129,30 +1057,25 @@ def test_invalid_backslashes_inside_fstring_context(self): r"\\"[0], ] ] - self.assertAllRaise( - SyntaxError, "unexpected character after line continuation", cases - ) + self.assertAllRaise(SyntaxError, 'unexpected character after line continuation', + cases) def test_no_escapes_for_braces(self): """ Only literal curly braces begin an expression. """ # \x7b is '{'. - self.assertEqual(f"\x7b1+1}}", "{1+1}") - self.assertEqual(f"\x7b1+1", "{1+1") - self.assertEqual(f"\u007b1+1", "{1+1") - self.assertEqual(f"\N{LEFT CURLY BRACKET}1+1\N{RIGHT CURLY BRACKET}", "{1+1}") + self.assertEqual(f'\x7b1+1}}', '{1+1}') + self.assertEqual(f'\x7b1+1', '{1+1') + self.assertEqual(f'\u007b1+1', '{1+1') + self.assertEqual(f'\N{LEFT CURLY BRACKET}1+1\N{RIGHT CURLY BRACKET}', '{1+1}') def test_newlines_in_expressions(self): - self.assertEqual(f"{0}", "0") - self.assertEqual( - rf"""{3+ -4}""", - "7", - ) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + self.assertEqual(f'{0}', '0') + self.assertEqual(rf'''{3+ +4}''', '7') + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_lambda(self): x = 5 self.assertEqual(f'{(lambda y:x*y)("8")!r}', "'88888'") @@ -1162,67 +1085,60 @@ def test_lambda(self): # lambda doesn't work without parens, because the colon # makes the parser think it's a format_spec # emit warning if we can match a format_spec - self.assertAllRaise( - SyntaxError, - "f-string: lambda expressions are not allowed " "without parentheses", - [ - "f'{lambda x:x}'", - "f'{lambda :x}'", - "f'{lambda *arg, :x}'", - "f'{1, lambda:x}'", - "f'{lambda x:}'", - "f'{lambda :}'", - ], - ) + self.assertAllRaise(SyntaxError, + "f-string: lambda expressions are not allowed " + "without parentheses", + ["f'{lambda x:x}'", + "f'{lambda :x}'", + "f'{lambda *arg, :x}'", + "f'{1, lambda:x}'", + "f'{lambda x:}'", + "f'{lambda :}'", + ]) # Ensure the detection of invalid lambdas doesn't trigger detection # for valid lambdas in the second error pass with self.assertRaisesRegex(SyntaxError, "invalid syntax"): compile("lambda name_3=f'{name_4}': {name_3}\n1 $ 1", "", "exec") # but don't emit the paren warning in general cases - with self.assertRaisesRegex( - SyntaxError, "f-string: expecting a valid expression after '{'" - ): + with self.assertRaisesRegex(SyntaxError, "f-string: expecting a valid expression after '{'"): eval("f'{+ lambda:None}'") def test_valid_prefixes(self): - self.assertEqual(f"{1}", "1") - self.assertEqual(Rf"{2}", "2") - self.assertEqual(Rf"{3}", "3") + self.assertEqual(F'{1}', "1") + self.assertEqual(FR'{2}', "2") + self.assertEqual(fR'{3}', "3") def test_roundtrip_raw_quotes(self): - self.assertEqual(rf"\'", "\\'") - self.assertEqual(rf"\"", '\\"') - self.assertEqual(rf"\"\'", "\\\"\\'") - self.assertEqual(rf"\'\"", "\\'\\\"") - self.assertEqual(rf"\"\'\"", '\\"\\\'\\"') - self.assertEqual(rf"\'\"\'", "\\'\\\"\\'") - self.assertEqual(rf"\"\'\"\'", "\\\"\\'\\\"\\'") - - # TODO: RUSTPYTHON - @unittest.expectedFailure + self.assertEqual(fr"\'", "\\'") + self.assertEqual(fr'\"', '\\"') + self.assertEqual(fr'\"\'', '\\"\\\'') + self.assertEqual(fr'\'\"', '\\\'\\"') + self.assertEqual(fr'\"\'\"', '\\"\\\'\\"') + self.assertEqual(fr'\'\"\'', '\\\'\\"\\\'') + self.assertEqual(fr'\"\'\"\'', '\\"\\\'\\"\\\'') + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_fstring_backslash_before_double_bracket(self): deprecated_cases = [ - (r"f'\{{\}}'", "\\{\\}"), - (r"f'\{{'", "\\{"), - (r"f'\{{{1+1}'", "\\{2"), - (r"f'\}}{1+1}'", "\\}2"), - (r"f'{1+1}\}}'", "2\\}"), + (r"f'\{{\}}'", '\\{\\}'), + (r"f'\{{'", '\\{'), + (r"f'\{{{1+1}'", '\\{2'), + (r"f'\}}{1+1}'", '\\}2'), + (r"f'{1+1}\}}'", '2\\}') ] - for case, expected_result in deprecated_cases: with self.subTest(case=case, expected_result=expected_result): with self.assertWarns(SyntaxWarning): result = eval(case) self.assertEqual(result, expected_result) - self.assertEqual(rf"\{{\}}", "\\{\\}") - self.assertEqual(rf"\{{", "\\{") - self.assertEqual(rf"\{{{1+1}", "\\{2") - self.assertEqual(rf"\}}{1+1}", "\\}2") - self.assertEqual(rf"{1+1}\}}", "2\\}") - - # TODO: RUSTPYTHON - @unittest.expectedFailure + self.assertEqual(fr'\{{\}}', '\\{\\}') + self.assertEqual(fr'\{{', '\\{') + self.assertEqual(fr'\{{{1+1}', '\\{2') + self.assertEqual(fr'\}}{1+1}', '\\}2') + self.assertEqual(fr'{1+1}\}}', '2\\}') + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_fstring_backslash_before_double_bracket_warns_once(self): with self.assertWarns(SyntaxWarning) as w: eval(r"f'\{{'") @@ -1230,18 +1146,18 @@ def test_fstring_backslash_before_double_bracket_warns_once(self): self.assertEqual(w.warnings[0].category, SyntaxWarning) def test_fstring_backslash_prefix_raw(self): - self.assertEqual(f"\\", "\\") - self.assertEqual(f"\\\\", "\\\\") - self.assertEqual(rf"\\", r"\\") - self.assertEqual(rf"\\\\", r"\\\\") - self.assertEqual(rf"\\", r"\\") - self.assertEqual(rf"\\\\", r"\\\\") - self.assertEqual(Rf"\\", R"\\") - self.assertEqual(Rf"\\\\", R"\\\\") - self.assertEqual(Rf"\\", R"\\") - self.assertEqual(Rf"\\\\", R"\\\\") - self.assertEqual(Rf"\\", R"\\") - self.assertEqual(Rf"\\\\", R"\\\\") + self.assertEqual(f'\\', '\\') + self.assertEqual(f'\\\\', '\\\\') + self.assertEqual(fr'\\', r'\\') + self.assertEqual(fr'\\\\', r'\\\\') + self.assertEqual(rf'\\', r'\\') + self.assertEqual(rf'\\\\', r'\\\\') + self.assertEqual(Rf'\\', R'\\') + self.assertEqual(Rf'\\\\', R'\\\\') + self.assertEqual(fR'\\', R'\\') + self.assertEqual(fR'\\\\', R'\\\\') + self.assertEqual(FR'\\', R'\\') + self.assertEqual(FR'\\\\', R'\\\\') def test_fstring_format_spec_greedy_matching(self): self.assertEqual(f"{1:}}}", "1}") @@ -1251,8 +1167,8 @@ def test_yield(self): # Not terribly useful, but make sure the yield turns # a function into a generator def fn(y): - f"y:{yield y*2}" - f"{yield}" + f'y:{yield y*2}' + f'{yield}' g = fn(4) self.assertEqual(next(g), 8) @@ -1260,331 +1176,287 @@ def fn(y): def test_yield_send(self): def fn(x): - yield f"x:{yield (lambda i: x * i)}" + yield f'x:{yield (lambda i: x * i)}' g = fn(10) the_lambda = next(g) self.assertEqual(the_lambda(4), 40) - self.assertEqual(g.send("string"), "x:string") + self.assertEqual(g.send('string'), 'x:string') - # TODO: RUSTPYTHON SyntaxError - # def test_expressions_with_triple_quoted_strings(self): - # self.assertEqual(f"{'''x'''}", 'x') - # self.assertEqual(f"{'''eric's'''}", "eric's") + def test_expressions_with_triple_quoted_strings(self): + self.assertEqual(f"{'''x'''}", 'x') + self.assertEqual(f"{'''eric's'''}", "eric's") - # # Test concatenation within an expression - # self.assertEqual(f'{"x" """eric"s""" "y"}', 'xeric"sy') - # self.assertEqual(f'{"x" """eric"s"""}', 'xeric"s') - # self.assertEqual(f'{"""eric"s""" "y"}', 'eric"sy') - # self.assertEqual(f'{"""x""" """eric"s""" "y"}', 'xeric"sy') - # self.assertEqual(f'{"""x""" """eric"s""" """y"""}', 'xeric"sy') - # self.assertEqual(f'{r"""x""" """eric"s""" """y"""}', 'xeric"sy') + # Test concatenation within an expression + self.assertEqual(f'{"x" """eric"s""" "y"}', 'xeric"sy') + self.assertEqual(f'{"x" """eric"s"""}', 'xeric"s') + self.assertEqual(f'{"""eric"s""" "y"}', 'eric"sy') + self.assertEqual(f'{"""x""" """eric"s""" "y"}', 'xeric"sy') + self.assertEqual(f'{"""x""" """eric"s""" """y"""}', 'xeric"sy') + self.assertEqual(f'{r"""x""" """eric"s""" """y"""}', 'xeric"sy') def test_multiple_vars(self): x = 98 - y = "abc" - self.assertEqual(f"{x}{y}", "98abc") + y = 'abc' + self.assertEqual(f'{x}{y}', '98abc') - self.assertEqual(f"X{x}{y}", "X98abc") - self.assertEqual(f"{x}X{y}", "98Xabc") - self.assertEqual(f"{x}{y}X", "98abcX") + self.assertEqual(f'X{x}{y}', 'X98abc') + self.assertEqual(f'{x}X{y}', '98Xabc') + self.assertEqual(f'{x}{y}X', '98abcX') - self.assertEqual(f"X{x}Y{y}", "X98Yabc") - self.assertEqual(f"X{x}{y}Y", "X98abcY") - self.assertEqual(f"{x}X{y}Y", "98XabcY") + self.assertEqual(f'X{x}Y{y}', 'X98Yabc') + self.assertEqual(f'X{x}{y}Y', 'X98abcY') + self.assertEqual(f'{x}X{y}Y', '98XabcY') - self.assertEqual(f"X{x}Y{y}Z", "X98YabcZ") + self.assertEqual(f'X{x}Y{y}Z', 'X98YabcZ') def test_closure(self): def outer(x): def inner(): - return f"x:{x}" - + return f'x:{x}' return inner - self.assertEqual(outer("987")(), "x:987") - self.assertEqual(outer(7)(), "x:7") + self.assertEqual(outer('987')(), 'x:987') + self.assertEqual(outer(7)(), 'x:7') def test_arguments(self): y = 2 - def f(x, width): - return f"x={x*y:{width}}" + return f'x={x*y:{width}}' - self.assertEqual(f("foo", 10), "x=foofoo ") - x = "bar" - self.assertEqual(f(10, 10), "x= 20") + self.assertEqual(f('foo', 10), 'x=foofoo ') + x = 'bar' + self.assertEqual(f(10, 10), 'x= 20') def test_locals(self): value = 123 - self.assertEqual(f"v:{value}", "v:123") + self.assertEqual(f'v:{value}', 'v:123') def test_missing_variable(self): with self.assertRaises(NameError): - f"v:{value}" + f'v:{value}' def test_missing_format_spec(self): class O: def __format__(self, spec): if not spec: - return "*" + return '*' return spec - self.assertEqual(f"{O():x}", "x") - self.assertEqual(f"{O()}", "*") - self.assertEqual(f"{O():}", "*") + self.assertEqual(f'{O():x}', 'x') + self.assertEqual(f'{O()}', '*') + self.assertEqual(f'{O():}', '*') - self.assertEqual(f"{3:}", "3") - self.assertEqual(f"{3!s:}", "3") + self.assertEqual(f'{3:}', '3') + self.assertEqual(f'{3!s:}', '3') def test_global(self): - self.assertEqual(f"g:{a_global}", "g:global variable") - self.assertEqual(f"g:{a_global!r}", "g:'global variable'") + self.assertEqual(f'g:{a_global}', 'g:global variable') + self.assertEqual(f'g:{a_global!r}', "g:'global variable'") - a_local = "local variable" - self.assertEqual( - f"g:{a_global} l:{a_local}", "g:global variable l:local variable" - ) - self.assertEqual(f"g:{a_global!r}", "g:'global variable'") - self.assertEqual( - f"g:{a_global} l:{a_local!r}", "g:global variable l:'local variable'" - ) + a_local = 'local variable' + self.assertEqual(f'g:{a_global} l:{a_local}', + 'g:global variable l:local variable') + self.assertEqual(f'g:{a_global!r}', + "g:'global variable'") + self.assertEqual(f'g:{a_global} l:{a_local!r}', + "g:global variable l:'local variable'") - self.assertIn("module 'unittest' from", f"{unittest}") + self.assertIn("module 'unittest' from", f'{unittest}') def test_shadowed_global(self): - a_global = "really a local" - self.assertEqual(f"g:{a_global}", "g:really a local") - self.assertEqual(f"g:{a_global!r}", "g:'really a local'") - - a_local = "local variable" - self.assertEqual( - f"g:{a_global} l:{a_local}", "g:really a local l:local variable" - ) - self.assertEqual(f"g:{a_global!r}", "g:'really a local'") - self.assertEqual( - f"g:{a_global} l:{a_local!r}", "g:really a local l:'local variable'" - ) + a_global = 'really a local' + self.assertEqual(f'g:{a_global}', 'g:really a local') + self.assertEqual(f'g:{a_global!r}', "g:'really a local'") + + a_local = 'local variable' + self.assertEqual(f'g:{a_global} l:{a_local}', + 'g:really a local l:local variable') + self.assertEqual(f'g:{a_global!r}', + "g:'really a local'") + self.assertEqual(f'g:{a_global} l:{a_local!r}', + "g:really a local l:'local variable'") def test_call(self): def foo(x): - return "x=" + str(x) + return 'x=' + str(x) - self.assertEqual(f"{foo(10)}", "x=10") + self.assertEqual(f'{foo(10)}', 'x=10') def test_nested_fstrings(self): y = 5 - self.assertEqual(f'{f"{0}"*3}', "000") - self.assertEqual(f'{f"{y}"*3}', "555") + self.assertEqual(f'{f"{0}"*3}', '000') + self.assertEqual(f'{f"{y}"*3}', '555') def test_invalid_string_prefixes(self): - single_quote_cases = [ - "fu''", - "uf''", - "Fu''", - "fU''", - "Uf''", - "uF''", - "ufr''", - "urf''", - "fur''", - "fru''", - "rfu''", - "ruf''", - "FUR''", - "Fur''", - "fb''", - "fB''", - "Fb''", - "FB''", - "bf''", - "bF''", - "Bf''", - "BF''", - ] + single_quote_cases = ["fu''", + "uf''", + "Fu''", + "fU''", + "Uf''", + "uF''", + "ufr''", + "urf''", + "fur''", + "fru''", + "rfu''", + "ruf''", + "FUR''", + "Fur''", + "fb''", + "fB''", + "Fb''", + "FB''", + "bf''", + "bF''", + "Bf''", + "BF''",] double_quote_cases = [case.replace("'", '"') for case in single_quote_cases] - self.assertAllRaise( - SyntaxError, "invalid syntax", single_quote_cases + double_quote_cases - ) + self.assertAllRaise(SyntaxError, 'invalid syntax', + single_quote_cases + double_quote_cases) def test_leading_trailing_spaces(self): - self.assertEqual(f"{ 3}", "3") - self.assertEqual(f"{ 3}", "3") - self.assertEqual(f"{3 }", "3") - self.assertEqual(f"{3 }", "3") + self.assertEqual(f'{ 3}', '3') + self.assertEqual(f'{ 3}', '3') + self.assertEqual(f'{3 }', '3') + self.assertEqual(f'{3 }', '3') - self.assertEqual(f"expr={ {x: y for x, y in [(1, 2), ]}}", "expr={1: 2}") - self.assertEqual(f"expr={ {x: y for x, y in [(1, 2), ]} }", "expr={1: 2}") + self.assertEqual(f'expr={ {x: y for x, y in [(1, 2), ]}}', + 'expr={1: 2}') + self.assertEqual(f'expr={ {x: y for x, y in [(1, 2), ]} }', + 'expr={1: 2}') def test_not_equal(self): # There's a special test for this because there's a special # case in the f-string parser to look for != as not ending an # expression. Normally it would, while looking for !s or !r. - self.assertEqual(f"{3!=4}", "True") - self.assertEqual(f"{3!=4:}", "True") - self.assertEqual(f"{3!=4!s}", "True") - self.assertEqual(f"{3!=4!s:.3}", "Tru") + self.assertEqual(f'{3!=4}', 'True') + self.assertEqual(f'{3!=4:}', 'True') + self.assertEqual(f'{3!=4!s}', 'True') + self.assertEqual(f'{3!=4!s:.3}', 'Tru') def test_equal_equal(self): # Because an expression ending in = has special meaning, # there's a special test for ==. Make sure it works. - self.assertEqual(f"{0==1}", "False") + self.assertEqual(f'{0==1}', 'False') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_conversions(self): - self.assertEqual(f"{3.14:10.10}", " 3.14") - self.assertEqual(f"{3.14!s:10.10}", "3.14 ") - self.assertEqual(f"{3.14!r:10.10}", "3.14 ") - self.assertEqual(f"{3.14!a:10.10}", "3.14 ") + self.assertEqual(f'{3.14:10.10}', ' 3.14') + self.assertEqual(f'{1.25!s:10.10}', '1.25 ') + self.assertEqual(f'{1.25!r:10.10}', '1.25 ') + self.assertEqual(f'{1.25!a:10.10}', '1.25 ') - self.assertEqual(f'{"a"}', "a") + self.assertEqual(f'{"a"}', 'a') self.assertEqual(f'{"a"!r}', "'a'") self.assertEqual(f'{"a"!a}', "'a'") # Conversions can have trailing whitespace after them since it # does not provide any significance - # TODO: RUSTPYTHON SyntaxError - # self.assertEqual(f"{3!s }", "3") - # self.assertEqual(f"{3.14!s :10.10}", "3.14 ") + self.assertEqual(f"{3!s }", "3") + self.assertEqual(f'{1.25!s :10.10}', '1.25 ') # Not a conversion. self.assertEqual(f'{"a!r"}', "a!r") # Not a conversion, but show that ! is allowed in a format spec. - self.assertEqual(f"{3.14:!<10.10}", "3.14!!!!!!") - - self.assertAllRaise( - SyntaxError, - "f-string: expecting '}'", - [ - "f'{3!'", - "f'{3!s'", - "f'{3!g'", - ], - ) - - self.assertAllRaise( - SyntaxError, - "f-string: missing conversion character", - [ - "f'{3!}'", - "f'{3!:'", - "f'{3!:}'", - ], - ) - - for conv_identifier in "g", "A", "G", "ä", "ɐ": - self.assertAllRaise( - SyntaxError, - "f-string: invalid conversion character %r: " - "expected 's', 'r', or 'a'" % conv_identifier, - ["f'{3!" + conv_identifier + "}'"], - ) - - for conv_non_identifier in "3", "!": - self.assertAllRaise( - SyntaxError, - "f-string: invalid conversion character", - ["f'{3!" + conv_non_identifier + "}'"], - ) - - for conv in " s", " s ": - self.assertAllRaise( - SyntaxError, - "f-string: conversion type must come right after the" - " exclamanation mark", - ["f'{3!" + conv + "}'"], - ) - - self.assertAllRaise( - SyntaxError, - "f-string: invalid conversion character 'ss': " "expected 's', 'r', or 'a'", - [ - "f'{3!ss}'", - "f'{3!ss:}'", - "f'{3!ss:s}'", - ], - ) + self.assertEqual(f'{3.14:!<10.10}', '3.14!!!!!!') + + self.assertAllRaise(SyntaxError, "f-string: expecting '}'", + ["f'{3!'", + "f'{3!s'", + "f'{3!g'", + ]) + + self.assertAllRaise(SyntaxError, 'f-string: missing conversion character', + ["f'{3!}'", + "f'{3!:'", + "f'{3!:}'", + ]) + + for conv_identifier in 'g', 'A', 'G', 'ä', 'ɐ': + self.assertAllRaise(SyntaxError, + "f-string: invalid conversion character %r: " + "expected 's', 'r', or 'a'" % conv_identifier, + ["f'{3!" + conv_identifier + "}'"]) + + for conv_non_identifier in '3', '!': + self.assertAllRaise(SyntaxError, + "f-string: invalid conversion character", + ["f'{3!" + conv_non_identifier + "}'"]) + + for conv in ' s', ' s ': + self.assertAllRaise(SyntaxError, + "f-string: conversion type must come right after the" + " exclamation mark", + ["f'{3!" + conv + "}'"]) + + self.assertAllRaise(SyntaxError, + "f-string: invalid conversion character 'ss': " + "expected 's', 'r', or 'a'", + ["f'{3!ss}'", + "f'{3!ss:}'", + "f'{3!ss:s}'", + ]) def test_assignment(self): - self.assertAllRaise( - SyntaxError, - r"invalid syntax", - [ - "f'' = 3", - "f'{0}' = x", - "f'{x}' = x", - ], - ) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + self.assertAllRaise(SyntaxError, r'invalid syntax', + ["f'' = 3", + "f'{0}' = x", + "f'{x}' = x", + ]) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_del(self): - self.assertAllRaise( - SyntaxError, - "invalid syntax", - [ - "del f''", - "del '' f''", - ], - ) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + self.assertAllRaise(SyntaxError, 'invalid syntax', + ["del f''", + "del '' f''", + ]) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_mismatched_braces(self): - self.assertAllRaise( - SyntaxError, - "f-string: single '}' is not allowed", - [ - "f'{{}'", - "f'{{}}}'", - "f'}'", - "f'x}'", - "f'x}x'", - r"f'\u007b}'", - # Can't have { or } in a format spec. - "f'{3:}>10}'", - "f'{3:}}>10}'", - ], - ) - - self.assertAllRaise( - SyntaxError, - "f-string: expecting '}'", - [ - "f'{3'", - "f'{3!'", - "f'{3:'", - "f'{3!s'", - "f'{3!s:'", - "f'{3!s:3'", - "f'x{'", - "f'x{x'", - "f'{x'", - "f'{3:s'", - "f'{{{'", - "f'{{}}{'", - "f'{'", - "f'{i='", # See gh-93418. - ], - ) - - self.assertAllRaise( - SyntaxError, - "f-string: expecting a valid expression after '{'", - [ - "f'{3:{{>10}'", - ], - ) + self.assertAllRaise(SyntaxError, "f-string: single '}' is not allowed", + ["f'{{}'", + "f'{{}}}'", + "f'}'", + "f'x}'", + "f'x}x'", + r"f'\u007b}'", + + # Can't have { or } in a format spec. + "f'{3:}>10}'", + "f'{3:}}>10}'", + ]) + + self.assertAllRaise(SyntaxError, "f-string: expecting '}'", + ["f'{3'", + "f'{3!'", + "f'{3:'", + "f'{3!s'", + "f'{3!s:'", + "f'{3!s:3'", + "f'x{'", + "f'x{x'", + "f'{x'", + "f'{3:s'", + "f'{{{'", + "f'{{}}{'", + "f'{'", + "f'{i='", # See gh-93418. + ]) + + self.assertAllRaise(SyntaxError, + "f-string: expecting a valid expression after '{'", + ["f'{3:{{>10}'", + ]) # But these are just normal strings. - self.assertEqual(f'{"{"}', "{") - self.assertEqual(f'{"}"}', "}") - self.assertEqual(f'{3:{"}"}>10}', "}}}}}}}}}3") - self.assertEqual(f'{2:{"{"}>10}', "{{{{{{{{{2") + self.assertEqual(f'{"{"}', '{') + self.assertEqual(f'{"}"}', '}') + self.assertEqual(f'{3:{"}"}>10}', '}}}}}}}}}3') + self.assertEqual(f'{2:{"{"}>10}', '{{{{{{{{{2') def test_if_conditional(self): # There's special logic in compile.c to test if the @@ -1593,7 +1465,7 @@ def test_if_conditional(self): def test_fstring(x, expected): flag = 0 - if f"{x}": + if f'{x}': flag = 1 else: flag = 2 @@ -1601,7 +1473,7 @@ def test_fstring(x, expected): def test_concat_empty(x, expected): flag = 0 - if "" f"{x}": + if '' f'{x}': flag = 1 else: flag = 2 @@ -1609,151 +1481,139 @@ def test_concat_empty(x, expected): def test_concat_non_empty(x, expected): flag = 0 - if " " f"{x}": + if ' ' f'{x}': flag = 1 else: flag = 2 self.assertEqual(flag, expected) - test_fstring("", 2) - test_fstring(" ", 1) + test_fstring('', 2) + test_fstring(' ', 1) - test_concat_empty("", 2) - test_concat_empty(" ", 1) + test_concat_empty('', 2) + test_concat_empty(' ', 1) - test_concat_non_empty("", 1) - test_concat_non_empty(" ", 1) + test_concat_non_empty('', 1) + test_concat_non_empty(' ', 1) def test_empty_format_specifier(self): - x = "test" - self.assertEqual(f"{x}", "test") - self.assertEqual(f"{x:}", "test") - self.assertEqual(f"{x!s:}", "test") - self.assertEqual(f"{x!r:}", "'test'") + x = 'test' + self.assertEqual(f'{x}', 'test') + self.assertEqual(f'{x:}', 'test') + self.assertEqual(f'{x!s:}', 'test') + self.assertEqual(f'{x!r:}', "'test'") def test_str_format_differences(self): - d = { - "a": "string", - 0: "integer", - } + d = {'a': 'string', + 0: 'integer', + } a = 0 - self.assertEqual(f"{d[0]}", "integer") - self.assertEqual(f'{d["a"]}', "string") - self.assertEqual(f"{d[a]}", "integer") - self.assertEqual("{d[a]}".format(d=d), "string") - self.assertEqual("{d[0]}".format(d=d), "integer") - - # TODO: RUSTPYTHON - @unittest.expectedFailure + self.assertEqual(f'{d[0]}', 'integer') + self.assertEqual(f'{d["a"]}', 'string') + self.assertEqual(f'{d[a]}', 'integer') + self.assertEqual('{d[a]}'.format(d=d), 'string') + self.assertEqual('{d[0]}'.format(d=d), 'integer') + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_errors(self): # see issue 26287 - self.assertAllRaise( - TypeError, - "unsupported", - [ - r"f'{(lambda: 0):x}'", - r"f'{(0,):x}'", - ], - ) - self.assertAllRaise( - ValueError, - "Unknown format code", - [ - r"f'{1000:j}'", - r"f'{1000:j}'", - ], - ) + self.assertAllRaise(TypeError, 'unsupported', + [r"f'{(lambda: 0):x}'", + r"f'{(0,):x}'", + ]) + self.assertAllRaise(ValueError, 'Unknown format code', + [r"f'{1000:j}'", + r"f'{1000:j}'", + ]) def test_filename_in_syntaxerror(self): # see issue 38964 with temp_cwd() as cwd: - file_path = os.path.join(cwd, "t.py") - with open(file_path, "w", encoding="utf-8") as f: - f.write('f"{a b}"') # This generates a SyntaxError - _, _, stderr = assert_python_failure(file_path, PYTHONIOENCODING="ascii") - self.assertIn(file_path.encode("ascii", "backslashreplace"), stderr) + file_path = os.path.join(cwd, 't.py') + with open(file_path, 'w', encoding="utf-8") as f: + f.write('f"{a b}"') # This generates a SyntaxError + _, _, stderr = assert_python_failure(file_path, + PYTHONIOENCODING='ascii') + self.assertIn(file_path.encode('ascii', 'backslashreplace'), stderr) def test_loop(self): for i in range(1000): - self.assertEqual(f"i:{i}", "i:" + str(i)) + self.assertEqual(f'i:{i}', 'i:' + str(i)) def test_dict(self): - d = { - '"': "dquote", - "'": "squote", - "foo": "bar", - } - self.assertEqual(f"""{d["'"]}""", "squote") - self.assertEqual(f"""{d['"']}""", "dquote") + d = {'"': 'dquote', + "'": 'squote', + 'foo': 'bar', + } + self.assertEqual(f'''{d["'"]}''', 'squote') + self.assertEqual(f"""{d['"']}""", 'dquote') - self.assertEqual(f'{d["foo"]}', "bar") - self.assertEqual(f"{d['foo']}", "bar") + self.assertEqual(f'{d["foo"]}', 'bar') + self.assertEqual(f"{d['foo']}", 'bar') def test_backslash_char(self): # Check eval of a backslash followed by a control char. # See bpo-30682: this used to raise an assert in pydebug mode. - self.assertEqual(eval('f"\\\n"'), "") - self.assertEqual(eval('f"\\\r"'), "") + self.assertEqual(eval('f"\\\n"'), '') + self.assertEqual(eval('f"\\\r"'), '') + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: '1+2 = # my comment\n 3' != '1+2 = \n 3' def test_debug_conversion(self): - x = "A string" - self.assertEqual(f"{x=}", "x=" + repr(x)) - self.assertEqual(f"{x =}", "x =" + repr(x)) - self.assertEqual(f"{x=!s}", "x=" + str(x)) - self.assertEqual(f"{x=!r}", "x=" + repr(x)) - self.assertEqual(f"{x=!a}", "x=" + ascii(x)) + x = 'A string' + self.assertEqual(f'{x=}', 'x=' + repr(x)) + self.assertEqual(f'{x =}', 'x =' + repr(x)) + self.assertEqual(f'{x=!s}', 'x=' + str(x)) + self.assertEqual(f'{x=!r}', 'x=' + repr(x)) + self.assertEqual(f'{x=!a}', 'x=' + ascii(x)) x = 2.71828 - self.assertEqual(f"{x=:.2f}", "x=" + format(x, ".2f")) - self.assertEqual(f"{x=:}", "x=" + format(x, "")) - self.assertEqual(f"{x=!r:^20}", "x=" + format(repr(x), "^20")) - self.assertEqual(f"{x=!s:^20}", "x=" + format(str(x), "^20")) - self.assertEqual(f"{x=!a:^20}", "x=" + format(ascii(x), "^20")) + self.assertEqual(f'{x=:.2f}', 'x=' + format(x, '.2f')) + self.assertEqual(f'{x=:}', 'x=' + format(x, '')) + self.assertEqual(f'{x=!r:^20}', 'x=' + format(repr(x), '^20')) + self.assertEqual(f'{x=!s:^20}', 'x=' + format(str(x), '^20')) + self.assertEqual(f'{x=!a:^20}', 'x=' + format(ascii(x), '^20')) x = 9 - self.assertEqual(f"{3*x+15=}", "3*x+15=42") + self.assertEqual(f'{3*x+15=}', '3*x+15=42') # There is code in ast.c that deals with non-ascii expression values. So, # use a unicode identifier to trigger that. tenπ = 31.4 - self.assertEqual(f"{tenπ=:.2f}", "tenπ=31.40") + self.assertEqual(f'{tenπ=:.2f}', 'tenπ=31.40') # Also test with Unicode in non-identifiers. - self.assertEqual(f'{"Σ"=}', "\"Σ\"='Σ'") + self.assertEqual(f'{"Σ"=}', '"Σ"=\'Σ\'') # Make sure nested fstrings still work. - self.assertEqual(f'{f"{3.1415=:.1f}":*^20}', "*****3.1415=3.1*****") + self.assertEqual(f'{f"{3.1415=:.1f}":*^20}', '*****3.1415=3.1*****') # Make sure text before and after an expression with = works # correctly. - pi = "π" - self.assertEqual(f"alpha α {pi=} ω omega", "alpha α pi='π' ω omega") + pi = 'π' + self.assertEqual(f'alpha α {pi=} ω omega', "alpha α pi='π' ω omega") # Check multi-line expressions. - self.assertEqual( - f"""{ + self.assertEqual(f'''{ 3 -=}""", - "\n3\n=3", - ) +=}''', '\n3\n=3') # Since = is handled specially, make sure all existing uses of # it still work. - self.assertEqual(f"{0==1}", "False") - self.assertEqual(f"{0!=1}", "True") - self.assertEqual(f"{0<=1}", "True") - self.assertEqual(f"{0>=1}", "False") - self.assertEqual(f'{(x:="5")}', "5") - self.assertEqual(x, "5") - self.assertEqual(f"{(x:=5)}", "5") + self.assertEqual(f'{0==1}', 'False') + self.assertEqual(f'{0!=1}', 'True') + self.assertEqual(f'{0<=1}', 'True') + self.assertEqual(f'{0>=1}', 'False') + self.assertEqual(f'{(x:="5")}', '5') + self.assertEqual(x, '5') + self.assertEqual(f'{(x:=5)}', '5') self.assertEqual(x, 5) - self.assertEqual(f'{"="}', "=") + self.assertEqual(f'{"="}', '=') x = 20 # This isn't an assignment expression, it's 'x', with a format # spec of '=10'. See test_walrus: you need to use parens. - self.assertEqual(f"{x:=10}", " 20") + self.assertEqual(f'{x:=10}', ' 20') # Test named function parameters, to make sure '=' parsing works # there. @@ -1762,54 +1622,60 @@ def f(a): oldx = x x = a return oldx - x = 0 - self.assertEqual(f'{f(a="3=")}', "0") - self.assertEqual(x, "3=") - self.assertEqual(f"{f(a=4)}", "3=") + self.assertEqual(f'{f(a="3=")}', '0') + self.assertEqual(x, '3=') + self.assertEqual(f'{f(a=4)}', '3=') self.assertEqual(x, 4) # Check debug expressions in format spec y = 20 self.assertEqual(f"{2:{y=}}", "yyyyyyyyyyyyyyyyyyy2") - self.assertEqual( - f"{datetime.datetime.now():h1{y=}h2{y=}h3{y=}}", "h1y=20h2y=20h3y=20" - ) + self.assertEqual(f"{datetime.datetime.now():h1{y=}h2{y=}h3{y=}}", + 'h1y=20h2y=20h3y=20') # Make sure __format__ is being called. class C: def __format__(self, s): - return f"FORMAT-{s}" - + return f'FORMAT-{s}' def __repr__(self): - return "REPR" + return 'REPR' - self.assertEqual(f"{C()=}", "C()=REPR") - self.assertEqual(f"{C()=!r}", "C()=REPR") - self.assertEqual(f"{C()=:}", "C()=FORMAT-") - self.assertEqual(f"{C()=: }", "C()=FORMAT- ") - self.assertEqual(f"{C()=:x}", "C()=FORMAT-x") - self.assertEqual(f"{C()=!r:*^20}", "C()=********REPR********") - self.assertEqual(f"{C():{20=}}", "FORMAT-20=20") + self.assertEqual(f'{C()=}', 'C()=REPR') + self.assertEqual(f'{C()=!r}', 'C()=REPR') + self.assertEqual(f'{C()=:}', 'C()=FORMAT-') + self.assertEqual(f'{C()=: }', 'C()=FORMAT- ') + self.assertEqual(f'{C()=:x}', 'C()=FORMAT-x') + self.assertEqual(f'{C()=!r:*^20}', 'C()=********REPR********') + self.assertEqual(f"{C():{20=}}", 'FORMAT-20=20') self.assertRaises(SyntaxError, eval, "f'{C=]'") + # Make sure leading and following text works. - x = "foo" - self.assertEqual(f"X{x=}Y", "Xx=" + repr(x) + "Y") + x = 'foo' + self.assertEqual(f'X{x=}Y', 'Xx='+repr(x)+'Y') # Make sure whitespace around the = works. - self.assertEqual(f"X{x =}Y", "Xx =" + repr(x) + "Y") - self.assertEqual(f"X{x= }Y", "Xx= " + repr(x) + "Y") - self.assertEqual(f"X{x = }Y", "Xx = " + repr(x) + "Y") + self.assertEqual(f'X{x =}Y', 'Xx ='+repr(x)+'Y') + self.assertEqual(f'X{x= }Y', 'Xx= '+repr(x)+'Y') + self.assertEqual(f'X{x = }Y', 'Xx = '+repr(x)+'Y') self.assertEqual(f"sadsd {1 + 1 = :{1 + 1:1d}f}", "sadsd 1 + 1 = 2.000000") -# TODO: RUSTPYTHON SyntaxError -# self.assertEqual( -# f"{1+2 = # my comment -# }", -# "1+2 = \n 3", -# ) + self.assertEqual(f"{1+2 = # my comment + }", '1+2 = \n 3') + + self.assertEqual(f'{""" # booo + """=}', '""" # booo\n """=\' # booo\\n \'') + + self.assertEqual(f'{" # nooo "=}', '" # nooo "=\' # nooo \'') + self.assertEqual(f'{" \" # nooo \" "=}', '" \\" # nooo \\" "=\' " # nooo " \'') + + self.assertEqual(f'{ # some comment goes here + """hello"""=}', ' \n """hello"""=\'hello\'') + self.assertEqual(f'{"""# this is not a comment + a""" # this is a comment + }', '# this is not a comment\n a') # These next lines contains tabs. Backslash escapes don't # work in f-strings. @@ -1817,65 +1683,68 @@ def __repr__(self): # this will be to dynamically created and exec the f-strings. But # that's such a hassle I'll save it for another day. For now, convert # the tabs to spaces just to shut up patchcheck. - # self.assertEqual(f'X{x =}Y', 'Xx\t='+repr(x)+'Y') - # self.assertEqual(f'X{x = }Y', 'Xx\t=\t'+repr(x)+'Y') + #self.assertEqual(f'X{x =}Y', 'Xx\t='+repr(x)+'Y') + #self.assertEqual(f'X{x = }Y', 'Xx\t=\t'+repr(x)+'Y') + + def test_debug_expressions_are_raw_strings(self): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', SyntaxWarning) + self.assertEqual(eval("""f'{b"\\N{OX}"=}'"""), 'b"\\N{OX}"=b\'\\\\N{OX}\'') + self.assertEqual(f'{r"\xff"=}', 'r"\\xff"=\'\\\\xff\'') + self.assertEqual(f'{r"\n"=}', 'r"\\n"=\'\\\\n\'') + self.assertEqual(f"{'\''=}", "'\\''=\"'\"") + self.assertEqual(f'{'\xc5'=}', r"'\xc5'='Å'") def test_walrus(self): x = 20 # This isn't an assignment expression, it's 'x', with a format # spec of '=10'. - self.assertEqual(f"{x:=10}", " 20") + self.assertEqual(f'{x:=10}', ' 20') # This is an assignment expression, which requires parens. - self.assertEqual(f"{(x:=10)}", "10") + self.assertEqual(f'{(x:=10)}', '10') self.assertEqual(x, 10) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid_syntax_error_message(self): - with self.assertRaisesRegex( - SyntaxError, "f-string: expecting '=', or '!', or ':', or '}'" - ): + with self.assertRaisesRegex(SyntaxError, + "f-string: expecting '=', or '!', or ':', or '}'"): compile("f'{a $ b}'", "?", "exec") def test_with_two_commas_in_format_specifier(self): error_msg = re.escape("Cannot specify ',' with ','.") with self.assertRaisesRegex(ValueError, error_msg): - f"{1:,,}" + f'{1:,,}' def test_with_two_underscore_in_format_specifier(self): error_msg = re.escape("Cannot specify '_' with '_'.") with self.assertRaisesRegex(ValueError, error_msg): - f"{1:__}" + f'{1:__}' def test_with_a_commas_and_an_underscore_in_format_specifier(self): error_msg = re.escape("Cannot specify both ',' and '_'.") with self.assertRaisesRegex(ValueError, error_msg): - f"{1:,_}" + f'{1:,_}' def test_with_an_underscore_and_a_comma_in_format_specifier(self): error_msg = re.escape("Cannot specify both ',' and '_'.") with self.assertRaisesRegex(ValueError, error_msg): - f"{1:_,}" + f'{1:_,}' - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_syntax_error_for_starred_expressions(self): with self.assertRaisesRegex(SyntaxError, "can't use starred expression here"): compile("f'{*a}'", "?", "exec") - with self.assertRaisesRegex( - SyntaxError, "f-string: expecting a valid expression after '{'" - ): + with self.assertRaisesRegex(SyntaxError, + "f-string: expecting a valid expression after '{'"): compile("f'{**a}'", "?", "exec") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_not_closing_quotes(self): self.assertAllRaise(SyntaxError, "unterminated f-string literal", ['f"', "f'"]) - self.assertAllRaise( - SyntaxError, "unterminated triple-quoted f-string literal", ['f"""', "f'''"] - ) + self.assertAllRaise(SyntaxError, "unterminated triple-quoted f-string literal", + ['f"""', "f'''"]) # Ensure that the errors are reported at the correct line number. data = '''\ x = 1 + 1 @@ -1891,56 +1760,125 @@ def test_not_closing_quotes(self): except SyntaxError as e: self.assertEqual(e.text, 'z = f"""') self.assertEqual(e.lineno, 3) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_syntax_error_after_debug(self): - self.assertAllRaise( - SyntaxError, - "f-string: expecting a valid expression after '{'", - [ - "f'{1=}{;'", - "f'{1=}{+;'", - "f'{1=}{2}{;'", - "f'{1=}{3}{;'", - ], - ) - self.assertAllRaise( - SyntaxError, - "f-string: expecting '=', or '!', or ':', or '}'", - [ - "f'{1=}{1;'", - "f'{1=}{1;}'", - ], - ) + self.assertAllRaise(SyntaxError, "f-string: expecting a valid expression after '{'", + [ + "f'{1=}{;'", + "f'{1=}{+;'", + "f'{1=}{2}{;'", + "f'{1=}{3}{;'", + ]) + self.assertAllRaise(SyntaxError, "f-string: expecting '=', or '!', or ':', or '}'", + [ + "f'{1=}{1;'", + "f'{1=}{1;}'", + ]) def test_debug_in_file(self): with temp_cwd(): - script = "script.py" - with open("script.py", "w") as f: + script = 'script.py' + with open('script.py', 'w') as f: f.write(f"""\ print(f'''{{ 3 =}}''')""") _, stdout, _ = assert_python_ok(script) - self.assertEqual( - stdout.decode("utf-8").strip().replace("\r\n", "\n").replace("\r", "\n"), - "3\n=3", - ) + self.assertEqual(stdout.decode('utf-8').strip().replace('\r\n', '\n').replace('\r', '\n'), + "3\n=3") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_syntax_warning_infinite_recursion_in_file(self): with temp_cwd(): - script = "script.py" - with open(script, "w") as f: + script = 'script.py' + with open(script, 'w') as f: f.write(r"print(f'\{1}')") _, stdout, stderr = assert_python_ok(script) - self.assertIn(rb"\1", stdout) + self.assertIn(rb'\1', stdout) self.assertEqual(len(stderr.strip().splitlines()), 2) + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'dis' has no attribute 'get_instructions' + def test_fstring_without_formatting_bytecode(self): + # f-string without any formatting should emit the same bytecode + # as a normal string. See gh-99606. + def get_code(s): + return [(i.opname, i.oparg) for i in dis.get_instructions(s)] + + for s in ["", "some string"]: + self.assertEqual(get_code(f"'{s}'"), get_code(f"f'{s}'")) + + def test_gh129093(self): + self.assertEqual(f'{1==2=}', '1==2=False') + self.assertEqual(f'{1 == 2=}', '1 == 2=False') + self.assertEqual(f'{1!=2=}', '1!=2=True') + self.assertEqual(f'{1 != 2=}', '1 != 2=True') + + self.assertEqual(f'{(1) != 2=}', '(1) != 2=True') + self.assertEqual(f'{(1*2) != (3)=}', '(1*2) != (3)=True') + + self.assertEqual(f'{1 != 2 == 3 != 4=}', '1 != 2 == 3 != 4=False') + self.assertEqual(f'{1 == 2 != 3 == 4=}', '1 == 2 != 3 == 4=False') + + self.assertEqual(f'{f'{1==2=}'=}', "f'{1==2=}'='1==2=False'") + self.assertEqual(f'{f'{1 == 2=}'=}', "f'{1 == 2=}'='1 == 2=False'") + self.assertEqual(f'{f'{1!=2=}'=}', "f'{1!=2=}'='1!=2=True'") + self.assertEqual(f'{f'{1 != 2=}'=}', "f'{1 != 2=}'='1 != 2=True'") + + def test_newlines_in_format_specifiers(self): + cases = [ + """f'{1:d\n}'""", + """f'__{ + 1:d + }__'""", + '''f"{value:. + {'2f'}}"''', + '''f"{value: + {'.2f'}f}"''', + '''f"{value: + #{'x'}}"''', + ] + self.assertAllRaise(SyntaxError, "f-string: newlines are not allowed in format specifiers", cases) + + valid_cases = [ + """f'''__{ + 1:d + }__'''""", + """f'''{1:d\n}'''""", + ] + + for case in valid_cases: + compile(case, "", "exec") + + def test_raw_fstring_format_spec(self): + # Test raw f-string format spec behavior (Issue #137314). + # + # Raw f-strings should preserve literal backslashes in format specifications, + # not interpret them as escape sequences. + class UnchangedFormat: + """Test helper that returns the format spec unchanged.""" + def __format__(self, format): + return format + + # Test basic escape sequences + self.assertEqual(f"{UnchangedFormat():\xFF}", 'ÿ') + self.assertEqual(rf"{UnchangedFormat():\xFF}", '\\xFF') + + # Test nested expressions with raw/non-raw combinations + self.assertEqual(rf"{UnchangedFormat():{'\xFF'}}", 'ÿ') + self.assertEqual(f"{UnchangedFormat():{r'\xFF'}}", '\\xFF') + self.assertEqual(rf"{UnchangedFormat():{r'\xFF'}}", '\\xFF') + + # Test continuation character in format specs + self.assertEqual(f"""{UnchangedFormat():{'a'\ + 'b'}}""", 'ab') + self.assertEqual(rf"""{UnchangedFormat():{'a'\ + 'b'}}""", 'ab') + + # Test multiple format specs in same raw f-string + self.assertEqual(rf"{UnchangedFormat():\xFF} {UnchangedFormat():\n}", '\\xFF \\n') + -if __name__ == "__main__": +if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index 2e28d3cfb7..3475c026bb 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -352,6 +352,14 @@ def test_invalid_paths(self): with self.assertRaisesRegex(ValueError, 'embedded null'): func(b'/tmp\x00abcds') + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON') + def test_samestat_on_symlink(self): + return super().test_samestat_on_symlink() + + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON') + def test_samefile_on_symlink(self): + return super().test_samefile_on_symlink() + # Following TestCase is not supposed to be run from test_genericpath. # It is inherited by other test modules (ntpath, posixpath). diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py index e11ea81a7d..c3fb8939a6 100644 --- a/Lib/test/test_glob.py +++ b/Lib/test/test_glob.py @@ -1,9 +1,12 @@ import glob import os +import re import shutil import sys import unittest +import warnings +from test.support import is_wasi, Py_DEBUG from test.support.os_helper import (TESTFN, skip_unless_symlink, can_symlink, create_empty_file, change_cwd) @@ -167,37 +170,45 @@ def test_glob_directory_names(self): self.norm('aab', 'F')]) def test_glob_directory_with_trailing_slash(self): - # Patterns ending with a slash shouldn't match non-dirs - res = glob.glob(self.norm('Z*Z') + os.sep) - self.assertEqual(res, []) - res = glob.glob(self.norm('ZZZ') + os.sep) - self.assertEqual(res, []) - # When there is a wildcard pattern which ends with os.sep, glob() - # doesn't blow up. - res = glob.glob(self.norm('aa*') + os.sep) - self.assertEqual(len(res), 2) - # either of these results is reasonable - self.assertIn(set(res), [ - {self.norm('aaa'), self.norm('aab')}, - {self.norm('aaa') + os.sep, self.norm('aab') + os.sep}, - ]) + seps = (os.sep, os.altsep) if os.altsep else (os.sep,) + for sep in seps: + # Patterns ending with a slash shouldn't match non-dirs + self.assertEqual(glob.glob(self.norm('Z*Z') + sep), []) + self.assertEqual(glob.glob(self.norm('ZZZ') + sep), []) + self.assertEqual(glob.glob(self.norm('aaa') + sep), + [self.norm('aaa') + sep]) + # Preserving the redundant separators is an implementation detail. + self.assertEqual(glob.glob(self.norm('aaa') + sep*2), + [self.norm('aaa') + sep*2]) + # When there is a wildcard pattern which ends with a pathname + # separator, glob() doesn't blow. + # The result should end with the pathname separator. + # Normalizing the trailing separator is an implementation detail. + eq = self.assertSequencesEqual_noorder + eq(glob.glob(self.norm('aa*') + sep), + [self.norm('aaa') + os.sep, self.norm('aab') + os.sep]) + # Stripping the redundant separators is an implementation detail. + eq(glob.glob(self.norm('aa*') + sep*2), + [self.norm('aaa') + os.sep, self.norm('aab') + os.sep]) def test_glob_bytes_directory_with_trailing_slash(self): # Same as test_glob_directory_with_trailing_slash, but with a # bytes argument. - res = glob.glob(os.fsencode(self.norm('Z*Z') + os.sep)) - self.assertEqual(res, []) - res = glob.glob(os.fsencode(self.norm('ZZZ') + os.sep)) - self.assertEqual(res, []) - res = glob.glob(os.fsencode(self.norm('aa*') + os.sep)) - self.assertEqual(len(res), 2) - # either of these results is reasonable - self.assertIn(set(res), [ - {os.fsencode(self.norm('aaa')), - os.fsencode(self.norm('aab'))}, - {os.fsencode(self.norm('aaa') + os.sep), - os.fsencode(self.norm('aab') + os.sep)}, - ]) + seps = (os.sep, os.altsep) if os.altsep else (os.sep,) + for sep in seps: + self.assertEqual(glob.glob(os.fsencode(self.norm('Z*Z') + sep)), []) + self.assertEqual(glob.glob(os.fsencode(self.norm('ZZZ') + sep)), []) + self.assertEqual(glob.glob(os.fsencode(self.norm('aaa') + sep)), + [os.fsencode(self.norm('aaa') + sep)]) + self.assertEqual(glob.glob(os.fsencode(self.norm('aaa') + sep*2)), + [os.fsencode(self.norm('aaa') + sep*2)]) + eq = self.assertSequencesEqual_noorder + eq(glob.glob(os.fsencode(self.norm('aa*') + sep)), + [os.fsencode(self.norm('aaa') + os.sep), + os.fsencode(self.norm('aab') + os.sep)]) + eq(glob.glob(os.fsencode(self.norm('aa*') + sep*2)), + [os.fsencode(self.norm('aaa') + os.sep), + os.fsencode(self.norm('aab') + os.sep)]) @skip_unless_symlink def test_glob_symlinks(self): @@ -205,8 +216,7 @@ def test_glob_symlinks(self): eq(self.glob('sym3'), [self.norm('sym3')]) eq(self.glob('sym3', '*'), [self.norm('sym3', 'EF'), self.norm('sym3', 'efg')]) - self.assertIn(self.glob('sym3' + os.sep), - [[self.norm('sym3')], [self.norm('sym3') + os.sep]]) + eq(self.glob('sym3' + os.sep), [self.norm('sym3') + os.sep]) eq(self.glob('*', '*F'), [self.norm('aaa', 'zzzF'), self.norm('aab', 'F'), self.norm('sym3', 'EF')]) @@ -364,6 +374,8 @@ def test_glob_named_pipe(self): self.assertEqual(self.rglob('mypipe', 'sub'), []) self.assertEqual(self.rglob('mypipe', '*'), []) + + @unittest.skipIf(is_wasi and Py_DEBUG, "requires too much stack") def test_glob_many_open_files(self): depth = 30 base = os.path.join(self.tempdir, 'deep') @@ -381,10 +393,134 @@ def test_glob_many_open_files(self): for it in iters: self.assertEqual(next(it), p) + def test_glob0(self): + with self.assertWarns(DeprecationWarning): + glob.glob0(self.tempdir, 'a') + + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + eq = self.assertSequencesEqual_noorder + eq(glob.glob0(self.tempdir, 'a'), ['a']) + eq(glob.glob0(self.tempdir, '.bb'), ['.bb']) + eq(glob.glob0(self.tempdir, '.b*'), []) + eq(glob.glob0(self.tempdir, 'b'), []) + eq(glob.glob0(self.tempdir, '?'), []) + eq(glob.glob0(self.tempdir, '*a'), []) + eq(glob.glob0(self.tempdir, 'a*'), []) + + def test_glob1(self): + with self.assertWarns(DeprecationWarning): + glob.glob1(self.tempdir, 'a') + + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + eq = self.assertSequencesEqual_noorder + eq(glob.glob1(self.tempdir, 'a'), ['a']) + eq(glob.glob1(self.tempdir, '.bb'), ['.bb']) + eq(glob.glob1(self.tempdir, '.b*'), ['.bb']) + eq(glob.glob1(self.tempdir, 'b'), []) + eq(glob.glob1(self.tempdir, '?'), ['a']) + eq(glob.glob1(self.tempdir, '*a'), ['a', 'aaa']) + eq(glob.glob1(self.tempdir, 'a*'), ['a', 'aaa', 'aab']) + + def test_translate_matching(self): + match = re.compile(glob.translate('*')).match + self.assertIsNotNone(match('foo')) + self.assertIsNotNone(match('foo.bar')) + self.assertIsNone(match('.foo')) + match = re.compile(glob.translate('.*')).match + self.assertIsNotNone(match('.foo')) + match = re.compile(glob.translate('**', recursive=True)).match + self.assertIsNotNone(match('foo')) + self.assertIsNone(match('.foo')) + self.assertIsNotNone(match(os.path.join('foo', 'bar'))) + self.assertIsNone(match(os.path.join('foo', '.bar'))) + self.assertIsNone(match(os.path.join('.foo', 'bar'))) + self.assertIsNone(match(os.path.join('.foo', '.bar'))) + match = re.compile(glob.translate('**/*', recursive=True)).match + self.assertIsNotNone(match(os.path.join('foo', 'bar'))) + self.assertIsNone(match(os.path.join('foo', '.bar'))) + self.assertIsNone(match(os.path.join('.foo', 'bar'))) + self.assertIsNone(match(os.path.join('.foo', '.bar'))) + match = re.compile(glob.translate('*/**', recursive=True)).match + self.assertIsNotNone(match(os.path.join('foo', 'bar'))) + self.assertIsNone(match(os.path.join('foo', '.bar'))) + self.assertIsNone(match(os.path.join('.foo', 'bar'))) + self.assertIsNone(match(os.path.join('.foo', '.bar'))) + match = re.compile(glob.translate('**/.bar', recursive=True)).match + self.assertIsNotNone(match(os.path.join('foo', '.bar'))) + self.assertIsNone(match(os.path.join('.foo', '.bar'))) + match = re.compile(glob.translate('**/*.*', recursive=True)).match + self.assertIsNone(match(os.path.join('foo', 'bar'))) + self.assertIsNone(match(os.path.join('foo', '.bar'))) + self.assertIsNotNone(match(os.path.join('foo', 'bar.txt'))) + self.assertIsNone(match(os.path.join('foo', '.bar.txt'))) + + def test_translate(self): + def fn(pat): + return glob.translate(pat, seps='/') + self.assertEqual(fn('foo'), r'(?s:foo)\Z') + self.assertEqual(fn('foo/bar'), r'(?s:foo/bar)\Z') + self.assertEqual(fn('*'), r'(?s:[^/.][^/]*)\Z') + self.assertEqual(fn('?'), r'(?s:(?!\.)[^/])\Z') + self.assertEqual(fn('a*'), r'(?s:a[^/]*)\Z') + self.assertEqual(fn('*a'), r'(?s:(?!\.)[^/]*a)\Z') + self.assertEqual(fn('.*'), r'(?s:\.[^/]*)\Z') + self.assertEqual(fn('?aa'), r'(?s:(?!\.)[^/]aa)\Z') + self.assertEqual(fn('aa?'), r'(?s:aa[^/])\Z') + self.assertEqual(fn('aa[ab]'), r'(?s:aa[ab])\Z') + self.assertEqual(fn('**'), r'(?s:(?!\.)[^/]*)\Z') + self.assertEqual(fn('***'), r'(?s:(?!\.)[^/]*)\Z') + self.assertEqual(fn('a**'), r'(?s:a[^/]*)\Z') + self.assertEqual(fn('**b'), r'(?s:(?!\.)[^/]*b)\Z') + self.assertEqual(fn('/**/*/*.*/**'), + r'(?s:/(?!\.)[^/]*/[^/.][^/]*/(?!\.)[^/]*\.[^/]*/(?!\.)[^/]*)\Z') + + def test_translate_include_hidden(self): + def fn(pat): + return glob.translate(pat, include_hidden=True, seps='/') + self.assertEqual(fn('foo'), r'(?s:foo)\Z') + self.assertEqual(fn('foo/bar'), r'(?s:foo/bar)\Z') + self.assertEqual(fn('*'), r'(?s:[^/]+)\Z') + self.assertEqual(fn('?'), r'(?s:[^/])\Z') + self.assertEqual(fn('a*'), r'(?s:a[^/]*)\Z') + self.assertEqual(fn('*a'), r'(?s:[^/]*a)\Z') + self.assertEqual(fn('.*'), r'(?s:\.[^/]*)\Z') + self.assertEqual(fn('?aa'), r'(?s:[^/]aa)\Z') + self.assertEqual(fn('aa?'), r'(?s:aa[^/])\Z') + self.assertEqual(fn('aa[ab]'), r'(?s:aa[ab])\Z') + self.assertEqual(fn('**'), r'(?s:[^/]*)\Z') + self.assertEqual(fn('***'), r'(?s:[^/]*)\Z') + self.assertEqual(fn('a**'), r'(?s:a[^/]*)\Z') + self.assertEqual(fn('**b'), r'(?s:[^/]*b)\Z') + self.assertEqual(fn('/**/*/*.*/**'), r'(?s:/[^/]*/[^/]+/[^/]*\.[^/]*/[^/]*)\Z') + + def test_translate_recursive(self): + def fn(pat): + return glob.translate(pat, recursive=True, include_hidden=True, seps='/') + self.assertEqual(fn('*'), r'(?s:[^/]+)\Z') + self.assertEqual(fn('?'), r'(?s:[^/])\Z') + self.assertEqual(fn('**'), r'(?s:.*)\Z') + self.assertEqual(fn('**/**'), r'(?s:.*)\Z') + self.assertEqual(fn('***'), r'(?s:[^/]*)\Z') + self.assertEqual(fn('a**'), r'(?s:a[^/]*)\Z') + self.assertEqual(fn('**b'), r'(?s:[^/]*b)\Z') + self.assertEqual(fn('/**/*/*.*/**'), r'(?s:/(?:.+/)?[^/]+/[^/]*\.[^/]*/.*)\Z') + + def test_translate_seps(self): + def fn(pat): + return glob.translate(pat, recursive=True, include_hidden=True, seps=['/', '\\']) + self.assertEqual(fn('foo/bar\\baz'), r'(?s:foo[/\\]bar[/\\]baz)\Z') + self.assertEqual(fn('**/*'), r'(?s:(?:.+[/\\])?[^/\\]+)\Z') + @skip_unless_symlink class SymlinkLoopGlobTests(unittest.TestCase): + # gh-109959: On Linux, glob._isdir() and glob._lexists() can return False + # randomly when checking the "link/" symbolic link. + # https://github.com/python/cpython/issues/109959#issuecomment-2577550700 + @unittest.skip("flaky test") def test_selflink(self): tempdir = TESTFN + "_dir" os.makedirs(tempdir) diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py index 61fa24fab5..380bbe4017 100644 --- a/Lib/test/test_htmlparser.py +++ b/Lib/test/test_htmlparser.py @@ -5,6 +5,7 @@ import unittest from unittest.mock import patch +from test import support class EventCollector(html.parser.HTMLParser): @@ -80,6 +81,13 @@ def handle_entityref(self, data): self.fail('This should never be called with convert_charrefs=True') +# The normal event collector normalizes the events in get_events, +# so we override it to return the original list of events. +class EventCollectorNoNormalize(EventCollector): + def get_events(self): + return self.events + + class TestCaseBase(unittest.TestCase): def get_collector(self): @@ -264,8 +272,7 @@ def test_get_starttag_text(self): ("starttag", "foo:bar", [("one", "1"), ("two", "2")]), ("starttag_text", s)]) - def test_cdata_content(self): - contents = [ + @support.subTests('content', [ ' ¬-an-entity-ref;', "", '

', @@ -278,70 +285,223 @@ def test_cdata_content(self): 'src="http://www.example.org/r=\'+new ' 'Date().getTime()+\'"><\\/s\'+\'cript>\');\n//]]>'), '\n\n', - 'foo = "";', '', - # these two should be invalid according to the HTML 5 spec, - # section 8.1.2.2 - #'foo = ', - #'foo = ', - ] - elements = ['script', 'style', 'SCRIPT', 'STYLE', 'Script', 'Style'] - for content in contents: - for element in elements: - element_lower = element.lower() - s = '<{element}>{content}'.format(element=element, - content=content) - self._run_check(s, [("starttag", element_lower, []), - ("data", content), - ("endtag", element_lower)]) - - def test_cdata_with_closing_tags(self): + 'foo = ""', + 'foo = ""', + 'foo = ""', + 'foo = ""', + 'foo = ""', + 'foo = ""', + ]) + def test_script_content(self, content): + s = f'' + self._run_check(s, [("starttag", "script", []), + ("data", content), + ("endtag", "script")]) + + @support.subTests('content', [ + 'a::before { content: ""; }', + 'a::before { content: "¬-an-entity-ref;"; }', + 'a::before { content: ""; }', + 'a::before { content: "\u2603"; }', + 'a::before { content: "< /style>"; }', + 'a::before { content: ""; }', + 'a::before { content: ""; }', + 'a::before { content: ""; }', + 'a::before { content: ""; }', + 'a::before { content: ""; }', + ]) + def test_style_content(self, content): + s = f'' + self._run_check(s, [("starttag", "style", []), + ("data", content), + ("endtag", "style")]) + + @support.subTests('content', [ + '', + "", + '', + '', + '', + '\u2603', + '< /title>', + '', + '', + '', + '', + '', + ]) + def test_title_content(self, content): + source = f"{content}" + self._run_check(source, [ + ("starttag", "title", []), + ("data", content), + ("endtag", "title"), + ]) + + @support.subTests('content', [ + '', + "", + '', + '', + '', + '\u2603', + '< /textarea>', + '', + '', + '', + '', + ]) + def test_textarea_content(self, content): + source = f"" + self._run_check(source, [ + ("starttag", "textarea", []), + ("data", content), + ("endtag", "textarea"), + ]) + + @support.subTests('endtag', ['script', 'SCRIPT', 'script ', 'script\n', + 'script/', 'script foo=bar', 'script foo=">"']) + def test_script_closing_tag(self, endtag): # see issue #13358 # make sure that HTMLParser calls handle_data only once for each CDATA. - # The normal event collector normalizes the events in get_events, - # so we override it to return the original list of events. - class Collector(EventCollector): - def get_events(self): - return self.events - content = """ ¬-an-entity-ref;

''""" - for element in [' script', 'script ', ' script ', - '\nscript', 'script\n', '\nscript\n']: - element_lower = element.lower().strip() - s = '{content}{tail}' + self._run_check(s, [("starttag", "script", []), + ("data", content if end else content + tail)], + collector=EventCollectorNoNormalize(convert_charrefs=False)) + + @support.subTests('tail,end', [ + ('', False), + ('<', False), + ('', True), + ]) + def test_eof_in_title(self, tail, end): + s = f'Egg & Spam{tail}' + self._run_check(s, [("starttag", "title", []), + ("data", "Egg & Spam" + ('' if end else tail))], + collector=EventCollectorNoNormalize(convert_charrefs=True)) + self._run_check(s, [("starttag", "title", []), + ('data', 'Egg '), + ('entityref', 'amp'), + ('data', ' Spam' + ('' if end else tail))], + collector=EventCollectorNoNormalize(convert_charrefs=False)) def test_comments(self): html = ("<!-- I'm a valid comment -->" '<!--me too!-->' '<!------>' + '<!----->' '<!---->' + # abrupt-closing-of-empty-comment + '<!--->' + '<!-->' '<!----I have many hyphens---->' '<!-- I have a > in the middle -->' - '<!-- and I have -- in the middle! -->') + '<!-- and I have -- in the middle! -->' + '<!--incorrectly-closed-comment--!>' + '<!----!>' + '<!----!-->' + '<!---- >-->' + '<!---!>-->' + '<!--!>-->' + # nested-comment + '<!-- <!-- nested --> -->' + '<!--<!-->' + '<!--<!--!>' + ) expected = [('comment', " I'm a valid comment "), ('comment', 'me too!'), ('comment', '--'), + ('comment', '-'), + ('comment', ''), + ('comment', ''), ('comment', ''), ('comment', '--I have many hyphens--'), ('comment', ' I have a > in the middle '), - ('comment', ' and I have -- in the middle! ')] + ('comment', ' and I have -- in the middle! '), + ('comment', 'incorrectly-closed-comment'), + ('comment', ''), + ('comment', '--!'), + ('comment', '-- >'), + ('comment', '-!>'), + ('comment', '!>'), + ('comment', ' <!-- nested '), ('data', ' -->'), + ('comment', '<!'), + ('comment', '<!'), + ] self._run_check(html, expected) def test_condcoms(self): @@ -430,28 +590,34 @@ def test_tolerant_parsing(self): ('data', '<'), ('starttag', 'bc<', [('a', None)]), ('endtag', 'html'), - ('data', '\n<img src="URL>'), - ('comment', '/img'), - ('endtag', 'html<')]) + ('data', '\n')]) def test_starttag_junk_chars(self): + self._run_check("<", [('data', '<')]) + self._run_check("<>", [('data', '<>')]) + self._run_check("< >", [('data', '< >')]) + self._run_check("< ", [('data', '< ')]) self._run_check("</>", []) + self._run_check("<$>", [('data', '<$>')]) self._run_check("</$>", [('comment', '$')]) self._run_check("</", [('data', '</')]) - self._run_check("</a", [('data', '</a')]) + self._run_check("</a", []) + self._run_check("</ a>", [('comment', ' a')]) + self._run_check("</ a", [('comment', ' a')]) self._run_check("<a<a>", [('starttag', 'a<a', [])]) self._run_check("</a<a>", [('endtag', 'a<a')]) - self._run_check("<!", [('data', '<!')]) - self._run_check("<a", [('data', '<a')]) - self._run_check("<a foo='bar'", [('data', "<a foo='bar'")]) - self._run_check("<a foo='bar", [('data', "<a foo='bar")]) - self._run_check("<a foo='>'", [('data', "<a foo='>'")]) - self._run_check("<a foo='>", [('data', "<a foo='>")]) + self._run_check("<!", [('comment', '')]) + self._run_check("<a", []) + self._run_check("<a foo='bar'", []) + self._run_check("<a foo='bar", []) + self._run_check("<a foo='>'", []) + self._run_check("<a foo='>", []) self._run_check("<a$>", [('starttag', 'a$', [])]) self._run_check("<a$b>", [('starttag', 'a$b', [])]) self._run_check("<a$b/>", [('startendtag', 'a$b', [])]) self._run_check("<a$b >", [('starttag', 'a$b', [])]) self._run_check("<a$b />", [('startendtag', 'a$b', [])]) + self._run_check("</a$b>", [('endtag', 'a$b')]) def test_slashes_in_starttag(self): self._run_check('<a foo="var"/>', [('startendtag', 'a', [('foo', 'var')])]) @@ -484,6 +650,10 @@ def test_slashes_in_starttag(self): ] self._run_check(html, expected) + def test_slashes_in_endtag(self): + self._run_check('</a/>', [('endtag', 'a')]) + self._run_check('</a foo="var"/>', [('endtag', 'a')]) + def test_declaration_junk_chars(self): self._run_check("<!DOCTYPE foo $ >", [('decl', 'DOCTYPE foo $ ')]) @@ -518,15 +688,11 @@ def test_invalid_end_tags(self): self._run_check(html, expected) def test_broken_invalid_end_tag(self): - # This is technically wrong (the "> shouldn't be included in the 'data') - # but is probably not worth fixing it (in addition to all the cases of - # the previous test, it would require a full attribute parsing). - # see #13993 html = '<b>This</b attr=">"> confuses the parser' expected = [('starttag', 'b', []), ('data', 'This'), ('endtag', 'b'), - ('data', '"> confuses the parser')] + ('data', ' confuses the parser')] self._run_check(html, expected) def test_correct_detection_of_start_tags(self): @@ -576,21 +742,50 @@ def test_EOF_in_charref(self): for html, expected in data: self._run_check(html, expected) - def test_EOF_in_comments_or_decls(self): + def test_eof_in_comments(self): data = [ - ('<!', [('data', '<!')]), - ('<!-', [('data', '<!-')]), - ('<!--', [('data', '<!--')]), - ('<![', [('data', '<![')]), - ('<![CDATA[', [('data', '<![CDATA[')]), - ('<![CDATA[x', [('data', '<![CDATA[x')]), - ('<!DOCTYPE', [('data', '<!DOCTYPE')]), - ('<!DOCTYPE HTML', [('data', '<!DOCTYPE HTML')]), + ('<!--', [('comment', '')]), + ('<!---', [('comment', '')]), + ('<!----', [('comment', '')]), + ('<!-----', [('comment', '-')]), + ('<!------', [('comment', '--')]), + ('<!----!', [('comment', '')]), + ('<!---!', [('comment', '-!')]), + ('<!---!>', [('comment', '-!>')]), + ('<!--foo', [('comment', 'foo')]), + ('<!--foo-', [('comment', 'foo')]), + ('<!--foo--', [('comment', 'foo')]), + ('<!--foo--!', [('comment', 'foo')]), + ('<!--<!--', [('comment', '<!')]), + ('<!--<!--!', [('comment', '<!')]), ] for html, expected in data: self._run_check(html, expected) + + def test_eof_in_declarations(self): + data = [ + ('<!', [('comment', '')]), + ('<!-', [('comment', '-')]), + ('<![', [('comment', '[')]), + ('<![CDATA[', [('unknown decl', 'CDATA[')]), + ('<![CDATA[x', [('unknown decl', 'CDATA[x')]), + ('<![CDATA[x]', [('unknown decl', 'CDATA[x]')]), + ('<![CDATA[x]]', [('unknown decl', 'CDATA[x]]')]), + ('<!DOCTYPE', [('decl', 'DOCTYPE')]), + ('<!DOCTYPE ', [('decl', 'DOCTYPE ')]), + ('<!DOCTYPE html', [('decl', 'DOCTYPE html')]), + ('<!DOCTYPE html ', [('decl', 'DOCTYPE html ')]), + ('<!DOCTYPE html PUBLIC', [('decl', 'DOCTYPE html PUBLIC')]), + ('<!DOCTYPE html PUBLIC "foo', [('decl', 'DOCTYPE html PUBLIC "foo')]), + ('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "foo', + [('decl', 'DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "foo')]), + ] + for html, expected in data: + self._run_check(html, expected) + def test_bogus_comments(self): - html = ('<! not really a comment >' + html = ('<!ELEMENT br EMPTY>' + '<! not really a comment >' '<! not a comment either -->' '<! -- close enough -->' '<!><!<-- this was an empty comment>' @@ -604,6 +799,7 @@ def test_bogus_comments(self): '<![CDATA]]>' # required '[' after CDATA ) expected = [ + ('comment', 'ELEMENT br EMPTY'), ('comment', ' not really a comment '), ('comment', ' not a comment either --'), ('comment', ' -- close enough --'), @@ -684,6 +880,26 @@ def test_convert_charrefs_dropped_text(self): ('endtag', 'a'), ('data', ' bar & baz')] ) + @support.requires_resource('cpu') + def test_eof_no_quadratic_complexity(self): + # Each of these examples used to take about an hour. + # Now they take a fraction of a second. + def check(source): + parser = html.parser.HTMLParser() + parser.feed(source) + parser.close() + n = 120_000 + check("<a " * n) + check("<a a=" * n) + check("</a " * 14 * n) + check("</a a=" * 11 * n) + check("<!--" * 4 * n) + check("<!" * 60 * n) + check("<?" * 19 * n) + check("</$" * 15 * n) + check("<![CDATA[" * 9 * n) + check("<!doctype" * 35 * n) + class AttributesTestCase(TestCaseBase): @@ -692,9 +908,15 @@ def test_attr_syntax(self): ("starttag", "a", [("b", "v"), ("c", "v"), ("d", "v"), ("e", None)]) ] self._run_check("""<a b='v' c="v" d=v e>""", output) - self._run_check("""<a b = 'v' c = "v" d = v e>""", output) - self._run_check("""<a\nb\n=\n'v'\nc\n=\n"v"\nd\n=\nv\ne>""", output) - self._run_check("""<a\tb\t=\t'v'\tc\t=\t"v"\td\t=\tv\te>""", output) + self._run_check("<a foo==bar>", [('starttag', 'a', [('foo', '=bar')])]) + self._run_check("<a foo =bar>", [('starttag', 'a', [('foo', 'bar')])]) + self._run_check("<a foo\t=bar>", [('starttag', 'a', [('foo', 'bar')])]) + self._run_check("<a foo\v=bar>", [('starttag', 'a', [('foo\v', 'bar')])]) + self._run_check("<a foo\xa0=bar>", [('starttag', 'a', [('foo\xa0', 'bar')])]) + self._run_check("<a foo= bar>", [('starttag', 'a', [('foo', 'bar')])]) + self._run_check("<a foo=\tbar>", [('starttag', 'a', [('foo', 'bar')])]) + self._run_check("<a foo=\vbar>", [('starttag', 'a', [('foo', '\vbar')])]) + self._run_check("<a foo=\xa0bar>", [('starttag', 'a', [('foo', '\xa0bar')])]) def test_attr_values(self): self._run_check("""<a b='xxx\n\txxx' c="yyy\t\nyyy" d='\txyz\n'>""", @@ -703,6 +925,10 @@ def test_attr_values(self): ("d", "\txyz\n")])]) self._run_check("""<a b='' c="">""", [("starttag", "a", [("b", ""), ("c", "")])]) + self._run_check("<a b=\tx c=\ny>", + [('starttag', 'a', [('b', 'x'), ('c', 'y')])]) + self._run_check("<a b=\v c=\xa0>", + [("starttag", "a", [("b", "\v"), ("c", "\xa0")])]) # Regression test for SF patch #669683. self._run_check("<e a=rgb(1,2,3)>", [("starttag", "e", [("a", "rgb(1,2,3)")])]) @@ -769,13 +995,17 @@ def test_malformed_attributes(self): ) expected = [ ('starttag', 'a', [('href', "test'style='color:red;bad1'")]), - ('data', 'test - bad1'), ('endtag', 'a'), + ('data', 'test - bad1'), + ('endtag', 'a'), ('starttag', 'a', [('href', "test'+style='color:red;ba2'")]), - ('data', 'test - bad2'), ('endtag', 'a'), + ('data', 'test - bad2'), + ('endtag', 'a'), ('starttag', 'a', [('href', "test'\xa0style='color:red;bad3'")]), - ('data', 'test - bad3'), ('endtag', 'a'), + ('data', 'test - bad3'), + ('endtag', 'a'), ('starttag', 'a', [('href', "test'\xa0style='color:red;bad4'")]), - ('data', 'test - bad4'), ('endtag', 'a') + ('data', 'test - bad4'), + ('endtag', 'a'), ] self._run_check(html, expected) diff --git a/Lib/test/test_importlib/source/test_file_loader.py b/Lib/test/test_importlib/source/test_file_loader.py index 9ade197a23..d487fc9b82 100644 --- a/Lib/test/test_importlib/source/test_file_loader.py +++ b/Lib/test/test_importlib/source/test_file_loader.py @@ -236,7 +236,6 @@ def test_unloadable(self): warnings.simplefilter('ignore', DeprecationWarning) loader.load_module('bad name') - @unittest.skip("TODO: RUSTPYTHON; successful only for Frozen") @util.writes_bytecode_files def test_checked_hash_based_pyc(self): with util.create_modules('_temp') as mapping: @@ -293,7 +292,6 @@ def test_overridden_checked_hash_based_pyc(self): loader.exec_module(mod) self.assertEqual(mod.state, 'old') - @unittest.skip("TODO: RUSTPYTHON; successful only for Frozen") @util.writes_bytecode_files def test_unchecked_hash_based_pyc(self): with util.create_modules('_temp') as mapping: @@ -324,7 +322,6 @@ def test_unchecked_hash_based_pyc(self): data[8:16], ) - @unittest.skip("TODO: RUSTPYTHON; successful only for Frozen") @util.writes_bytecode_files def test_overridden_unchecked_hash_based_pyc(self): with util.create_modules('_temp') as mapping, \ @@ -362,6 +359,23 @@ def test_overridden_unchecked_hash_based_pyc(self): ) = util.test_both(SimpleTest, importlib=importlib, machinery=machinery, abc=importlib_abc, util=importlib_util) +# TODO: RUSTPYTHON, get rid of this entire class when all of the following tests are fixed +class Source_SimpleTest(Source_SimpleTest): + # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed + @unittest.expectedFailure + def test_checked_hash_based_pyc(self): + super().test_checked_hash_based_pyc() + + # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed + @unittest.expectedFailure + def test_unchecked_hash_based_pyc(self): + super().test_unchecked_hash_based_pyc() + + # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed + @unittest.expectedFailure + def test_overridden_unchecked_hash_based_pyc(self): + super().test_overridden_unchecked_hash_based_pyc() + class SourceDateEpochTestMeta(SourceDateEpochTestMeta, type(Source_SimpleTest)): @@ -677,24 +691,57 @@ class SourceLoaderBadBytecodeTestPEP451( pass -# (Frozen_SourceBadBytecodePEP451, -# Source_SourceBadBytecodePEP451 -# ) = util.test_both(SourceLoaderBadBytecodeTestPEP451, importlib=importlib, -# machinery=machinery, abc=importlib_abc, -# util=importlib_util) +(Frozen_SourceBadBytecodePEP451, + Source_SourceBadBytecodePEP451 + ) = util.test_both(SourceLoaderBadBytecodeTestPEP451, importlib=importlib, + machinery=machinery, abc=importlib_abc, + util=importlib_util) + +# TODO: RUSTPYTHON, get rid of this entire class when all of the following tests are fixed +class Source_SourceBadBytecodePEP451(Source_SourceBadBytecodePEP451): + # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed + @unittest.expectedFailure + def test_bad_marshal(self): + super().test_bad_marshal() + + # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed + @unittest.expectedFailure + def test_no_marshal(self): + super().test_no_marshal() + + # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed + @unittest.expectedFailure + def test_non_code_marshal(self): + super().test_non_code_marshal() + +class SourceLoaderBadBytecodeTestPEP302( + SourceLoaderBadBytecodeTest, BadBytecodeTestPEP302): + pass -# TODO: RUSTPYTHON -# class SourceLoaderBadBytecodeTestPEP302( -# SourceLoaderBadBytecodeTest, BadBytecodeTestPEP302): -# pass +(Frozen_SourceBadBytecodePEP302, + Source_SourceBadBytecodePEP302 + ) = util.test_both(SourceLoaderBadBytecodeTestPEP302, importlib=importlib, + machinery=machinery, abc=importlib_abc, + util=importlib_util) -# (Frozen_SourceBadBytecodePEP302, -# Source_SourceBadBytecodePEP302 -# ) = util.test_both(SourceLoaderBadBytecodeTestPEP302, importlib=importlib, -# machinery=machinery, abc=importlib_abc, -# util=importlib_util) +# TODO: RUSTPYTHON, get rid of this entire class when all of the following tests are fixed +class Source_SourceBadBytecodePEP302(Source_SourceBadBytecodePEP302): + # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed + @unittest.expectedFailure + def test_bad_marshal(self): + super().test_bad_marshal() + + # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed + @unittest.expectedFailure + def test_no_marshal(self): + super().test_no_marshal() + + # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed + @unittest.expectedFailure + def test_non_code_marshal(self): + super().test_non_code_marshal() class SourcelessLoaderBadBytecodeTest: @@ -776,23 +823,87 @@ class SourcelessLoaderBadBytecodeTestPEP451(SourcelessLoaderBadBytecodeTest, pass -# (Frozen_SourcelessBadBytecodePEP451, -# Source_SourcelessBadBytecodePEP451 -# ) = util.test_both(SourcelessLoaderBadBytecodeTestPEP451, importlib=importlib, -# machinery=machinery, abc=importlib_abc, -# util=importlib_util) +(Frozen_SourcelessBadBytecodePEP451, + Source_SourcelessBadBytecodePEP451 + ) = util.test_both(SourcelessLoaderBadBytecodeTestPEP451, importlib=importlib, + machinery=machinery, abc=importlib_abc, + util=importlib_util) + +# TODO: RUSTPYTHON, get rid of this entire class when all of the following tests are fixed +class Source_SourcelessBadBytecodePEP451(Source_SourcelessBadBytecodePEP451): + # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed + @unittest.expectedFailure + def test_magic_only(self): + super().test_magic_only() + + # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed + @unittest.expectedFailure + def test_no_marshal(self): + super().test_no_marshal() + + # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed + @unittest.expectedFailure + def test_partial_flags(self): + super().test_partial_flags() + + # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed + @unittest.expectedFailure + def test_partial_hash(self): + super().test_partial_hash() + + # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed + @unittest.expectedFailure + def test_partial_size(self): + super().test_partial_size() + + # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed + @unittest.expectedFailure + def test_partial_timestamp(self): + super().test_partial_timestamp() + + +class SourcelessLoaderBadBytecodeTestPEP302(SourcelessLoaderBadBytecodeTest, + BadBytecodeTestPEP302): + pass + + +(Frozen_SourcelessBadBytecodePEP302, + Source_SourcelessBadBytecodePEP302 + ) = util.test_both(SourcelessLoaderBadBytecodeTestPEP302, importlib=importlib, + machinery=machinery, abc=importlib_abc, + util=importlib_util) +# TODO: RUSTPYTHON, get rid of this entire class when all of the following tests are fixed +class Source_SourcelessBadBytecodePEP302(Source_SourcelessBadBytecodePEP302): + # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed + @unittest.expectedFailure + def test_magic_only(self): + super().test_magic_only() -# class SourcelessLoaderBadBytecodeTestPEP302(SourcelessLoaderBadBytecodeTest, -# BadBytecodeTestPEP302): -# pass + # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed + @unittest.expectedFailure + def test_no_marshal(self): + super().test_no_marshal() + # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed + @unittest.expectedFailure + def test_partial_flags(self): + super().test_partial_flags() -# (Frozen_SourcelessBadBytecodePEP302, -# Source_SourcelessBadBytecodePEP302 -# ) = util.test_both(SourcelessLoaderBadBytecodeTestPEP302, importlib=importlib, -# machinery=machinery, abc=importlib_abc, -# util=importlib_util) + # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed + @unittest.expectedFailure + def test_partial_hash(self): + super().test_partial_hash() + + # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed + @unittest.expectedFailure + def test_partial_size(self): + super().test_partial_size() + + # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed + @unittest.expectedFailure + def test_partial_timestamp(self): + super().test_partial_timestamp() if __name__ == '__main__': diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index e91d454b72..272098782d 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -39,11 +39,9 @@ from test import support from test.support.script_helper import ( assert_python_ok, assert_python_failure, run_python_until_end) -from test.support import import_helper -from test.support import os_helper -from test.support import threading_helper -from test.support import warnings_helper -from test.support import skip_if_sanitizer +from test.support import ( + import_helper, is_apple, os_helper, threading_helper, warnings_helper, +) from test.support.os_helper import FakePath import codecs @@ -66,10 +64,6 @@ def byteslike(*pos, **kw): class EmptyStruct(ctypes.Structure): pass -# Does io.IOBase finalizer log the exception if the close() method fails? -# The exception is ignored silently by default in release build. -IOBASE_EMITS_UNRAISABLE = (support.Py_DEBUG or sys.flags.dev_mode) - def _default_chunk_size(): """Get the default TextIOWrapper chunk size""" @@ -631,10 +625,10 @@ def test_raw_bytes_io(self): self.read_ops(f, True) def test_large_file_ops(self): - # On Windows and Mac OSX this test consumes large resources; It takes - # a long time to build the >2 GiB file and takes >2 GiB of disk space - # therefore the resource must be enabled to run this test. - if sys.platform[:3] == 'win' or sys.platform == 'darwin': + # On Windows and Apple platforms this test consumes large resources; It + # takes a long time to build the >2 GiB file and takes >2 GiB of disk + # space therefore the resource must be enabled to run this test. + if sys.platform[:3] == 'win' or is_apple: support.requires( 'largefile', 'test requires %s bytes and a long time to run' % self.LARGE) @@ -645,11 +639,9 @@ def test_large_file_ops(self): def test_with_open(self): for bufsize in (0, 100): - f = None with self.open(os_helper.TESTFN, "wb", bufsize) as f: f.write(b"xxx") self.assertEqual(f.closed, True) - f = None try: with self.open(os_helper.TESTFN, "wb", bufsize) as f: 1/0 @@ -788,8 +780,7 @@ def test_closefd_attr(self): file = self.open(f.fileno(), "r", encoding="utf-8", closefd=False) self.assertEqual(file.buffer.raw.closefd, False) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_garbage_collection(self): # FileIO objects are collected, and collecting them flushes # all data to disk. @@ -904,7 +895,7 @@ def test_bad_opener_negative_1(self): def badopener(fname, flags): return -1 with self.assertRaises(ValueError) as cm: - open('non-existent', 'r', opener=badopener) + self.open('non-existent', 'r', opener=badopener) self.assertEqual(str(cm.exception), 'opener returned -1') def test_bad_opener_other_negative(self): @@ -912,7 +903,7 @@ def test_bad_opener_other_negative(self): def badopener(fname, flags): return -2 with self.assertRaises(ValueError) as cm: - open('non-existent', 'r', opener=badopener) + self.open('non-existent', 'r', opener=badopener) self.assertEqual(str(cm.exception), 'opener returned -2') def test_opener_invalid_fd(self): @@ -1048,11 +1039,41 @@ def flush(self): # Silence destructor error R.flush = lambda self: None + @threading_helper.requires_working_threading() + def test_write_readline_races(self): + # gh-134908: Concurrent iteration over a file caused races + thread_count = 2 + write_count = 100 + read_count = 100 + + def writer(file, barrier): + barrier.wait() + for _ in range(write_count): + file.write("x") + + def reader(file, barrier): + barrier.wait() + for _ in range(read_count): + for line in file: + self.assertEqual(line, "") + + with self.open(os_helper.TESTFN, "w+") as f: + barrier = threading.Barrier(thread_count + 1) + reader = threading.Thread(target=reader, args=(f, barrier)) + writers = [threading.Thread(target=writer, args=(f, barrier)) + for _ in range(thread_count)] + with threading_helper.catch_threading_exception() as cm: + with threading_helper.start_threads(writers + [reader]): + pass + self.assertIsNone(cm.exc_type) + + self.assertEqual(os.stat(os_helper.TESTFN).st_size, + write_count * thread_count) + class CIOTest(IOTest): - # TODO: RUSTPYTHON, cyclic gc - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; cyclic gc def test_IOBase_finalize(self): # Issue #12149: segmentation fault on _PyIOBase_finalize when both a # class which inherits IOBase and an object of this class are caught @@ -1071,10 +1092,9 @@ def close(self): support.gc_collect() self.assertIsNone(wr(), wr) - # TODO: RUSTPYTHON, AssertionError: filter ('', ResourceWarning) did not catch any warning - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: filter ('', ResourceWarning) did not catch any warning def test_destructor(self): - super().test_destructor(self) + return super().test_destructor() @support.cpython_only class TestIOCTypes(unittest.TestCase): @@ -1165,9 +1185,32 @@ def test_disallow_instantiation(self): _io = self._io support.check_disallow_instantiation(self, _io._BytesIOBuffer) + def test_stringio_setstate(self): + # gh-127182: Calling __setstate__() with invalid arguments must not crash + obj = self._io.StringIO() + with self.assertRaisesRegex( + TypeError, + 'initial_value must be str or None, not int', + ): + obj.__setstate__((1, '', 0, {})) + + obj.__setstate__((None, '', 0, {})) # should not crash + self.assertEqual(obj.getvalue(), '') + + obj.__setstate__(('', '', 0, {})) + self.assertEqual(obj.getvalue(), '') + class PyIOTest(IOTest): pass + @unittest.expectedFailure # TODO: RUSTPYTHON; OSError: Negative file descriptor + def test_bad_opener_negative_1(): + return super().test_bad_opener_negative_1() + + @unittest.expectedFailure # TODO: RUSTPYTHON; OSError: Negative file descriptor + def test_bad_opener_other_negative(): + return super().test_bad_opener_other_negative() + @support.cpython_only class APIMismatchTest(unittest.TestCase): @@ -1175,7 +1218,7 @@ class APIMismatchTest(unittest.TestCase): def test_RawIOBase_io_in_pyio_match(self): """Test that pyio RawIOBase class has all c RawIOBase methods""" mismatch = support.detect_api_mismatch(pyio.RawIOBase, io.RawIOBase, - ignore=('__weakref__',)) + ignore=('__weakref__', '__static_attributes__')) self.assertEqual(mismatch, set(), msg='Python RawIOBase does not have all C RawIOBase methods') def test_RawIOBase_pyio_in_io_match(self): @@ -1244,6 +1287,7 @@ def _with(): # a ValueError. self.assertRaises(ValueError, _with) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_error_through_destructor(self): # Test that the exception state is not modified by a destructor, # even if close() fails. @@ -1252,10 +1296,7 @@ def test_error_through_destructor(self): with self.assertRaises(AttributeError): self.tp(rawio).xyzzy - if not IOBASE_EMITS_UNRAISABLE: - self.assertIsNone(cm.unraisable) - elif cm.unraisable is not None: - self.assertEqual(cm.unraisable.exc_type, OSError) + self.assertEqual(cm.unraisable.exc_type, OSError) def test_repr(self): raw = self.MockRawIO() @@ -1271,11 +1312,9 @@ def test_recursive_repr(self): # Issue #25455 raw = self.MockRawIO() b = self.tp(raw) - with support.swap_attr(raw, 'name', b): - try: + with support.swap_attr(raw, 'name', b), support.infinite_recursion(25): + with self.assertRaises(RuntimeError): repr(b) # Should not crash - except RuntimeError: - pass def test_flush_error_on_close(self): # Test that buffered file is closed despite failed flush @@ -1356,6 +1395,28 @@ def test_readonly_attributes(self): with self.assertRaises(AttributeError): buf.raw = x + def test_pickling_subclass(self): + global MyBufferedIO + class MyBufferedIO(self.tp): + def __init__(self, raw, tag): + super().__init__(raw) + self.tag = tag + def __getstate__(self): + return self.tag, self.raw.getvalue() + def __setstate__(slf, state): + tag, value = state + slf.__init__(self.BytesIO(value), tag) + + raw = self.BytesIO(b'data') + buf = MyBufferedIO(raw, tag='ham') + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + pickled = pickle.dumps(buf, proto) + newbuf = pickle.loads(pickled) + self.assertEqual(newbuf.raw.getvalue(), b'data') + self.assertEqual(newbuf.tag, 'ham') + del MyBufferedIO + class SizeofTest: @@ -1717,20 +1778,6 @@ def test_seek_character_device_file(self): class CBufferedReaderTest(BufferedReaderTest, SizeofTest): tp = io.BufferedReader - @unittest.skip("TODO: RUSTPYTHON, fallible allocation") - @skip_if_sanitizer(memory=True, address=True, thread=True, - reason="sanitizer defaults to crashing " - "instead of returning NULL for malloc failure.") - def test_constructor(self): - BufferedReaderTest.test_constructor(self) - # The allocation can succeed on 32-bit builds, e.g. with more - # than 2 GiB RAM and a 64-bit kernel. - if sys.maxsize > 0x7FFFFFFF: - rawio = self.MockRawIO() - bufio = self.tp(rawio) - self.assertRaises((OverflowError, MemoryError, ValueError), - bufio.__init__, rawio, sys.maxsize) - def test_initialization(self): rawio = self.MockRawIO([b"abc"]) bufio = self.tp(rawio) @@ -1748,8 +1795,7 @@ def test_misbehaved_io_read(self): # checking this is not so easy. self.assertRaises(OSError, bufio.read, 10) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_garbage_collection(self): # C BufferedReader objects are collected. # The Python version has __del__, so it ends into gc.garbage instead @@ -1766,40 +1812,44 @@ def test_garbage_collection(self): def test_args_error(self): # Issue #17275 with self.assertRaisesRegex(TypeError, "BufferedReader"): - self.tp(io.BytesIO(), 1024, 1024, 1024) + self.tp(self.BytesIO(), 1024, 1024, 1024) def test_bad_readinto_value(self): - rawio = io.BufferedReader(io.BytesIO(b"12")) + rawio = self.tp(self.BytesIO(b"12")) rawio.readinto = lambda buf: -1 bufio = self.tp(rawio) with self.assertRaises(OSError) as cm: bufio.readline() self.assertIsNone(cm.exception.__cause__) - # TODO: RUSTPYTHON, TypeError: 'bytes' object cannot be interpreted as an integer") - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: 'bytes' object cannot be interpreted as an integer") def test_bad_readinto_type(self): - rawio = io.BufferedReader(io.BytesIO(b"12")) + rawio = self.tp(self.BytesIO(b"12")) rawio.readinto = lambda buf: b'' bufio = self.tp(rawio) with self.assertRaises(OSError) as cm: bufio.readline() self.assertIsInstance(cm.exception.__cause__, TypeError) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_flush_error_on_close(self): - super().test_flush_error_on_close() - - # TODO: RUSTPYTHON, AssertionError: UnsupportedOperation not raised by truncate - @unittest.expectedFailure - def test_truncate_on_read_only(self): # TODO: RUSTPYTHON, remove when this passes - super().test_truncate_on_read_only() # TODO: RUSTPYTHON, remove when this passes + return super().test_flush_error_on_close() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_seek_character_device_file(self): - super().test_seek_character_device_file() + return super().test_seek_character_device_file() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: UnsupportedOperation not raised by truncate + def test_truncate_on_read_only(self): + return super().test_truncate_on_read_only() + + @unittest.skip('TODO: RUSTPYTHON; fallible allocation') + def test_constructor(self): + return super().test_constructor() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_pickling_subclass(self): + return super().test_pickling_subclass() class PyBufferedReaderTest(BufferedReaderTest): @@ -1909,8 +1959,7 @@ def _seekrel(bufio): def test_writes_and_truncates(self): self.check_writes(lambda bufio: bufio.truncate(bufio.tell())) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_non_blocking(self): raw = self.MockNonBlockWriterIO() bufio = self.tp(raw, 8) @@ -2107,20 +2156,6 @@ def test_slow_close_from_thread(self): class CBufferedWriterTest(BufferedWriterTest, SizeofTest): tp = io.BufferedWriter - @unittest.skip("TODO: RUSTPYTHON, fallible allocation") - @skip_if_sanitizer(memory=True, address=True, thread=True, - reason="sanitizer defaults to crashing " - "instead of returning NULL for malloc failure.") - def test_constructor(self): - BufferedWriterTest.test_constructor(self) - # The allocation can succeed on 32-bit builds, e.g. with more - # than 2 GiB RAM and a 64-bit kernel. - if sys.maxsize > 0x7FFFFFFF: - rawio = self.MockRawIO() - bufio = self.tp(rawio) - self.assertRaises((OverflowError, MemoryError, ValueError), - bufio.__init__, rawio, sys.maxsize) - def test_initialization(self): rawio = self.MockRawIO() bufio = self.tp(rawio) @@ -2131,8 +2166,7 @@ def test_initialization(self): self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-1) self.assertRaises(ValueError, bufio.write, b"def") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_garbage_collection(self): # C BufferedWriter objects are collected, and collecting them flushes # all data to disk. @@ -2155,11 +2189,17 @@ def test_args_error(self): with self.assertRaisesRegex(TypeError, "BufferedWriter"): self.tp(self.BytesIO(), 1024, 1024, 1024) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_flush_error_on_close(self): - super().test_flush_error_on_close() + return super().test_flush_error_on_close() + + @unittest.skip('TODO: RUSTPYTHON; fallible allocation') + def test_constructor(self): + return super().test_constructor() + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_pickling_subclass(self): + return super().test_pickling_subclass() class PyBufferedWriterTest(BufferedWriterTest): tp = pyio.BufferedWriter @@ -2637,22 +2677,7 @@ def test_interleaved_readline_write(self): class CBufferedRandomTest(BufferedRandomTest, SizeofTest): tp = io.BufferedRandom - @unittest.skip("TODO: RUSTPYTHON, fallible allocation") - @skip_if_sanitizer(memory=True, address=True, thread=True, - reason="sanitizer defaults to crashing " - "instead of returning NULL for malloc failure.") - def test_constructor(self): - BufferedRandomTest.test_constructor(self) - # The allocation can succeed on 32-bit builds, e.g. with more - # than 2 GiB RAM and a 64-bit kernel. - if sys.maxsize > 0x7FFFFFFF: - rawio = self.MockRawIO() - bufio = self.tp(rawio) - self.assertRaises((OverflowError, MemoryError, ValueError), - bufio.__init__, rawio, sys.maxsize) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_garbage_collection(self): CBufferedReaderTest.test_garbage_collection(self) CBufferedWriterTest.test_garbage_collection(self) @@ -2662,20 +2687,25 @@ def test_args_error(self): with self.assertRaisesRegex(TypeError, "BufferedRandom"): self.tp(self.BytesIO(), 1024, 1024, 1024) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_flush_error_on_close(self): - super().test_flush_error_on_close() + return super().test_flush_error_on_close() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_seek_character_device_file(self): - super().test_seek_character_device_file() + return super().test_seek_character_device_file() - # TODO: RUSTPYTHON; f.read1(1) returns b'a' - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; f.read1(1) returns b'a' def test_read1_after_write(self): - super().test_read1_after_write() + return super().test_read1_after_write() + + @unittest.skip('TODO: RUSTPYTHON; fallible allocation') + def test_constructor(self): + return super().test_constructor() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_pickling_subclass(self): + return super().test_pickling_subclass() class PyBufferedRandomTest(BufferedRandomTest): @@ -2935,11 +2965,16 @@ def test_recursive_repr(self): # Issue #25455 raw = self.BytesIO() t = self.TextIOWrapper(raw, encoding="utf-8") - with support.swap_attr(raw, 'name', t): - try: + with support.swap_attr(raw, 'name', t), support.infinite_recursion(25): + with self.assertRaises(RuntimeError): repr(t) # Should not crash - except RuntimeError: - pass + + def test_subclass_repr(self): + class TestSubclass(self.TextIOWrapper): + pass + + f = TestSubclass(self.StringIO()) + self.assertIn(TestSubclass.__name__, repr(f)) def test_line_buffering(self): r = self.BytesIO() @@ -3166,6 +3201,7 @@ def flush(self): support.gc_collect() self.assertEqual(record, [1, 2, 3]) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_error_through_destructor(self): # Test that the exception state is not modified by a destructor, # even if close() fails. @@ -3174,10 +3210,7 @@ def test_error_through_destructor(self): with self.assertRaises(AttributeError): self.TextIOWrapper(rawio, encoding="utf-8").xyzzy - if not IOBASE_EMITS_UNRAISABLE: - self.assertIsNone(cm.unraisable) - elif cm.unraisable is not None: - self.assertEqual(cm.unraisable.exc_type, OSError) + self.assertEqual(cm.unraisable.exc_type, OSError) # Systematic tests of the text I/O API @@ -3327,8 +3360,7 @@ def test_seek_and_tell_with_data(data, min_pos=0): finally: StatefulIncrementalDecoder.codecEnabled = 0 - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_multibyte_seek_and_tell(self): f = self.open(os_helper.TESTFN, "w", encoding="euc_jp") f.write("AB\n\u3046\u3048\n") @@ -3344,8 +3376,7 @@ def test_multibyte_seek_and_tell(self): self.assertEqual(f.tell(), p1) f.close() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_seek_with_encoder_state(self): f = self.open(os_helper.TESTFN, "w", encoding="euc_jis_2004") f.write("\u00e6\u0300") @@ -3359,8 +3390,7 @@ def test_seek_with_encoder_state(self): self.assertEqual(f.readline(), "\u00e6\u0300\u0300") f.close() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_encoded_writes(self): data = "1234567890" tests = ("utf-16", @@ -3499,8 +3529,7 @@ def test_issue2282(self): self.assertEqual(buffer.seekable(), txt.seekable()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_append_bom(self): # The BOM is not written again when appending to a non-empty file filename = os_helper.TESTFN @@ -3516,8 +3545,7 @@ def test_append_bom(self): with self.open(filename, 'rb') as f: self.assertEqual(f.read(), 'aaaxxx'.encode(charset)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_seek_bom(self): # Same test, but when seeking manually filename = os_helper.TESTFN @@ -3533,8 +3561,7 @@ def test_seek_bom(self): with self.open(filename, 'rb') as f: self.assertEqual(f.read(), 'bbbzzz'.encode(charset)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_seek_append_bom(self): # Same test, but first seek to the start and then to the end filename = os_helper.TESTFN @@ -3795,17 +3822,14 @@ def _check_create_at_shutdown(self, **kwargs): codecs.lookup('utf-8') class C: - def __init__(self): - self.buf = io.BytesIO() def __del__(self): - io.TextIOWrapper(self.buf, **{kwargs}) + io.TextIOWrapper(io.BytesIO(), **{kwargs}) print("ok") c = C() """.format(iomod=iomod, kwargs=kwargs) return assert_python_ok("-c", code) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_create_at_shutdown_without_encoding(self): rc, out, err = self._check_create_at_shutdown() if err: @@ -3815,8 +3839,7 @@ def test_create_at_shutdown_without_encoding(self): else: self.assertEqual("ok", out.decode().strip()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_create_at_shutdown_with_encoding(self): rc, out, err = self._check_create_at_shutdown(encoding='utf-8', errors='strict') @@ -4042,6 +4065,28 @@ def test_issue35928(self): f.write(res) self.assertEqual(res + f.readline(), 'foo\nbar\n') + def test_pickling_subclass(self): + global MyTextIO + class MyTextIO(self.TextIOWrapper): + def __init__(self, raw, tag): + super().__init__(raw) + self.tag = tag + def __getstate__(self): + return self.tag, self.buffer.getvalue() + def __setstate__(slf, state): + tag, value = state + slf.__init__(self.BytesIO(value), tag) + + raw = self.BytesIO(b'data') + txt = MyTextIO(raw, 'ham') + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + pickled = pickle.dumps(txt, proto) + newtxt = pickle.loads(pickled) + self.assertEqual(newtxt.buffer.getvalue(), b'data') + self.assertEqual(newtxt.tag, 'ham') + del MyTextIO + class MemviewBytesIO(io.BytesIO): '''A BytesIO object whose read method returns memoryviews @@ -4066,98 +4111,7 @@ class CTextIOWrapperTest(TextIOWrapperTest): io = io shutdown_error = "LookupError: unknown encoding: ascii" - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_constructor(self): - super().test_constructor() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_detach(self): - super().test_detach() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_reconfigure_encoding_read(self): - super().test_reconfigure_encoding_read() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_reconfigure_line_buffering(self): - super().test_reconfigure_line_buffering() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_basic_io(self): - super().test_basic_io() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_telling(self): - super().test_telling() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_uninitialized(self): - super().test_uninitialized() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_non_text_encoding_codecs_are_rejected(self): - super().test_non_text_encoding_codecs_are_rejected() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_repr(self): - super().test_repr() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newlines(self): - super().test_newlines() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newlines_input(self): - super().test_newlines_input() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_reconfigure_write_through(self): - super().test_reconfigure_write_through() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_reconfigure_write_fromascii(self): - super().test_reconfigure_write_fromascii() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_reconfigure_write(self): - super().test_reconfigure_write() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_reconfigure_defaults(self): - super().test_reconfigure_defaults() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_reconfigure_newline(self): - super().test_reconfigure_newline() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_reconfigure_errors(self): - super().test_reconfigure_errors() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_reconfigure_locale(self): - super().test_reconfigure_locale() - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_initialization(self): r = self.BytesIO(b"\xc3\xa9\n\n") b = self.BufferedReader(r, 1000) @@ -4168,8 +4122,7 @@ def test_initialization(self): t = self.TextIOWrapper.__new__(self.TextIOWrapper) self.assertRaises(Exception, repr, t) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_garbage_collection(self): # C TextIOWrapper objects are collected, and collecting them flushes # all data to disk. @@ -4233,20 +4186,121 @@ def write(self, data): t.write("x"*chunk_size) self.assertEqual([b"abcdef", b"ghi", b"x"*chunk_size], buf._write_stack) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_issue119506(self): + chunk_size = 8192 + + class MockIO(self.MockRawIO): + written = False + def write(self, data): + if not self.written: + self.written = True + t.write("middle") + return super().write(data) + + buf = MockIO() + t = self.TextIOWrapper(buf) + t.write("abc") + t.write("def") + # writing data which size >= chunk_size cause flushing buffer before write. + t.write("g" * chunk_size) + t.flush() + + self.assertEqual([b"abcdef", b"middle", b"g"*chunk_size], + buf._write_stack) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_basic_io(self): + return super().test_basic_io() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_constructor(self): + return super().test_constructor() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_detach(self): + return super().test_detach() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_newlines(self): + return super().test_newlines() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_newlines_input(self): + return super().test_newlines_input() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_non_text_encoding_codecs_are_rejected(self): + return super().test_non_text_encoding_codecs_are_rejected() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_reconfigure_defaults(self): + return super().test_reconfigure_defaults() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_reconfigure_encoding_read(self): + return super().test_reconfigure_encoding_read() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_reconfigure_errors(self): + return super().test_reconfigure_errors() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_reconfigure_line_buffering(self): + return super().test_reconfigure_line_buffering() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_reconfigure_locale(self): + return super().test_reconfigure_locale() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_reconfigure_newline(self): + return super().test_reconfigure_newline() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_reconfigure_write(self): + return super().test_reconfigure_write() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_reconfigure_write_fromascii(self): + return super().test_reconfigure_write_fromascii() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_reconfigure_write_through(self): + return super().test_reconfigure_write_through() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_repr(self): + return super().test_repr() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_telling(self): + return super().test_telling() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_uninitialized(self): + return super().test_uninitialized() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_recursive_repr(self): + return super().test_recursive_repr() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_pickling_subclass(self): + return super().test_pickling_subclass() + class PyTextIOWrapperTest(TextIOWrapperTest): io = pyio shutdown_error = "LookupError: unknown encoding: ascii" - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised def test_constructor(self): - super().test_constructor() + return super().test_constructor() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_newlines(self): - super().test_newlines() + return super().test_newlines() class IncrementalNewlineDecoderTest(unittest.TestCase): @@ -4326,8 +4380,7 @@ def _decode_bytewise(s): self.assertEqual(decoder.decode(input), "abc") self.assertEqual(decoder.newlines, None) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_newline_decoder(self): encodings = ( # None meaning the IncrementalNewlineDecoder takes unicode input @@ -4489,8 +4542,7 @@ def test_io_after_close(self): self.assertRaises(ValueError, f.writelines, []) self.assertRaises(ValueError, next, f) - # TODO: RUSTPYTHON, cyclic gc - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; cyclic gc def test_blockingioerror(self): # Various BlockingIOError issues class C(str): @@ -4538,15 +4590,14 @@ def test_abc_inheritance_official(self): self._check_abc_inheritance(io) def _check_warn_on_dealloc(self, *args, **kwargs): - f = open(*args, **kwargs) + f = self.open(*args, **kwargs) r = repr(f) with self.assertWarns(ResourceWarning) as cm: f = None support.gc_collect() self.assertIn(r, str(cm.warning.args[0])) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_warn_on_dealloc(self): self._check_warn_on_dealloc(os_helper.TESTFN, "wb", buffering=0) self._check_warn_on_dealloc(os_helper.TESTFN, "wb") @@ -4569,10 +4620,9 @@ def cleanup_fds(): r, w = os.pipe() fds += r, w with warnings_helper.check_no_resource_warning(self): - open(r, *args, closefd=False, **kwargs) + self.open(r, *args, closefd=False, **kwargs) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_warn_on_dealloc_fd(self): self._check_warn_on_dealloc_fd("rb", buffering=0) @@ -4602,16 +4652,14 @@ def test_pickling(self): with self.assertRaisesRegex(TypeError, msg): pickle.dumps(f, protocol) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipIf( support.is_emscripten, "fstat() of a pipe fd is not supported" ) def test_nonblock_pipe_write_bigbuf(self): self._test_nonblock_pipe_write(16*1024) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipIf( support.is_emscripten, "fstat() of a pipe fd is not supported" ) @@ -4733,8 +4781,7 @@ def test_check_encoding_errors(self): proc = assert_python_failure('-X', 'dev', '-c', code) self.assertEqual(proc.rc, 10, proc) - # TODO: RUSTPYTHON, AssertionError: 0 != 2 - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 0 != 2 def test_check_encoding_warning(self): # PEP 597: Raise warning when encoding is not specified # and sys.flags.warn_default_encoding is set. @@ -4758,8 +4805,7 @@ def test_check_encoding_warning(self): self.assertTrue( warnings[1].startswith(b"<string>:8: EncodingWarning: ")) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_text_encoding(self): # PEP 597, bpo-47000. io.text_encoding() returns "locale" or "utf-8" # based on sys.flags.utf8_mode @@ -4837,10 +4883,9 @@ def test_daemon_threads_shutdown_stdout_deadlock(self): def test_daemon_threads_shutdown_stderr_deadlock(self): self.check_daemon_threads_shutdown_deadlock('stderr') - # TODO: RUSTPYTHON, AssertionError: 22 != 10 : _PythonRunResult(rc=22, out=b'', err=b'') - @unittest.expectedFailure - def test_check_encoding_errors(self): # TODO: RUSTPYTHON, remove when this passes - super().test_check_encoding_errors() # TODO: RUSTPYTHON, remove when this passes + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 22 != 10 : _PythonRunResult(rc=22, out=b'', err=b'') + def test_check_encoding_errors(self): + return super().test_check_encoding_errors() class PyMiscIOTest(MiscIOTest): @@ -5014,16 +5059,14 @@ def alarm_handler(sig, frame): os.close(w) os.close(r) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @requires_alarm @support.requires_resource('walltime') def test_interrupted_read_retry_buffered(self): self.check_interrupted_read_retry(lambda x: x.decode('latin1'), mode="rb") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @requires_alarm @support.requires_resource('walltime') def test_interrupted_read_retry_text(self): @@ -5098,15 +5141,13 @@ def alarm2(sig, frame): if e.errno != errno.EBADF: raise - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @requires_alarm @support.requires_resource('walltime') def test_interrupted_write_retry_buffered(self): self.check_interrupted_write_retry(b"x", mode="wb") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @requires_alarm @support.requires_resource('walltime') def test_interrupted_write_retry_text(self): diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 61fe109728..03dadb71f7 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -30,10 +30,11 @@ def inner(self): with warnings.catch_warnings(): warnings.simplefilter("ignore", category=DeprecationWarning) testfunc(self) - with warnings.catch_warnings(): - warnings.simplefilter("error", category=DeprecationWarning) - with self.assertRaises((DeprecationWarning, AssertionError, SystemError)): - testfunc(self) + # XXX: RUSTPYTHON; Patch to make tests pass. It will be removed once 3.14 is released anyway. + # with warnings.catch_warnings(): + # warnings.simplefilter("error", category=DeprecationWarning) + # with self.assertRaises((DeprecationWarning, AssertionError, SystemError)): + # testfunc(self) return inner @@ -145,7 +146,7 @@ def expand(it, i=0): c = expand(compare[took:]) self.assertEqual(a, c); - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; [7, 7, 8, 10] != <itertools.accumulate object at 0xb4000073f32b4480> @pickle_deprecated def test_accumulate(self): self.assertEqual(list(accumulate(range(10))), # one positional arg @@ -247,7 +248,6 @@ def test_chain_from_iterable(self): self.assertRaises(TypeError, list, chain.from_iterable([2, 3])) self.assertEqual(list(islice(chain.from_iterable(repeat(range(5))), 2)), [0, 1]) - @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_chain_reducible(self): for oper in [copy.deepcopy] + picklecopiers: @@ -567,7 +567,6 @@ def test_combinatorics(self): self.assertEqual(comb, list(filter(set(perm).__contains__, cwr))) # comb: cwr that is a perm self.assertEqual(comb, sorted(set(cwr) & set(perm))) # comb: both a cwr and a perm - @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_compress(self): self.assertEqual(list(compress(data='ABCDEF', selectors=[1,0,1,0,1,1])), list('ACEF')) @@ -602,7 +601,6 @@ def test_compress(self): next(testIntermediate) self.assertEqual(list(op(testIntermediate)), list(result2)) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: DeprecationWarning not triggered @pickle_deprecated def test_count(self): self.assertEqual(lzip('abc',count()), [('a', 0), ('b', 1), ('c', 2)]) @@ -663,7 +661,7 @@ def test_count(self): #check proper internal error handling for large "step' sizes count(1, maxsize+5); sys.exc_info() - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; 'count(10.5)' != 'count(10.5, 1.0)' @pickle_deprecated def test_count_with_step(self): self.assertEqual(lzip('abc',count(2,3)), [('a', 2), ('b', 5), ('c', 8)]) @@ -1037,7 +1035,6 @@ def test_filter(self): c = filter(isEven, range(6)) self.pickletest(proto, c) - @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_filterfalse(self): self.assertEqual(list(filterfalse(isEven, range(6))), [1,3,5]) @@ -1145,7 +1142,6 @@ def test_zip_longest_tuple_reuse(self): ids = list(map(id, list(zip_longest('abc', 'def')))) self.assertEqual(len(dict.fromkeys(ids)), len(ids)) - @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_zip_longest_pickling(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -1369,7 +1365,6 @@ def test_product_tuple_reuse(self): self.assertEqual(len(set(map(id, product('abc', 'def')))), 1) self.assertNotEqual(len(set(map(id, list(product('abc', 'def'))))), 1) - @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_product_pickling(self): # check copy, deepcopy, pickle @@ -1398,7 +1393,6 @@ def test_product_issue_25021(self): p.__setstate__((0, 0, 0x1000)) # will access tuple element 1 if not clamped self.assertRaises(StopIteration, next, p) - @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_repeat(self): self.assertEqual(list(repeat(object='a', times=3)), ['a', 'a', 'a']) @@ -1464,7 +1458,6 @@ def test_map(self): c = map(tupleize, 'abc', count()) self.pickletest(proto, c) - @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_starmap(self): self.assertEqual(list(starmap(operator.pow, zip(range(3), range(1,7)))), @@ -1589,7 +1582,6 @@ def __index__(self): self.assertEqual(list(islice(range(100), IntLike(10), IntLike(50), IntLike(5))), list(range(10,50,5))) - @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_takewhile(self): data = [1, 3, 5, 20, 2, 4, 6, 8] @@ -1950,7 +1942,6 @@ class TestExamples(unittest.TestCase): def test_accumulate(self): self.assertEqual(list(accumulate([1,2,3,4,5])), [1, 3, 6, 10, 15]) - @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_accumulate_reducible(self): # check copy, deepcopy, pickle diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py index 42d8dcbbe1..ed061384f1 100644 --- a/Lib/test/test_list.py +++ b/Lib/test/test_list.py @@ -48,8 +48,7 @@ def test_keyword_args(self): with self.assertRaisesRegex(TypeError, 'keyword argument'): list(sequence=[]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_keywords_in_subclass(self): class subclass(list): pass @@ -105,8 +104,7 @@ def test_empty_slice(self): x[:] = x self.assertEqual(x, []) - # TODO: RUSTPYTHON - @unittest.skip("TODO: RUSTPYTHON crash") + @unittest.skip("TODO: RUSTPYTHON; crash") def test_list_resize_overflow(self): # gh-97616: test new_allocated * sizeof(PyObject*) overflow # check in list_resize() @@ -120,8 +118,7 @@ def test_list_resize_overflow(self): with self.assertRaises((MemoryError, OverflowError)): lst *= size - # TODO: RUSTPYTHON - @unittest.skip("TODO: RUSTPYTHON hangs") + @unittest.skip("TODO: RUSTPYTHON; hangs") def test_repr_mutate(self): class Obj: @staticmethod @@ -230,7 +227,6 @@ class L(list): pass with self.assertRaises(TypeError): (3,) + L([1,2]) - # TODO: RUSTPYTHON @unittest.skip("TODO: RUSTPYTHON; hang") def test_equal_operator_modifying_operand(self): # test fix for seg fault reported in bpo-38588 part 2. @@ -257,7 +253,6 @@ def __eq__(self, other): list4 = [1] self.assertFalse(list3 == list4) - # TODO: RUSTPYTHON @unittest.skip("TODO: RUSTPYTHON; hang") def test_lt_operator_modifying_operand(self): # See gh-120298 diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 0039d34b5e..84a659ebe4 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -1115,8 +1115,7 @@ class SMTPHandlerTest(BaseTest): # bpo-14314, bpo-19665, bpo-34092: don't wait forever TIMEOUT = support.LONG_TIMEOUT - # TODO: RUSTPYTHON - @unittest.skip(reason="RUSTPYTHON hangs") + @unittest.skip("TODO: RUSTPYTHON; hangs") def test_basic(self): sockmap = {} server = TestSMTPServer((socket_helper.HOST, 0), self.process_message, 0.001, @@ -2154,8 +2153,7 @@ def handle_request(self, request): request.end_headers() self.handled.set() - # TODO: RUSTPYTHON - @unittest.skip("TODO: RUSTPYTHON; flaky test") + @unittest.skip('TODO: RUSTPYTHON; flaky test') def test_output(self): # The log message sent to the HTTPHandler is properly received. logger = logging.getLogger("http") @@ -4060,8 +4058,7 @@ def _mpinit_issue121723(qspec, message_to_log): # log a message (this creates a record put in the queue) logging.getLogger().info(message_to_log) - # TODO: RUSTPYTHON; ImportError: cannot import name 'SemLock' - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ImportError: cannot import name 'SemLock' @skip_if_tsan_fork @support.requires_subprocess() def test_multiprocessing_queues(self): @@ -4121,8 +4118,7 @@ def test_90195(self): # Logger should be enabled, since explicitly mentioned self.assertFalse(logger.disabled) - # TODO: RUSTPYTHON; ImportError: cannot import name 'SemLock' - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ImportError: cannot import name 'SemLock' def test_111615(self): # See gh-111615 import_helper.import_module('_multiprocessing') # see gh-113692 @@ -4171,91 +4167,6 @@ def __init__(self, *args, **kwargs): handler = logging.getHandlerByName('custom') self.assertEqual(handler.custom_kwargs, custom_kwargs) - # TODO: RUSTPYTHON; ImportError: cannot import name 'SemLock' - @unittest.expectedFailure - # See gh-91555 and gh-90321 - @support.requires_subprocess() - def test_deadlock_in_queue(self): - queue = multiprocessing.Queue() - handler = logging.handlers.QueueHandler(queue) - logger = multiprocessing.get_logger() - level = logger.level - try: - logger.setLevel(logging.DEBUG) - logger.addHandler(handler) - logger.debug("deadlock") - finally: - logger.setLevel(level) - logger.removeHandler(handler) - - def test_recursion_in_custom_handler(self): - class BadHandler(logging.Handler): - def __init__(self): - super().__init__() - def emit(self, record): - logger.debug("recurse") - logger = logging.getLogger("test_recursion_in_custom_handler") - logger.addHandler(BadHandler()) - logger.setLevel(logging.DEBUG) - logger.debug("boom") - - @threading_helper.requires_working_threading() - def test_thread_supression_noninterference(self): - lock = threading.Lock() - logger = logging.getLogger("test_thread_supression_noninterference") - - # Block on the first call, allow others through - # - # NOTE: We need to bypass the base class's lock, otherwise that will - # block multiple calls to the same handler itself. - class BlockOnceHandler(TestHandler): - def __init__(self, barrier): - super().__init__(support.Matcher()) - self.barrier = barrier - - def createLock(self): - self.lock = None - - def handle(self, record): - self.emit(record) - - def emit(self, record): - if self.barrier: - barrier = self.barrier - self.barrier = None - barrier.wait() - with lock: - pass - super().emit(record) - logger.info("blow up if not supressed") - - barrier = threading.Barrier(2) - handler = BlockOnceHandler(barrier) - logger.addHandler(handler) - logger.setLevel(logging.DEBUG) - - t1 = threading.Thread(target=logger.debug, args=("1",)) - with lock: - - # Ensure first thread is blocked in the handler, hence supressing logging... - t1.start() - barrier.wait() - - # ...but the second thread should still be able to log... - t2 = threading.Thread(target=logger.debug, args=("2",)) - t2.start() - t2.join(timeout=3) - - self.assertEqual(len(handler.buffer), 1) - self.assertTrue(handler.matches(levelno=logging.DEBUG, message='2')) - - # The first thread should still be blocked here - self.assertTrue(t1.is_alive()) - - # Now the lock has been released the first thread should complete - t1.join() - self.assertEqual(len(handler.buffer), 2) - self.assertTrue(handler.matches(levelno=logging.DEBUG, message='1')) class ManagerTest(BaseTest): def test_manager_loggerclass(self): @@ -4663,8 +4574,7 @@ def test_dollars(self): f = logging.Formatter('${asctime}--', style='$') self.assertTrue(f.usesTime()) - # TODO: RUSTPYTHON; ValueError: Unexpected error parsing format string - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: Unexpected error parsing format string def test_format_validate(self): # Check correct formatting # Percentage style @@ -4838,8 +4748,7 @@ def test_defaults_parameter(self): def test_invalid_style(self): self.assertRaises(ValueError, logging.Formatter, None, None, 'x') - # TODO: RUSTPYTHON; AttributeError: 'struct_time' object has no attribute 'tm_gmtoff' - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'struct_time' object has no attribute 'tm_gmtoff' def test_time(self): r = self.get_record() dt = datetime.datetime(1993, 4, 21, 8, 3, 0, 0, utc) @@ -4854,8 +4763,7 @@ def test_time(self): f.format(r) self.assertEqual(r.asctime, '1993-04-21 08:03:00,123') - # TODO: RUSTPYTHON; AttributeError: 'struct_time' object has no attribute 'tm_gmtoff' - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'struct_time' object has no attribute 'tm_gmtoff' def test_default_msec_format_none(self): class NoMsecFormatter(logging.Formatter): default_msec_format = None @@ -5257,8 +5165,7 @@ def __init__(self, name='MyLogger', level=logging.NOTSET): h.close() logging.setLoggerClass(logging.Logger) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_logging_at_shutdown(self): # bpo-20037: Doing text I/O late at interpreter shutdown must not crash code = textwrap.dedent(""" @@ -5278,8 +5185,7 @@ def __del__(self): self.assertIn("exception in __del__", err) self.assertIn("ValueError: some error", err) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_logging_at_shutdown_open(self): # bpo-26789: FileHandler keeps a reference to the builtin open() # function to be able to open or reopen the file during Python @@ -6480,8 +6386,7 @@ def rotator(source, dest): rh.close() class TimedRotatingFileHandlerTest(BaseFileTest): - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON') @unittest.skipIf(support.is_wasi, "WASI does not have /dev/null.") def test_should_not_rollover(self): # See bpo-45401. Should only ever rollover regular files @@ -6535,8 +6440,7 @@ def test_rollover(self): print(tf.read()) self.assertTrue(found, msg=msg) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON') def test_rollover_at_midnight(self, weekly=False): os_helper.unlink(self.fn) now = datetime.datetime.now() @@ -6580,8 +6484,7 @@ def test_rollover_at_midnight(self, weekly=False): for i, line in enumerate(f): self.assertIn(f'testing1 {i}', line) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON') def test_rollover_at_weekday(self): self.test_rollover_at_midnight(weekly=True) diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index f2cafd97b3..96c20fe893 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -1527,6 +1527,14 @@ def test_expandvars(self): def test_expandvars_nonascii(self): return super().test_expandvars_nonascii() + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON') + def test_samefile_on_symlink(self): + return super().test_samefile_on_symlink() + + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON') + def test_samestat_on_symlink(self): + return super().test_samestat_on_symlink() + class PathLikeTests(NtpathTestCase): diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index a025a2d4ff..c86d910eef 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -13,7 +13,6 @@ import locale import os import pickle -import platform import select import selectors import shutil @@ -188,8 +187,7 @@ def test_access(self): os.close(f) self.assertTrue(os.access(os_helper.TESTFN, os.W_OK)) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON, BrokenPipeError: (32, 'The process cannot access the file because it is being used by another process. (os error 32)')") + @unittest.skipIf(sys.platform == 'win32', "TODO: RUSTPYTHON; BrokenPipeError: (32, 'The process cannot access the file because it is being used by another process. (os error 32)')") @unittest.skipIf( support.is_emscripten, "Test is unstable under Emscripten." ) @@ -716,8 +714,7 @@ def check_file_attributes(self, result): self.assertTrue(isinstance(result.st_file_attributes, int)) self.assertTrue(0 <= result.st_file_attributes <= 0xFFFFFFFF) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.stat return value doesnt have st_file_attributes attribute") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON; os.stat return value doesnt have st_file_attributes attribute') @unittest.skipUnless(sys.platform == "win32", "st_file_attributes is Win32 specific") def test_file_attributes(self): @@ -739,8 +736,7 @@ def test_file_attributes(self): result.st_file_attributes & stat.FILE_ATTRIBUTE_DIRECTORY, stat.FILE_ATTRIBUTE_DIRECTORY) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.stat (PermissionError: [Errno 5] Access is denied.)") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON; os.stat (PermissionError: [Errno 5] Access is denied.)') @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") def test_access_denied(self): # Default to FindFirstFile WIN32_FIND_DATA when access is @@ -763,8 +759,7 @@ def test_access_denied(self): self.assertNotEqual(result.st_size, 0) self.assertTrue(os.path.isfile(fname)) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.stat (PermissionError: [Errno 1] Incorrect function.)") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON; os.stat (PermissionError: [Errno 1] Incorrect function.)') @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") def test_stat_block_device(self): # bpo-38030: os.stat fails for block devices @@ -822,8 +817,7 @@ def _test_utime(self, set_time, filename=None): self.assertEqual(st.st_atime_ns, atime_ns) self.assertEqual(st.st_mtime_ns, mtime_ns) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: 2.002003 != 1.002003 within 1e-06 delta (1.0000000000000002 difference))") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON; (AssertionError: 2.002003 != 1.002003 within 1e-06 delta (1.0000000000000002 difference))') def test_utime(self): def set_time(filename, ns): # test the ns keyword parameter @@ -889,8 +883,7 @@ def set_time(filename, ns): os.utime(name, dir_fd=dirfd, ns=ns) self._test_utime(set_time) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: 2.002003 != 1.002003 within 1e-06 delta (1.0000000000000002 difference))") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON; (AssertionError: 2.002003 != 1.002003 within 1e-06 delta (1.0000000000000002 difference))') def test_utime_directory(self): def set_time(filename, ns): # test calling os.utime() on a directory @@ -919,24 +912,21 @@ def _test_utime_current(self, set_time): self.assertAlmostEqual(st.st_mtime, current, delta=delta, msg=msg) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: 3359485824.516508 != 1679742912.516503 within 0.05 delta (1679742912.000005 difference) : st_time=3359485824.516508, current=1679742912.516503, dt=1679742912.000005)") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON; (AssertionError: 3359485824.516508 != 1679742912.516503 within 0.05 delta (1679742912.000005 difference) : st_time=3359485824.516508, current=1679742912.516503, dt=1679742912.000005)') def test_utime_current(self): def set_time(filename): # Set to the current time in the new way os.utime(self.fname) self._test_utime_current(set_time) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: 3359485824.5186944 != 1679742912.5186892 within 0.05 delta (1679742912.0000052 difference) : st_time=3359485824.5186944, current=1679742912.5186892, dt=1679742912.0000052)") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON; (AssertionError: 3359485824.5186944 != 1679742912.5186892 within 0.05 delta (1679742912.0000052 difference) : st_time=3359485824.5186944, current=1679742912.5186892, dt=1679742912.0000052)') def test_utime_current_old(self): def set_time(filename): # Set to the current time in the old explicit way. os.utime(self.fname, None) self._test_utime_current(set_time) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_utime_nonexistent(self): now = time.time() filename = 'nonexistent' @@ -957,8 +947,7 @@ def get_file_system(self, path): return buf.value # return None if the filesystem is unknown - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (ModuleNotFoundError: No module named '_ctypes')") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; (ModuleNotFoundError: No module named '_ctypes')") def test_large_time(self): # Many filesystems are limited to the year 2038. At least, the test # pass with NTFS filesystem. @@ -969,7 +958,7 @@ def test_large_time(self): os.utime(self.fname, (large, large)) self.assertEqual(os.stat(self.fname).st_mtime, large) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: NotImplementedError not raised)") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON; (AssertionError: NotImplementedError not raised)') def test_utime_invalid_arguments(self): # seconds and nanoseconds parameters are mutually exclusive with self.assertRaises(ValueError): @@ -1171,9 +1160,8 @@ def test_putenv_unsetenv(self): stdout=subprocess.PIPE, text=True) self.assertEqual(proc.stdout.rstrip(), repr(None)) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: ValueError not raised by putenv)") # On OS X < 10.6, unsetenv() doesn't return a value (bpo-13415). + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON; (AssertionError: ValueError not raised by putenv)') @support.requires_mac_ver(10, 6) def test_putenv_unsetenv_error(self): # Empty variable name is invalid. @@ -1748,18 +1736,15 @@ def walk(self, top, **kwargs): bdirs[:] = list(map(os.fsencode, dirs)) bfiles[:] = list(map(os.fsencode, files)) - # TODO: RUSTPYTHON (TypeError: Can't mix strings and bytes in path components) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; (TypeError: Can't mix strings and bytes in path components) def test_compare_to_walk(self): return super().test_compare_to_walk() - # TODO: RUSTPYTHON (TypeError: Can't mix strings and bytes in path components) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; (TypeError: Can't mix strings and bytes in path components) def test_dir_fd(self): return super().test_dir_fd() - # TODO: RUSTPYTHON (TypeError: Can't mix strings and bytes in path components) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; (TypeError: Can't mix strings and bytes in path components) def test_yields_correct_dir_fd(self): return super().test_yields_correct_dir_fd() @@ -1811,8 +1796,7 @@ def test_mode(self): self.assertEqual(os.stat(path).st_mode & 0o777, 0o555) self.assertEqual(os.stat(parent).st_mode & 0o777, 0o775) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.umask not implemented yet for all platforms") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON; os.umask not implemented yet for all platforms') @unittest.skipIf( support.is_emscripten or support.is_wasi, "Emscripten's/WASI's umask is a stub." @@ -1831,8 +1815,7 @@ def test_exist_ok_existing_directory(self): # Issue #25583: A drive root could raise PermissionError on Windows os.makedirs(os.path.abspath('/'), exist_ok=True) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.umask not implemented yet for all platforms") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON; os.umask not implemented yet for all platforms') @unittest.skipIf( support.is_emscripten or support.is_wasi, "Emscripten's/WASI's umask is a stub." @@ -2119,8 +2102,7 @@ def test_urandom_failure(self): """ assert_python_ok('-c', code) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON on Windows (ModuleNotFoundError: No module named 'os')") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; on Windows (ModuleNotFoundError: No module named 'os')") def test_urandom_fd_closed(self): # Issue #21207: urandom() should reopen its fd to /dev/urandom if # closed. @@ -2135,8 +2117,7 @@ def test_urandom_fd_closed(self): """ rc, out, err = assert_python_ok('-Sc', code) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (ModuleNotFoundError: No module named 'os'") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; (ModuleNotFoundError: No module named 'os'") def test_urandom_fd_reopened(self): # Issue #21207: urandom() should detect its fd to /dev/urandom # changed to something else, and reopen it. @@ -2224,8 +2205,7 @@ def test_execv_with_bad_arglist(self): self.assertRaises(ValueError, os.execv, 'notepad', ('',)) self.assertRaises(ValueError, os.execv, 'notepad', ['']) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.execve not implemented yet for all platforms") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON; os.execve not implemented yet for all platforms') def test_execvpe_with_bad_arglist(self): self.assertRaises(ValueError, os.execvpe, 'notepad', [], None) self.assertRaises(ValueError, os.execvpe, 'notepad', [], {}) @@ -2285,8 +2265,7 @@ def test_internal_execvpe_str(self): if os.name != "nt": self._test_internal_execvpe(bytes) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.execve not implemented yet for all platforms") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON; os.execve not implemented yet for all platforms') def test_execve_invalid_env(self): args = [sys.executable, '-c', 'pass'] @@ -2308,8 +2287,7 @@ def test_execve_invalid_env(self): with self.assertRaises(ValueError): os.execve(args[0], args, newenv) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.execve not implemented yet for all platforms") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON; os.execve not implemented yet for all platforms') @unittest.skipUnless(sys.platform == "win32", "Win32-specific test") def test_execve_with_empty_path(self): # bpo-32890: Check GetLastError() misuse @@ -2390,8 +2368,7 @@ def check_bool(self, f, *args, **kwargs): with self.assertRaises(RuntimeWarning): f(fd, *args, **kwargs) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_fdopen(self): self.check(os.fdopen, encoding="utf-8") self.check_bool(os.fdopen, encoding="utf-8") @@ -2448,8 +2425,7 @@ def test_fchmod(self): def test_fchown(self): self.check(os.fchown, -1, -1) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(hasattr(os, 'fpathconf'), 'test needs os.fpathconf()') @unittest.skipIf( support.is_emscripten or support.is_wasi, @@ -2462,22 +2438,19 @@ def test_fpathconf(self): self.check_bool(os.pathconf, "PC_NAME_MAX") self.check_bool(os.fpathconf, "PC_NAME_MAX") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(hasattr(os, 'ftruncate'), 'test needs os.ftruncate()') def test_ftruncate(self): self.check(os.truncate, 0) self.check(os.ftruncate, 0) self.check_bool(os.truncate, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (OSError: [Errno 18] There are no more files.)") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON; (OSError: [Errno 18] There are no more files.)') @unittest.skipUnless(hasattr(os, 'lseek'), 'test needs os.lseek()') def test_lseek(self): self.check(os.lseek, 0, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (OSError: [Errno 18] There are no more files.)") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON; (OSError: [Errno 18] There are no more files.)') @unittest.skipUnless(hasattr(os, 'read'), 'test needs os.read()') def test_read(self): self.check(os.read, 1) @@ -2491,8 +2464,7 @@ def test_readv(self): def test_tcsetpgrpt(self): self.check(os.tcsetpgrp, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (OSError: [Errno 18] There are no more files.)") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON; (OSError: [Errno 18] There are no more files.)') @unittest.skipUnless(hasattr(os, 'write'), 'test needs os.write()') def test_write(self): self.check(os.write, b" ") @@ -2501,8 +2473,7 @@ def test_write(self): def test_writev(self): self.check(os.writev, [b'abc']) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.get_inheritable not implemented yet for all platforms") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON; os.get_inheritable not implemented yet for all platforms') @support.requires_subprocess() def test_inheritable(self): self.check(os.get_inheritable) @@ -2514,13 +2485,11 @@ def test_blocking(self): self.check(os.get_blocking) self.check(os.set_blocking, True) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_fchdir(self): return super().test_fchdir() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_fsync(self): return super().test_fsync() @@ -2563,7 +2532,6 @@ def test_unicode_name(self): self.file2 = self.file1 + "2" self._test_link(self.file1, self.file2) - @unittest.skipIf(sys.platform == "win32", "Posix specific tests") class PosixUidGidTests(unittest.TestCase): # uid_t and gid_t are 32-bit unsigned integers on Linux @@ -2765,14 +2733,12 @@ def _kill(self, sig): os.kill(proc.pid, sig) self.assertEqual(proc.wait(), sig) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (ModuleNotFoundError: No module named '_ctypes')") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; (ModuleNotFoundError: No module named '_ctypes')") def test_kill_sigterm(self): # SIGTERM doesn't mean anything special, but make sure it works self._kill(signal.SIGTERM) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (ModuleNotFoundError: No module named '_ctypes')") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; (ModuleNotFoundError: No module named '_ctypes')") def test_kill_int(self): # os.kill on Windows can take an int which gets set as the exit code self._kill(100) @@ -2831,8 +2797,7 @@ def test_CTRL_C_EVENT(self): self._kill_with_event(signal.CTRL_C_EVENT, "CTRL_C_EVENT") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_subprocess() def test_CTRL_BREAK_EVENT(self): self._kill_with_event(signal.CTRL_BREAK_EVENT, "CTRL_BREAK_EVENT") @@ -3186,8 +3151,7 @@ def tearDown(self): if os.path.lexists(self.junction): os.unlink(self.junction) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AttributeError: module '_winapi' has no attribute 'CreateJunction')") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; (AttributeError: module '_winapi' has no attribute 'CreateJunction')") def test_create_junction(self): _winapi.CreateJunction(self.junction_target, self.junction) self.assertTrue(os.path.lexists(self.junction)) @@ -3201,8 +3165,7 @@ def test_create_junction(self): self.assertEqual(os.path.normcase("\\\\?\\" + self.junction_target), os.path.normcase(os.readlink(self.junction))) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AttributeError: module '_winapi' has no attribute 'CreateJunction')") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; (AttributeError: module '_winapi' has no attribute 'CreateJunction')") def test_unlink_removes_junction(self): _winapi.CreateJunction(self.junction_target, self.junction) self.assertTrue(os.path.exists(self.junction)) @@ -3262,8 +3225,7 @@ def test_getfinalpathname_handles(self): self.assertEqual(0, handle_delta) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.stat (PermissionError: [Errno 5] Access is denied.)") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON; os.stat (PermissionError: [Errno 5] Access is denied.)') @support.requires_subprocess() def test_stat_unlink_race(self): # bpo-46785: the implementation of os.stat() falls back to reading @@ -3473,8 +3435,7 @@ def test_waitstatus_to_exitcode(self): with self.assertRaises(TypeError): os.waitstatus_to_exitcode(0.0) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.spawnv not implemented yet for all platforms") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON; os.spawnv not implemented yet for all platforms') @unittest.skipUnless(sys.platform == 'win32', 'win32-specific test') def test_waitpid_windows(self): # bpo-40138: test os.waitpid() and os.waitstatus_to_exitcode() @@ -3483,8 +3444,7 @@ def test_waitpid_windows(self): code = f'import _winapi; _winapi.ExitProcess({STATUS_CONTROL_C_EXIT})' self.check_waitpid(code, exitcode=STATUS_CONTROL_C_EXIT) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (OverflowError: Python int too large to convert to Rust i32)") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON; (OverflowError: Python int too large to convert to Rust i32)') @unittest.skipUnless(sys.platform == 'win32', 'win32-specific test') def test_waitstatus_to_exitcode_windows(self): max_exitcode = 2 ** 32 - 1 @@ -3608,8 +3568,7 @@ def test_nowait(self): pid = os.spawnv(os.P_NOWAIT, program, args) support.wait_process(pid, exitcode=self.exitcode) - # TODO: RUSTPYTHON fix spawnv bytes - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; fix spawnv bytes @requires_os_func('spawnve') def test_spawnve_bytes(self): # Test bytes handling in parse_arglist and parse_envlist (#28114) @@ -4224,16 +4183,8 @@ def test_eventfd_select(self): @unittest.skipIf(sys.platform == "android", "gh-124873: Test is flaky on Android") @support.requires_linux_version(2, 6, 30) class TimerfdTests(unittest.TestCase): - # 1 ms accuracy is reliably achievable on every platform except Android - # emulators, where we allow 10 ms (gh-108277). - - # XXX: RUSTPYTHON; AttributeError: module 'platform' has no attribute 'android_ver' - #if sys.platform == "android" and platform.android_ver().is_emulator: - if sys.platform == "android": - CLOCK_RES_PLACES = 2 - else: - CLOCK_RES_PLACES = 3 - + # gh-126112: Use 10 ms to tolerate slow buildbots + CLOCK_RES_PLACES = 2 # 10 ms CLOCK_RES = 10 ** -CLOCK_RES_PLACES CLOCK_RES_NS = 10 ** (9 - CLOCK_RES_PLACES) @@ -4577,8 +4528,7 @@ class Str(str): self.filenames = self.bytes_filenames + self.unicode_filenames - # TODO: RUSTPYTHON (AssertionError: b'@test_22106_tmp\xe7w\xf0' is not b'@test_22106_tmp\xe7w\xf0' : <built-in function chdir>) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; (AssertionError: b'@test_22106_tmp\xe7w\xf0' is not b'@test_22106_tmp\xe7w\xf0' : <built-in function chdir>) def test_oserror_filename(self): funcs = [ (self.filenames, os.chdir,), @@ -4671,8 +4621,7 @@ def test_process_cpu_count_affinity(self): # FD inheritance check is only useful for systems with process support. @support.requires_subprocess() class FDInheritanceTests(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.get_inheritable not implemented yet for all platforms") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON; os.get_inheritable not implemented yet for all platforms') def test_get_set_inheritable(self): fd = os.open(__file__, os.O_RDONLY) self.addCleanup(os.close, fd) @@ -4717,8 +4666,7 @@ def test_get_set_inheritable_o_path(self): os.set_inheritable(fd, False) self.assertEqual(os.get_inheritable(fd), False) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.get_inheritable not implemented yet for all platforms") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON; os.get_inheritable not implemented yet for all platforms') def test_get_set_inheritable_badf(self): fd = os_helper.make_bad_fd() @@ -4734,8 +4682,7 @@ def test_get_set_inheritable_badf(self): os.set_inheritable(fd, False) self.assertEqual(ctx.exception.errno, errno.EBADF) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.get_inheritable not implemented yet for all platforms") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON; os.get_inheritable not implemented yet for all platforms') def test_open(self): fd = os.open(__file__, os.O_RDONLY) self.addCleanup(os.close, fd) @@ -4749,8 +4696,7 @@ def test_pipe(self): self.assertEqual(os.get_inheritable(rfd), False) self.assertEqual(os.get_inheritable(wfd), False) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform == 'win32', "TODO: RUSTPYTHON; os.dup on windows") + @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON; os.dup on windows') def test_dup(self): fd1 = os.open(__file__, os.O_RDONLY) self.addCleanup(os.close, fd1) @@ -4759,15 +4705,13 @@ def test_dup(self): self.addCleanup(os.close, fd2) self.assertEqual(os.get_inheritable(fd2), False) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform == 'win32', "TODO: RUSTPYTHON; os.dup on windows") + @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON; os.dup on windows') def test_dup_standard_stream(self): fd = os.dup(1) self.addCleanup(os.close, fd) self.assertGreater(fd, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.dup not implemented yet for all platforms") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON; os.dup not implemented yet for all platforms') @unittest.skipUnless(sys.platform == 'win32', 'win32-specific test') def test_dup_nul(self): # os.dup() was creating inheritable fds for character files. @@ -4907,8 +4851,7 @@ class PathTConverterTests(unittest.TestCase): ('open', False, (os.O_RDONLY,), getattr(os, 'close', None)), ] - # TODO: RUSTPYTHON (AssertionError: TypeError not raised) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; (AssertionError: TypeError not raised) def test_path_t_converter(self): str_filename = os_helper.TESTFN if os.name == 'nt': @@ -4993,13 +4936,11 @@ def setUp(self): self.addCleanup(os_helper.rmtree, self.path) os.mkdir(self.path) - # TODO: RUSTPYTHON (AssertionError: TypeError not raised by DirEntry) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; (AssertionError: TypeError not raised by DirEntry) def test_uninstantiable(self): self.assertRaises(TypeError, os.DirEntry) - # TODO: RUSTPYTHON (pickle.PicklingError: Can't pickle <class '_os.DirEntry'>: it's not found as _os.DirEntry) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; (pickle.PicklingError: Can't pickle <class '_os.DirEntry'>: it's not found as _os.DirEntry) def test_unpickable(self): filename = create_file(os.path.join(self.path, "file.txt"), b'python') entry = [entry for entry in os.scandir(self.path)].pop() @@ -5044,8 +4985,7 @@ def assert_stat_equal(self, stat1, stat2, skip_fields): else: self.assertEqual(stat1, stat2) - # TODO: RUSTPYTHON (AssertionError: TypeError not raised by ScandirIter) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; (AssertionError: TypeError not raised by ScandirIter) def test_uninstantiable(self): scandir_iter = os.scandir(self.path) self.assertRaises(TypeError, type(scandir_iter)) @@ -5088,8 +5028,7 @@ def check_entry(self, entry, name, is_dir, is_file, is_symlink): entry_lstat, os.name == 'nt') - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform == "linux", "TODO: RUSTPYTHON, flaky test") + @unittest.skipIf(sys.platform == 'linux', 'TODO: RUSTPYTHON; flaky test') def test_attributes(self): link = os_helper.can_hardlink() symlink = os_helper.can_symlink() @@ -5189,8 +5128,7 @@ def test_fspath_protocol_bytes(self): self.assertEqual(fspath, os.path.join(os.fsencode(self.path),bytes_filename)) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON entry.is_dir() is False") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON; entry.is_dir() is False') def test_removed_dir(self): path = os.path.join(self.path, 'dir') @@ -5213,8 +5151,7 @@ def test_removed_dir(self): self.assertRaises(FileNotFoundError, entry.stat) self.assertRaises(FileNotFoundError, entry.stat, follow_symlinks=False) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON entry.is_file() is False") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON; entry.is_file() is False') def test_removed_file(self): entry = self.create_file_entry() os.unlink(entry.path) @@ -5275,8 +5212,7 @@ def test_bytes_like(self): with self.assertRaises(TypeError): os.scandir(path_bytes) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(os.listdir in os.supports_fd, 'fd support for listdir required for this test.') def test_fd(self): @@ -5303,8 +5239,7 @@ def test_fd(self): st = os.stat(entry.name, dir_fd=fd, follow_symlinks=False) self.assertEqual(entry.stat(follow_symlinks=False), st) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: FileNotFoundError not raised by scandir)") + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON; (AssertionError: FileNotFoundError not raised by scandir)') @unittest.skipIf(support.is_wasi, "WASI maps '' to cwd") def test_empty_path(self): self.assertRaises(FileNotFoundError, os.scandir, '') @@ -5360,8 +5295,7 @@ def test_context_manager_exception(self): with self.check_no_resource_warning(): del iterator - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_resource_warning(self): self.create_file("file.txt") self.create_file("file2.txt") @@ -5434,8 +5368,7 @@ class A(os.PathLike): def test_pathlike_class_getitem(self): self.assertIsInstance(os.PathLike[bytes], types.GenericAlias) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_pathlike_subclass_slots(self): class A(os.PathLike): __slots__ = () @@ -5443,8 +5376,7 @@ def __fspath__(self): return '' self.assertFalse(hasattr(A(), '__dict__')) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_fspath_set_to_None(self): class Foo: __fspath__ = None @@ -5547,8 +5479,7 @@ def test_fork_warns_when_non_python_thread_exists(self): self.assertEqual(err.decode("utf-8"), "") self.assertEqual(out.decode("utf-8"), "") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_fork_at_finalization(self): code = """if 1: import atexit diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 30d6b6d3c3..dc0c6fe51e 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1993,7 +1993,9 @@ def test_open_file(self): with open(outfile, encoding="utf-8") as f: self.assertEqual(f.read(), 'hello') - # TODO: RUSTPYTHON: FileNotFoundError: [Errno 2] No such file or directory (os error 2): '@test_55144_tmp' -> 'None' + # TODO: RUSTPYTHON: the rust runtime reopens closed stdio fds at startup, + # so this test fails, even though POSIX_SPAWN_CLOSE does + # actually have an effect @unittest.expectedFailure def test_close_file(self): closefile = os_helper.TESTFN diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py index 750afc1de7..6d10099c61 100644 --- a/Lib/test/test_py_compile.py +++ b/Lib/test/test_py_compile.py @@ -78,7 +78,6 @@ def test_absolute_path(self): self.assertTrue(os.path.exists(self.pyc_path)) self.assertFalse(os.path.exists(self.cache_path)) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_do_not_overwrite_symlinks(self): # In the face of a cfile argument being a symlink, bail out. # Issue #17222 diff --git a/Lib/test/test_raise.py b/Lib/test/test_raise.py index 3ada08f7dc..d7f0150907 100644 --- a/Lib/test/test_raise.py +++ b/Lib/test/test_raise.py @@ -185,6 +185,21 @@ def test_class_cause(self): else: self.fail("No exception raised") + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: 'classmethod' object is not callable + def test_class_cause_nonexception_result(self): + class ConstructsNone(BaseException): + @classmethod + def __new__(*args, **kwargs): + return None + try: + raise IndexError from ConstructsNone + except TypeError as e: + self.assertIn("should have returned an instance of BaseException", str(e)) + except IndexError: + self.fail("Wrong kind of exception raised") + else: + self.fail("No exception raised") + def test_instance_cause(self): cause = KeyError() try: @@ -233,8 +248,7 @@ class TestTracebackType(unittest.TestCase): def raiser(self): raise ValueError - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_attrs(self): try: self.raiser() diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index b2fee6207b..2492abff01 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -12,9 +12,16 @@ import textwrap import unittest import warnings -from test.support import no_tracing, verbose +from test.support import ( + force_not_colorized_test_class, + infinite_recursion, + no_tracing, + requires_resource, + requires_subprocess, + verbose, +) from test.support.import_helper import forget, make_legacy_pyc, unload -from test.support.os_helper import create_empty_file, temp_dir +from test.support.os_helper import create_empty_file, temp_dir, FakePath from test.support.script_helper import make_script, make_zip_script @@ -656,13 +663,14 @@ def test_basic_script(self): self._check_script(script_name, "<run_path>", script_name, script_name, expect_spec=False) - def test_basic_script_with_path_object(self): + def test_basic_script_with_pathlike_object(self): with temp_dir() as script_dir: mod_name = 'script' - script_name = pathlib.Path(self._make_test_script(script_dir, - mod_name)) - self._check_script(script_name, "<run_path>", script_name, - script_name, expect_spec=False) + script_name = self._make_test_script(script_dir, mod_name) + self._check_script(FakePath(script_name), "<run_path>", + script_name, + script_name, + expect_spec=False) def test_basic_script_no_suffix(self): with temp_dir() as script_dir: @@ -734,6 +742,7 @@ def test_zipfile_error(self): self._check_import_error(zip_name, msg) @no_tracing + @requires_resource('cpu') def test_main_recursion_error(self): with temp_dir() as script_dir, temp_dir() as dummy_dir: mod_name = '__main__' @@ -741,10 +750,10 @@ def test_main_recursion_error(self): "runpy.run_path(%r)\n") % dummy_dir script_name = self._make_test_script(script_dir, mod_name, source) zip_name, fname = make_zip_script(script_dir, 'test_zip', script_name) - self.assertRaises(RecursionError, run_path, zip_name) + with infinite_recursion(25): + self.assertRaises(RecursionError, run_path, zip_name) - # TODO: RUSTPYTHON, detect encoding comments in files - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; detect encoding comments in files def test_encoding(self): with temp_dir() as script_dir: filename = os.path.join(script_dir, 'script.py') @@ -757,6 +766,7 @@ def test_encoding(self): self.assertEqual(result['s'], "non-ASCII: h\xe9") +@force_not_colorized_test_class class TestExit(unittest.TestCase): STATUS_CONTROL_C_EXIT = 0xC000013A EXPECTED_CODE = ( @@ -783,16 +793,19 @@ def run(self, *args, **kwargs): ) super().run(*args, **kwargs) - def assertSigInt(self, *args, **kwargs): - proc = subprocess.run(*args, **kwargs, text=True, stderr=subprocess.PIPE) - self.assertTrue(proc.stderr.endswith("\nKeyboardInterrupt\n")) + @requires_subprocess() + def assertSigInt(self, cmd, *args, **kwargs): + # Use -E to ignore PYTHONSAFEPATH + cmd = [sys.executable, '-E', *cmd] + proc = subprocess.run(cmd, *args, **kwargs, text=True, stderr=subprocess.PIPE) + self.assertTrue(proc.stderr.endswith("\nKeyboardInterrupt\n"), proc.stderr) self.assertEqual(proc.returncode, self.EXPECTED_CODE) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; ") def test_pymain_run_file(self): - self.assertSigInt([sys.executable, self.ham]) + self.assertSigInt([self.ham]) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; ") def test_pymain_run_file_runpy_run_module(self): tmp = self.ham.parent run_module = tmp / "run_module.py" @@ -804,9 +817,9 @@ def test_pymain_run_file_runpy_run_module(self): """ ) ) - self.assertSigInt([sys.executable, run_module], cwd=tmp) + self.assertSigInt([run_module], cwd=tmp) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; ") def test_pymain_run_file_runpy_run_module_as_main(self): tmp = self.ham.parent run_module_as_main = tmp / "run_module_as_main.py" @@ -818,28 +831,27 @@ def test_pymain_run_file_runpy_run_module_as_main(self): """ ) ) - self.assertSigInt([sys.executable, run_module_as_main], cwd=tmp) + self.assertSigInt([run_module_as_main], cwd=tmp) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; ") def test_pymain_run_command_run_module(self): self.assertSigInt( - [sys.executable, "-c", "import runpy; runpy.run_module('ham')"], + ["-c", "import runpy; runpy.run_module('ham')"], cwd=self.ham.parent, ) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; ") def test_pymain_run_command(self): - self.assertSigInt([sys.executable, "-c", "import ham"], cwd=self.ham.parent) + self.assertSigInt(["-c", "import ham"], cwd=self.ham.parent) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_pymain_run_stdin(self): - self.assertSigInt([sys.executable], input="import ham", cwd=self.ham.parent) + self.assertSigInt([], input="import ham", cwd=self.ham.parent) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; ") def test_pymain_run_module(self): ham = self.ham - self.assertSigInt([sys.executable, "-m", ham.stem], cwd=ham.parent) + self.assertSigInt(["-m", ham.stem], cwd=ham.parent) if __name__ == "__main__": diff --git a/Lib/test/test_scope.py b/Lib/test/test_scope.py index 29a4ac3c16..662d8eefbc 100644 --- a/Lib/test/test_scope.py +++ b/Lib/test/test_scope.py @@ -177,6 +177,57 @@ def bar(): self.assertEqual(foo(a=42), 50) self.assertEqual(foo(), 25) + def testCellIsArgAndEscapes(self): + # We need to be sure that a cell passed in as an arg still + # gets wrapped in a new cell if the arg escapes into an + # inner function (closure). + + def external(): + value = 42 + def inner(): + return value + cell, = inner.__closure__ + return cell + cell_ext = external() + + def spam(arg): + def eggs(): + return arg + return eggs + + eggs = spam(cell_ext) + cell_closure, = eggs.__closure__ + cell_eggs = eggs() + + self.assertIs(cell_eggs, cell_ext) + self.assertIsNot(cell_eggs, cell_closure) + + def testCellIsLocalAndEscapes(self): + # We need to be sure that a cell bound to a local still + # gets wrapped in a new cell if the local escapes into an + # inner function (closure). + + def external(): + value = 42 + def inner(): + return value + cell, = inner.__closure__ + return cell + cell_ext = external() + + def spam(arg): + cell = arg + def eggs(): + return cell + return eggs + + eggs = spam(cell_ext) + cell_closure, = eggs.__closure__ + cell_eggs = eggs() + + self.assertIs(cell_eggs, cell_ext) + self.assertIsNot(cell_eggs, cell_closure) + def testRecursion(self): def f(x): @@ -641,10 +692,7 @@ def dec(self): self.assertEqual(c.dec(), 1) self.assertEqual(c.dec(), 0) - # TODO: RUSTPYTHON, figure out how to communicate that `y = 9` should be - # stored as a global rather than a STORE_NAME, even when - # the `global y` is in a nested subscope - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; figure out how to communicate that `y = 9` should be stored as a global rather than a STORE_NAME, even when the `global y` is in a nested subscope def testGlobalInParallelNestedFunctions(self): # A symbol table bug leaked the global statement from one # function to other nested functions in the same block. @@ -763,6 +811,30 @@ def dig(self): gc_collect() # For PyPy or other GCs. self.assertIsNone(ref()) + def test_multiple_nesting(self): + # Regression test for https://github.com/python/cpython/issues/121863 + class MultiplyNested: + def f1(self): + __arg = 1 + class D: + def g(self, __arg): + return __arg + return D().g(_MultiplyNested__arg=2) + + def f2(self): + __arg = 1 + class D: + def g(self, __arg): + return __arg + return D().g + + inst = MultiplyNested() + with self.assertRaises(TypeError): + inst.f1() + + closure = inst.f2() + with self.assertRaises(TypeError): + closure(_MultiplyNested__arg=2) if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index b64ccb37a5..1fe2aef39d 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -196,7 +196,7 @@ def test_rmtree_works_on_bytes(self): self.assertIsInstance(victim, bytes) shutil.rmtree(victim) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") + @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON; flaky') @os_helper.skip_unless_symlink def test_rmtree_fails_on_symlink_onerror(self): tmp = self.mkdtemp() @@ -216,7 +216,6 @@ def onerror(*args): self.assertEqual(errors[0][1], link) self.assertIsInstance(errors[0][2][1], OSError) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @os_helper.skip_unless_symlink def test_rmtree_fails_on_symlink_onexc(self): tmp = self.mkdtemp() @@ -1087,7 +1086,6 @@ def test_copymode_follow_symlinks(self): shutil.copymode(src_link, dst_link) self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @unittest.skipUnless(hasattr(os, 'lchmod') or os.name == 'nt', 'requires os.lchmod') @os_helper.skip_unless_symlink def test_copymode_symlink_to_symlink(self): @@ -1317,7 +1315,6 @@ def test_copy(self): self.assertTrue(os.path.exists(file2)) self.assertEqual(os.stat(file1).st_mode, os.stat(file2).st_mode) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @os_helper.skip_unless_symlink def test_copy_symlinks(self): tmp_dir = self.mkdtemp() @@ -1360,7 +1357,6 @@ def test_copy2(self): self.assertEqual(getattr(file1_stat, 'st_flags'), getattr(file2_stat, 'st_flags')) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @os_helper.skip_unless_symlink def test_copy2_symlinks(self): tmp_dir = self.mkdtemp() @@ -1445,7 +1441,6 @@ def _test_copy_dir(self, copy_func): ### shutil.copyfile - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @os_helper.skip_unless_symlink def test_copyfile_symlinks(self): tmp_dir = self.mkdtemp() @@ -1483,6 +1478,7 @@ def test_dont_copy_file_onto_link_to_itself(self): finally: shutil.rmtree(TESTFN, ignore_errors=True) + @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON; AssertionError: SameFileError not raised for copyfile') @os_helper.skip_unless_symlink def test_dont_copy_file_onto_symlink_to_itself(self): # bug 851123. diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 4ff187674e..f5b8d42dba 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -7,6 +7,7 @@ import unittest import test.support from test import support +from test.support.script_helper import assert_python_ok from test.support import os_helper from test.support import socket_helper from test.support import captured_stderr @@ -330,14 +331,14 @@ def test_getsitepackages(self): self.assertEqual(len(dirs), 2) wanted = os.path.join('xoxo', sys.platlibdir, # XXX: RUSTPYTHON - 'rustpython%d.%d' % sys.version_info[:2], + f'rustpython{sysconfig._get_python_version_abi()}', 'site-packages') self.assertEqual(dirs[0], wanted) else: self.assertEqual(len(dirs), 1) wanted = os.path.join('xoxo', 'lib', # XXX: RUSTPYTHON - 'rustpython%d.%d' % sys.version_info[:2], + f'rustpython{sysconfig._get_python_version_abi()}', 'site-packages') self.assertEqual(dirs[-1], wanted) else: @@ -358,9 +359,7 @@ def test_no_home_directory(self): with EnvironmentVarGuard() as environ, \ mock.patch('os.path.expanduser', lambda path: path): - - del environ['PYTHONUSERBASE'] - del environ['APPDATA'] + environ.unset('PYTHONUSERBASE', 'APPDATA') user_base = site.getuserbase() self.assertTrue(user_base.startswith('~' + os.sep), @@ -382,6 +381,19 @@ def test_no_home_directory(self): mock_addsitedir.assert_not_called() self.assertFalse(known_paths) + def test_gethistoryfile(self): + filename = 'file' + rc, out, err = assert_python_ok('-c', + f'import site; assert site.gethistoryfile() == "{filename}"', + PYTHON_HISTORY=filename) + self.assertEqual(rc, 0) + + # Check that PYTHON_HISTORY is ignored in isolated mode. + rc, out, err = assert_python_ok('-I', '-c', + f'import site; assert site.gethistoryfile() != "{filename}"', + PYTHON_HISTORY=filename) + self.assertEqual(rc, 0) + def test_trace(self): message = "bla-bla-bla" for verbose, out in (True, message + "\n"), (False, ""): @@ -509,6 +521,44 @@ def test_sitecustomize_executed(self): else: self.fail("sitecustomize not imported automatically") + @support.requires_subprocess() + def test_customization_modules_on_startup(self): + mod_names = [ + 'sitecustomize' + ] + + if site.ENABLE_USER_SITE: + mod_names.append('usercustomize') + + temp_dir = tempfile.mkdtemp() + self.addCleanup(os_helper.rmtree, temp_dir) + + with EnvironmentVarGuard() as environ: + environ['PYTHONPATH'] = temp_dir + + for module_name in mod_names: + os_helper.rmtree(temp_dir) + os.mkdir(temp_dir) + + customize_path = os.path.join(temp_dir, f'{module_name}.py') + eyecatcher = f'EXECUTED_{module_name}' + + with open(customize_path, 'w') as f: + f.write(f'print("{eyecatcher}")') + + output = subprocess.check_output([sys.executable, '-c', '""']) + self.assertIn(eyecatcher, output.decode('utf-8')) + + # -S blocks any site-packages + output = subprocess.check_output([sys.executable, '-S', '-c', '""']) + self.assertNotIn(eyecatcher, output.decode('utf-8')) + + # -s blocks user site-packages + if 'usercustomize' == module_name: + output = subprocess.check_output([sys.executable, '-s', '-c', '""']) + self.assertNotIn(eyecatcher, output.decode('utf-8')) + + @unittest.skipUnless(hasattr(urllib.request, "HTTPSHandler"), 'need SSL support to download license') @test.support.requires_resource('network') diff --git a/Lib/test/test_slice.py b/Lib/test/test_slice.py index 53d4c77616..6de7e73c39 100644 --- a/Lib/test/test_slice.py +++ b/Lib/test/test_slice.py @@ -286,8 +286,7 @@ def test_deepcopy(self): self.assertIsNot(s.stop, c.stop) self.assertIsNot(s.step, c.step) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cycle(self): class myobj(): pass o = myobj() diff --git a/Lib/test/test_sqlite3/test_regression.py b/Lib/test/test_sqlite3/test_regression.py index a658ff1f3c..d746be647c 100644 --- a/Lib/test/test_sqlite3/test_regression.py +++ b/Lib/test/test_sqlite3/test_regression.py @@ -195,8 +195,6 @@ def __del__(self): con.isolation_level = value self.assertEqual(con.isolation_level, "DEFERRED") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_cursor_constructor_call_check(self): """ Verifies that cursor methods check whether base class __init__ was @@ -221,8 +219,6 @@ def test_str_subclass(self): class MyStr(str): pass self.con.execute("select ?", (MyStr("abc"),)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_connection_constructor_call_check(self): """ Verifies that connection methods check whether base class __init__ was diff --git a/Lib/test/test_stat.py b/Lib/test/test_stat.py index ec4cca4528..49013a4bcd 100644 --- a/Lib/test/test_stat.py +++ b/Lib/test/test_stat.py @@ -325,21 +325,10 @@ def test_macosx_attribute_values(self): class TestFilemodeCStat(TestFilemode, unittest.TestCase): statmod = c_stat - # TODO: RUSTPYTHON - if sys.platform == "win32": - @unittest.expectedFailure - def test_link(self): - super().test_link() class TestFilemodePyStat(TestFilemode, unittest.TestCase): statmod = py_stat - # TODO: RUSTPYTHON - if sys.platform == "win32": - @unittest.expectedFailure - def test_link(self): - super().test_link() - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_str.py b/Lib/test/test_str.py index ef2d211a61..9d43a33cd9 100644 --- a/Lib/test/test_str.py +++ b/Lib/test/test_str.py @@ -112,8 +112,7 @@ def test_literals(self): # raw strings should not have unicode escapes self.assertNotEqual(r"\u0020", " ") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_ascii(self): self.assertEqual(ascii('abc'), "'abc'") self.assertEqual(ascii('ab\\c'), "'ab\\\\c'") @@ -566,7 +565,7 @@ def __str__(self): return self.sval self.checkraises(TypeError, ' ', 'join', [1, 2, 3]) self.checkraises(TypeError, ' ', 'join', ['1', '2', 3]) - @unittest.skip("TODO: RUSTPYTHON, oom handling") + @unittest.skip('TODO: RUSTPYTHON; oom handling') @unittest.skipIf(sys.maxsize > 2**32, 'needs too much memory on a 64-bit platform') def test_join_overflow(self): @@ -795,8 +794,7 @@ def test_isdecimal(self): for ch in ['\U0001D7F6', '\U00011066', '\U000104A0']: self.assertTrue(ch.isdecimal(), '{!a} is decimal.'.format(ch)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_isdigit(self): super().test_isdigit() self.checkequalnofix(True, '\u2460', 'isdigit') @@ -942,8 +940,7 @@ def test_upper(self): self.assertEqual('\U0008fffe'.upper(), '\U0008fffe') self.assertEqual('\u2177'.upper(), '\u2167') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_capitalize(self): string_tests.StringLikeTest.test_capitalize(self) self.assertEqual('\U0001044F'.capitalize(), '\U00010427') @@ -961,8 +958,7 @@ def test_capitalize(self): self.assertEqual('finnish'.capitalize(), 'Finnish') self.assertEqual('A\u0345\u03a3'.capitalize(), 'A\u0345\u03c2') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_title(self): super().test_title() self.assertEqual('\U0001044F'.title(), '\U00010427') @@ -980,8 +976,7 @@ def test_title(self): self.assertEqual('A\u03a3 \u1fa1xy'.title(), 'A\u03c2 \u1fa9xy') self.assertEqual('A\u03a3A'.title(), 'A\u03c3a') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_swapcase(self): string_tests.StringLikeTest.test_swapcase(self) self.assertEqual('\U0001044F'.swapcase(), '\U00010427') @@ -1081,8 +1076,7 @@ def test_issue18183(self): '\U00100000'.ljust(3, '\U00010000') '\U00100000'.rjust(3, '\U00010000') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_format(self): self.assertEqual(''.format(), '') self.assertEqual('a'.format(), 'a') @@ -1244,10 +1238,10 @@ def __repr__(self): self.assertEqual('{0:\x00^6}'.format(3), '\x00\x003\x00\x00\x00') self.assertEqual('{0:<6}'.format(3), '3 ') - self.assertEqual('{0:\x00<6}'.format(3.14), '3.14\x00\x00') - self.assertEqual('{0:\x01<6}'.format(3.14), '3.14\x01\x01') - self.assertEqual('{0:\x00^6}'.format(3.14), '\x003.14\x00') - self.assertEqual('{0:^6}'.format(3.14), ' 3.14 ') + self.assertEqual('{0:\x00<6}'.format(3.25), '3.25\x00\x00') + self.assertEqual('{0:\x01<6}'.format(3.25), '3.25\x01\x01') + self.assertEqual('{0:\x00^6}'.format(3.25), '\x003.25\x00') + self.assertEqual('{0:^6}'.format(3.25), ' 3.25 ') self.assertEqual('{0:\x00<12}'.format(3+2.0j), '(3+2j)\x00\x00\x00\x00\x00\x00') self.assertEqual('{0:\x01<12}'.format(3+2.0j), '(3+2j)\x01\x01\x01\x01\x01\x01') @@ -1466,21 +1460,19 @@ def __getitem__(self, key): self.assertRaises(TypeError, '{a}'.format_map, []) self.assertRaises(ZeroDivisionError, '{a}'.format_map, BadMapping()) - @unittest.skip("TODO: RUSTPYTHON, killed for chewing up RAM") + @unittest.skip('TODO: RUSTPYTHON; killed for chewing up RAM') def test_format_huge_precision(self): format_string = ".{}f".format(sys.maxsize + 1) with self.assertRaises(ValueError): result = format(2.34, format_string) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_format_huge_width(self): format_string = "{}f".format(sys.maxsize + 1) with self.assertRaises(ValueError): result = format(2.34, format_string) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_format_huge_item_number(self): format_string = "{{{}:.6f}}".format(sys.maxsize + 1) with self.assertRaises(ValueError): @@ -1516,8 +1508,7 @@ def __format__(self, spec): self.assertEqual('{:{f}}{g}{}'.format(1, 3, g='g', f=2), ' 1g3') self.assertEqual('{f:{}}{}{g}'.format(2, 4, f=1, g='g'), ' 14g') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_formatting(self): string_tests.StringLikeTest.test_formatting(self) # Testing Unicode formatting strings... @@ -1766,8 +1757,7 @@ def __str__(self): 'character buffers are decoded to unicode' ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_constructor_keyword_args(self): """Pass various keyword argument combinations to the constructor.""" # The object argument can be passed as a keyword. @@ -1777,8 +1767,7 @@ def test_constructor_keyword_args(self): self.assertEqual(str(b'foo', errors='strict'), 'foo') # not "b'foo'" self.assertEqual(str(object=b'foo', errors='strict'), 'foo') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_constructor_defaults(self): """Check the constructor argument defaults.""" # The object argument defaults to '' or b''. @@ -1790,8 +1779,7 @@ def test_constructor_defaults(self): # The errors argument defaults to strict. self.assertRaises(UnicodeDecodeError, str, utf8_cent, encoding='ascii') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_codecs_utf7(self): utfTests = [ ('A\u2262\u0391.', b'A+ImIDkQ.'), # RFC2152 example @@ -2301,8 +2289,7 @@ def test_codecs_errors(self): self.assertRaises(ValueError, complex, "\ud800") self.assertRaises(ValueError, complex, "\udf00") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_codecs(self): # Encoding self.assertEqual('hello'.encode('ascii'), b'hello') @@ -2432,8 +2419,7 @@ def test_ucs4(self): else: self.fail("Should have raised UnicodeDecodeError") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_conversion(self): # Make sure __str__() works properly class StrWithStr(str): @@ -2482,7 +2468,7 @@ def test_printable_repr(self): # This test only affects 32-bit platforms because expandtabs can only take # an int as the max value, not a 64-bit C long. If expandtabs is changed # to take a 64-bit long, this test should apply to all platforms. - @unittest.skip("TODO: RUSTPYTHON, oom handling") + @unittest.skip('TODO: RUSTPYTHON; oom handling') @unittest.skipIf(sys.maxsize > (1 << 32) or struct.calcsize('P') != 4, 'only applies to 32-bit platforms') def test_expandtabs_overflows_gracefully(self): @@ -2493,7 +2479,7 @@ def test_expandtabs_optimization(self): s = 'abc' self.assertIs(s.expandtabs(), s) - @unittest.skip("TODO: RUSTPYTHON, aborted: memory allocation of 9223372036854775759 bytes failed") + @unittest.skip('TODO: RUSTPYTHON; aborted: memory allocation of 9223372036854775759 bytes failed') def test_raiseMemError(self): asciifields = "nnb" compactfields = asciifields + "nP" @@ -2633,14 +2619,12 @@ def test_compare(self): self.assertTrue(astral >= bmp2) self.assertFalse(astral >= astral2) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_free_after_iterating(self): support.check_free_after_iterating(self, iter, str) support.check_free_after_iterating(self, reversed, str) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_check_encoding_errors(self): # bpo-37388: str(bytes) and str.decode() must check encoding and errors # arguments in dev mode @@ -2701,8 +2685,7 @@ def test_check_encoding_errors(self): proc = assert_python_failure('-X', 'dev', '-c', code) self.assertEqual(proc.rc, 10, proc) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_str_invalid_call(self): # too many args with self.assertRaisesRegex(TypeError, r"str expected at most 3 arguments, got 4"): diff --git a/Lib/test/test_string_literals.py b/Lib/test/test_string_literals.py index 098e8d3984..c6b2ffb9de 100644 --- a/Lib/test/test_string_literals.py +++ b/Lib/test/test_string_literals.py @@ -105,8 +105,7 @@ def test_eval_str_incomplete(self): self.assertRaises(SyntaxError, eval, r""" '\U000000' """) self.assertRaises(SyntaxError, eval, r""" '\U0000000' """) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_eval_str_invalid_escape(self): for b in range(1, 128): if b in b"""\n\r"'01234567NU\\abfnrtuvx""": @@ -145,8 +144,7 @@ def test_eval_str_invalid_escape(self): self.assertRegex(str(w[0].message), 'invalid escape sequence') self.assertEqual(w[0].filename, '<string>') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_eval_str_invalid_octal_escape(self): for i in range(0o400, 0o1000): with self.assertWarns(SyntaxWarning): @@ -172,8 +170,7 @@ def test_eval_str_invalid_octal_escape(self): self.assertEqual(exc.lineno, 2) self.assertEqual(exc.offset, 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid_escape_locations_with_offset(self): with warnings.catch_warnings(record=True) as w: warnings.simplefilter('error', category=SyntaxWarning) @@ -223,8 +220,7 @@ def test_eval_bytes_incomplete(self): self.assertRaises(SyntaxError, eval, r""" b'\x' """) self.assertRaises(SyntaxError, eval, r""" b'\x0' """) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_eval_bytes_invalid_escape(self): for b in range(1, 128): if b in b"""\n\r"'01234567\\abfnrtvx""": @@ -250,8 +246,7 @@ def test_eval_bytes_invalid_escape(self): self.assertEqual(exc.filename, '<string>') self.assertEqual(exc.lineno, 2) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_eval_bytes_invalid_octal_escape(self): for i in range(0o400, 0o1000): with self.assertWarns(SyntaxWarning): diff --git a/Lib/test/test_strtod.py b/Lib/test/test_strtod.py index b8b7a9d502..45fdcca92e 100644 --- a/Lib/test/test_strtod.py +++ b/Lib/test/test_strtod.py @@ -146,7 +146,7 @@ def test_short_halfway_cases(self): digits *= 5 exponent -= 1 - @unittest.skip("TODO: RUSTPYTHON, fails on debug mode, flaky in release mode") + @unittest.skip('TODO: RUSTPYTHON; fails on debug mode, flaky in release mode') def test_halfway_cases(self): # test halfway cases for the round-half-to-even rule for i in range(100 * TEST_SIZE): @@ -173,8 +173,7 @@ def test_halfway_cases(self): s = '{}e{}'.format(digits, exponent) self.check_strtod(s) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_boundaries(self): # boundaries expressed as triples (n, e, u), where # n*10**e is an approximation to the boundary value and @@ -195,8 +194,7 @@ def test_boundaries(self): u *= 10 e -= 1 - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_underflow_boundary(self): # test values close to 2**-1075, the underflow boundary; similar # to boundary_tests, except that the random error doesn't scale @@ -208,8 +206,7 @@ def test_underflow_boundary(self): s = '{}e{}'.format(digits, exponent) self.check_strtod(s) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bigcomp(self): for ndigs in 5, 10, 14, 15, 16, 17, 18, 19, 20, 40, 41, 50: dig10 = 10**ndigs @@ -219,8 +216,7 @@ def test_bigcomp(self): s = '{}e{}'.format(digits, exponent) self.check_strtod(s) - # TODO: RUSTPYTHON, Incorrectly rounded str->float conversion for -07e-321 - @unittest.skip("TODO: RUSTPYTHON; flaky test") + @unittest.skip('TODO: RUSTPYTHON; flaky test') def test_parsing(self): # make '0' more likely to be chosen than other digits digits = '000000123456789' @@ -288,8 +284,7 @@ def negative_exp(n): self.assertEqual(float(negative_exp(20000)), 1.0) self.assertEqual(float(negative_exp(30000)), 1.0) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_particular(self): # inputs that produced crashes or incorrectly rounded results with # previous versions of dtoa.c, for various reasons diff --git a/Lib/test/test_sundry.py b/Lib/test/test_sundry.py index 90af9da8f9..f4a8d434ed 100644 --- a/Lib/test/test_sundry.py +++ b/Lib/test/test_sundry.py @@ -1,14 +1,11 @@ """Do a minimal test of all the modules that aren't otherwise tested.""" import importlib -import platform -import sys from test import support from test.support import import_helper from test.support import warnings_helper import unittest class TestUntestedModules(unittest.TestCase): - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_untested_modules_can_be_imported(self): untested = ('encodings',) with warnings_helper.check_warnings(quiet=True): @@ -21,31 +18,6 @@ def test_untested_modules_can_be_imported(self): self.fail('{} has tests even though test_sundry claims ' 'otherwise'.format(name)) - import distutils.bcppcompiler - import distutils.ccompiler - import distutils.cygwinccompiler - import distutils.filelist - import distutils.text_file - import distutils.unixccompiler - - import distutils.command.bdist_dumb - if sys.platform.startswith('win') and not platform.win32_is_iot(): - import distutils.command.bdist_msi - import distutils.command.bdist - import distutils.command.bdist_rpm - import distutils.command.build_clib - import distutils.command.build_ext - import distutils.command.build - import distutils.command.clean - import distutils.command.config - import distutils.command.install_data - import distutils.command.install_egg_info - import distutils.command.install_headers - import distutils.command.install_lib - import distutils.command.register - import distutils.command.sdist - import distutils.command.upload - import html.entities try: @@ -54,5 +26,6 @@ def test_untested_modules_can_be_imported(self): if support.verbose: print("skipping tty") + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 7e46773047..29dd04995e 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -259,6 +259,36 @@ Traceback (most recent call last): SyntaxError: invalid syntax +Comprehensions without 'in' keyword: + +>>> [x for x if range(1)] +Traceback (most recent call last): +SyntaxError: 'in' expected after for-loop variables + +>>> tuple(x for x if range(1)) +Traceback (most recent call last): +SyntaxError: 'in' expected after for-loop variables + +>>> [x for x() in a] +Traceback (most recent call last): +SyntaxError: cannot assign to function call + +>>> [x for a, b, (c + 1, d()) in y] +Traceback (most recent call last): +SyntaxError: cannot assign to expression + +>>> [x for a, b, (c + 1, d()) if y] +Traceback (most recent call last): +SyntaxError: 'in' expected after for-loop variables + +>>> [x for x+1 in y] +Traceback (most recent call last): +SyntaxError: cannot assign to expression + +>>> [x for x+1, x() in y] +Traceback (most recent call last): +SyntaxError: cannot assign to expression + Comprehensions creating tuples without parentheses should produce a specialized error message: @@ -322,6 +352,13 @@ Traceback (most recent call last): SyntaxError: invalid syntax +# But prefixes of soft keywords should +# still raise specialized errors + +>>> (mat x) +Traceback (most recent call last): +SyntaxError: invalid syntax. Perhaps you forgot a comma? + From compiler_complex_args(): >>> def f(None=1): @@ -334,7 +371,12 @@ >>> def f(x, y=1, z): ... pass Traceback (most recent call last): -SyntaxError: non-default argument follows default argument +SyntaxError: parameter without a default follows parameter with a default + +>>> def f(x, /, y=1, z): +... pass +Traceback (most recent call last): +SyntaxError: parameter without a default follows parameter with a default >>> def f(x, None): ... pass @@ -555,8 +597,16 @@ Traceback (most recent call last): SyntaxError: expected default value expression -# TODO: RUSTPYTHON NameError: name 'PyCF_TYPE_COMMENTS' is not defined ->>> import ast; ast.parse(''' # doctest: +SKIP +>>> lambda a,d=3,c: None +Traceback (most recent call last): +SyntaxError: parameter without a default follows parameter with a default + +>>> lambda a,/,d=3,c: None +Traceback (most recent call last): +SyntaxError: parameter without a default follows parameter with a default + +>>> # TODO: RUSTPYTHON +>>> import ast; ast.parse(''' # doctest: +SKIP ... def f( ... *, # type: int ... a, # type: int @@ -582,46 +632,31 @@ >>> L = range(10) >>> f(x for x in L) [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - -# TODO: RUSTPYTHON does not raise. ->>> f(x for x in L, 1) # doctest: +SKIP +>>> f(x for x in L, 1) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized - -# TODO: RUSTPYTHON does not raise. ->>> f(x for x in L, y=1) # doctest: +SKIP +>>> f(x for x in L, y=1) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized - -# TODO: RUSTPYTHON does not raise. ->>> f(x for x in L, *[]) # doctest: +SKIP +>>> f(x for x in L, *[]) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized - -# TODO: RUSTPYTHON does not raise. ->>> f(x for x in L, **{}) # doctest: +SKIP +>>> f(x for x in L, **{}) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized - -# TODO: RUSTPYTHON does not raise. ->>> f(L, x for x in L) # doctest: +SKIP +>>> f(L, x for x in L) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized - -# TODO: RUSTPYTHON does not raise. ->>> f(x for x in L, y for y in L) # doctest: +SKIP +>>> f(x for x in L, y for y in L) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized - -# TODO: RUSTPYTHON does not raise. ->>> f(x for x in L,) # doctest: +SKIP +>>> f(x for x in L,) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized >>> f((x for x in L), 1) [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - -# TODO: RUSTPYTHON TypeError: metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases ->>> class C(x for x in L): # doctest: +SKIP +>>> # TODO: RUSTPYTHON +>>> class C(x for x in L): # doctest: +SKIP ... pass Traceback (most recent call last): SyntaxError: invalid syntax @@ -742,8 +777,8 @@ >>> f(x.y=1) Traceback (most recent call last): SyntaxError: expression cannot contain assignment, perhaps you meant "=="? -# TODO: RUSTPYTHON ->>> f((x)=2) # doctest: +SKIP +>>> # TODO: RUSTPYTHON +>>> f((x)=2) # doctest: +SKIP Traceback (most recent call last): SyntaxError: expression cannot contain assignment, perhaps you meant "=="? >>> f(True=1) @@ -758,11 +793,31 @@ >>> f(__debug__=1) Traceback (most recent call last): SyntaxError: cannot assign to __debug__ - -# TODO: RUSTPYTHON NameError: name '__annotations__' is not defined ->>> __debug__: int # doctest: +SKIP +>>> # TODO: RUSTPYTHON +>>> __debug__: int # doctest: +SKIP Traceback (most recent call last): SyntaxError: cannot assign to __debug__ +>>> f(a=) +Traceback (most recent call last): +SyntaxError: expected argument value expression +>>> f(a, b, c=) +Traceback (most recent call last): +SyntaxError: expected argument value expression +>>> f(a, b, c=, d) +Traceback (most recent call last): +SyntaxError: expected argument value expression +>>> f(*args=[0]) +Traceback (most recent call last): +SyntaxError: cannot assign to iterable argument unpacking +>>> f(a, b, *args=[0]) +Traceback (most recent call last): +SyntaxError: cannot assign to iterable argument unpacking +>>> f(**kwargs={'a': 1}) +Traceback (most recent call last): +SyntaxError: cannot assign to keyword argument unpacking +>>> f(a, b, *args, **kwargs={'a': 1}) +Traceback (most recent call last): +SyntaxError: cannot assign to keyword argument unpacking More set_context(): @@ -990,11 +1045,26 @@ Traceback (most recent call last): SyntaxError: expected ':' + >>> def f[T]() + ... pass + Traceback (most recent call last): + SyntaxError: expected ':' + >>> class A ... pass Traceback (most recent call last): SyntaxError: expected ':' + >>> class A[T] + ... pass + Traceback (most recent call last): + SyntaxError: expected ':' + + >>> class A[T]() + ... pass + Traceback (most recent call last): + SyntaxError: expected ':' + >>> class R&D: ... pass Traceback (most recent call last): @@ -1154,6 +1224,22 @@ Traceback (most recent call last): SyntaxError: expected '(' + >>> def f -> int: + Traceback (most recent call last): + SyntaxError: expected '(' + + >>> async def f -> int: # type: int + Traceback (most recent call last): + SyntaxError: expected '(' + + >>> async def f[T]: + Traceback (most recent call last): + SyntaxError: expected '(' + + >>> def f[T] -> str: + Traceback (most recent call last): + SyntaxError: expected '(' + Parenthesized arguments in function definitions >>> def f(x, (y, z), w): @@ -1432,11 +1518,21 @@ Traceback (most recent call last): IndentationError: expected an indented block after function definition on line 1 + >>> def foo[T](x, /, y, *, z=2): + ... pass + Traceback (most recent call last): + IndentationError: expected an indented block after function definition on line 1 + >>> class Blech(A): ... pass Traceback (most recent call last): IndentationError: expected an indented block after class definition on line 1 + >>> class Blech[T](A): + ... pass + Traceback (most recent call last): + IndentationError: expected an indented block after class definition on line 1 + >>> match something: ... pass Traceback (most recent call last): @@ -1469,14 +1565,16 @@ Check that an multiple exception types with missing parentheses raise a custom exception - >>> try: + >>> # TODO: RUSTPYTHON + >>> try: # doctest: +SKIP ... pass ... except A, B: ... pass Traceback (most recent call last): SyntaxError: multiple exception types must be parenthesized - >>> try: + >>> # TODO: RUSTPYTHON + >>> try: # doctest: +SKIP ... pass ... except A, B, C: ... pass @@ -1591,30 +1689,113 @@ Traceback (most recent call last): SyntaxError: trailing comma not allowed without surrounding parentheses +>>> import a from b +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a.y.z from b.y.z +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a from b as bar +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a.y.z from b.y.z as bar +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a, b,c from b +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a.y.z, b.y.z, c.y.z from b.y.z +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a,b,c from b as bar +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a.y.z, b.y.z, c.y.z from b.y.z as bar +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + # Check that we dont raise the "trailing comma" error if there is more # input to the left of the valid part that we parsed. ->>> from t import x,y, and 3 +>>> from t import x,y, and 3 Traceback (most recent call last): SyntaxError: invalid syntax -# TODO: RUSTPYTHON nothing raised. ->>> (): int # doctest: +SKIP +>>> from i import +Traceback (most recent call last): +SyntaxError: Expected one or more names after 'import' + +>>> from .. import +Traceback (most recent call last): +SyntaxError: Expected one or more names after 'import' + +>>> import +Traceback (most recent call last): +SyntaxError: Expected one or more names after 'import' + +>>> (): int Traceback (most recent call last): SyntaxError: only single target (not tuple) can be annotated -# TODO: RUSTPYTHON nothing raised. ->>> []: int # doctest: +SKIP +>>> []: int Traceback (most recent call last): SyntaxError: only single target (not list) can be annotated -# TODO: RUSTPYTHON nothing raised. ->>> (()): int # doctest: +SKIP +>>> (()): int Traceback (most recent call last): SyntaxError: only single target (not tuple) can be annotated -# TODO: RUSTPYTHON nothing raised. ->>> ([]): int # doctest: +SKIP +>>> ([]): int Traceback (most recent call last): SyntaxError: only single target (not list) can be annotated +# 'not' after operators: + +>>> 3 + not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> 3 * not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> + not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> - not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> ~ not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> 3 + - not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> 3 + not -1 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +# Check that we don't introduce misleading errors +>>> not 1 */ 2 +Traceback (most recent call last): +SyntaxError: invalid syntax + +>>> not 1 + +Traceback (most recent call last): +SyntaxError: invalid syntax + +>>> not + 1 + +Traceback (most recent call last): +SyntaxError: invalid syntax + Corner-cases that used to fail to raise the correct error: >>> def f(*, x=lambda __debug__:0): pass @@ -1635,8 +1816,7 @@ Corner-cases that used to crash: - # TODO: RUSTPYTHON nothing raised. - >>> def f(**__debug__): pass # doctest: +SKIP + >>> def f(**__debug__): pass Traceback (most recent call last): SyntaxError: cannot assign to __debug__ @@ -1650,8 +1830,8 @@ Invalid pattern matching constructs: - # TODO: RUSTPYTHON nothing raised. - >>> match ...: # doctest: +SKIP + >>> # TODO: RUSTPYTHON + >>> match ...: # doctest: +SKIP ... case 42 as _: ... ... Traceback (most recent call last): @@ -1751,22 +1931,22 @@ >>> A[*(1:2)] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression >>> A[*(1:2)] = 1 Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression >>> del A[*(1:2)] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression A[*:] and A[:*] >>> A[*:] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression >>> A[:*] Traceback (most recent call last): ... @@ -1777,7 +1957,7 @@ >>> A[*] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression A[**] @@ -1829,10 +2009,246 @@ def f(x: *b) Traceback (most recent call last): ... SyntaxError: invalid syntax + +Invalid bytes literals: + + >>> b"Ā" + Traceback (most recent call last): + ... + b"Ā" + ^^^ + SyntaxError: bytes can only contain ASCII literal characters + + >>> b"абвгде" + Traceback (most recent call last): + ... + b"абвгде" + ^^^^^^^^ + SyntaxError: bytes can only contain ASCII literal characters + + >>> b"abc ъющый" # first 3 letters are ascii + Traceback (most recent call last): + ... + b"abc ъющый" + ^^^^^^^^^^^ + SyntaxError: bytes can only contain ASCII literal characters + +Invalid expressions in type scopes: + + >>> type A[] = int + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> class A[]: ... + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> def some[](): ... + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> def some[]() + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> async def some[]: # type: int + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> def f[T: (x:=3)](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar bound + + >>> def f[T: ((x:= 3), int)](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar constraint + + >>> def f[T = ((x:=3))](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar default + + >>> async def f[T: (x:=3)](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar bound + + >>> async def f[T: ((x:= 3), int)](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar constraint + + >>> async def f[T = ((x:=3))](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar default + + >>> type A[T: (x:=3)] = int + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar bound + + >>> type A[T: ((x:= 3), int)] = int + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar constraint + + >>> type A[T = ((x:=3))] = int + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar default + + >>> def f[T: (yield)](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar bound + + >>> def f[T: (int, (yield))](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar constraint + + >>> def f[T = (yield)](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar default + + >>> def f[*Ts = (yield)](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVarTuple default + + >>> def f[**P = [(yield), int]](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a ParamSpec default + + >>> type A[T: (yield 3)] = int + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar bound + + >>> type A[T: (int, (yield 3))] = int + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar constraint + + >>> type A[T = (yield 3)] = int + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar default + + >>> type A[T: (await 3)] = int + Traceback (most recent call last): + ... + SyntaxError: await expression cannot be used within a TypeVar bound + + >>> type A[T: (yield from [])] = int + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar bound + + >>> class A[T: (yield 3)]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar bound + + >>> class A[T: (int, (yield 3))]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar constraint + + >>> class A[T = (yield)]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar default + + >>> class A[*Ts = (yield)]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVarTuple default + + >>> class A[**P = [(yield), int]]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a ParamSpec default + + >>> # TODO: RUSTPYTHON + >>> type A = (x := 3) # doctest: +SKIP + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a type alias + + >>> type A = (yield 3) + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a type alias + + >>> type A = (await 3) + Traceback (most recent call last): + ... + SyntaxError: await expression cannot be used within a type alias + + >>> type A = (yield from []) + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a type alias + + >>> class A[T]((x := 3)): ... + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within the definition of a generic + + >>> class A[T]((yield 3)): ... + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within the definition of a generic + + >>> class A[T]((await 3)): ... + Traceback (most recent call last): + ... + SyntaxError: await expression cannot be used within the definition of a generic + + >>> class A[T]((yield from [])): ... + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within the definition of a generic + + >>> f(**x, *y) + Traceback (most recent call last): + SyntaxError: iterable argument unpacking follows keyword argument unpacking + + >>> f(**x, *) + Traceback (most recent call last): + SyntaxError: Invalid star expression + + >>> f(x, *:) + Traceback (most recent call last): + SyntaxError: Invalid star expression + + >>> f(x, *) + Traceback (most recent call last): + SyntaxError: Invalid star expression + + >>> f(x = 5, *) + Traceback (most recent call last): + SyntaxError: Invalid star expression + + >>> f(x = 5, *:) + Traceback (most recent call last): + SyntaxError: Invalid star expression """ import re import doctest +import textwrap import unittest from test import support @@ -1869,8 +2285,7 @@ def _check_error(self, code, errtext, else: self.fail("compile() did not raise SyntaxError") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_expression_with_assignment(self): self._check_error( "print(end1 + end2 = ' ')", @@ -1878,16 +2293,14 @@ def test_expression_with_assignment(self): offset=7 ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_curly_brace_after_primary_raises_immediately(self): self._check_error("f{}", "invalid syntax", mode="single") def test_assign_call(self): self._check_error("f() = 1", "assign") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_assign_del(self): self._check_error("del (,)", "invalid syntax") self._check_error("del 1", "cannot delete literal") @@ -1939,9 +2352,6 @@ def error2(): """ self._check_error(source, "parameter and nonlocal", lineno=3) - def test_break_outside_loop(self): - self._check_error("break", "outside loop") - def test_yield_outside_function(self): self._check_error("if 0: yield", "outside function") self._check_error("if 0: yield\nelse: x=1", "outside function") @@ -1970,22 +2380,28 @@ def test_return_outside_function(self): "outside function") def test_break_outside_loop(self): - self._check_error("if 0: break", "outside loop") - self._check_error("if 0: break\nelse: x=1", "outside loop") - self._check_error("if 1: pass\nelse: break", "outside loop") - self._check_error("class C:\n if 0: break", "outside loop") + msg = "outside loop" + self._check_error("break", msg, lineno=1) + self._check_error("if 0: break", msg, lineno=1) + self._check_error("if 0: break\nelse: x=1", msg, lineno=1) + self._check_error("if 1: pass\nelse: break", msg, lineno=2) + self._check_error("class C:\n if 0: break", msg, lineno=2) self._check_error("class C:\n if 1: pass\n else: break", - "outside loop") + msg, lineno=3) + self._check_error("with object() as obj:\n break", + msg, lineno=2) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_continue_outside_loop(self): - self._check_error("if 0: continue", "not properly in loop") - self._check_error("if 0: continue\nelse: x=1", "not properly in loop") - self._check_error("if 1: pass\nelse: continue", "not properly in loop") - self._check_error("class C:\n if 0: continue", "not properly in loop") + msg = "not properly in loop" + self._check_error("if 0: continue", msg, lineno=1) + self._check_error("if 0: continue\nelse: x=1", msg, lineno=1) + self._check_error("if 1: pass\nelse: continue", msg, lineno=2) + self._check_error("class C:\n if 0: continue", msg, lineno=2) self._check_error("class C:\n if 1: pass\n else: continue", - "not properly in loop") + msg, lineno=3) + self._check_error("with object() as obj:\n continue", + msg, lineno=2) def test_unexpected_indent(self): self._check_error("foo()\n bar()\n", "unexpected indent", @@ -2000,35 +2416,42 @@ def test_bad_outdent(self): "unindent does not match .* level", subclass=IndentationError) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_kwargs_last(self): self._check_error("int(base=10, '2')", "positional argument follows keyword argument") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_kwargs_last2(self): self._check_error("int(**{'base': 10}, '2')", "positional argument follows " "keyword argument unpacking") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_kwargs_last3(self): self._check_error("int(**{'base': 10}, *['2'])", "iterable argument unpacking follows " "keyword argument unpacking") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_generator_in_function_call(self): self._check_error("foo(x, y for y in range(3) for z in range(2) if z , p)", "Generator expression must be parenthesized", lineno=1, end_lineno=1, offset=11, end_offset=53) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_except_then_except_star(self): + self._check_error("try: pass\nexcept ValueError: pass\nexcept* TypeError: pass", + r"cannot have both 'except' and 'except\*' on the same 'try'", + lineno=3, end_lineno=3, offset=1, end_offset=8) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_except_star_then_except(self): + self._check_error("try: pass\nexcept* ValueError: pass\nexcept TypeError: pass", + r"cannot have both 'except' and 'except\*' on the same 'try'", + lineno=3, end_lineno=3, offset=1, end_offset=7) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_empty_line_after_linecont(self): # See issue-40847 s = r"""\ @@ -2073,6 +2496,25 @@ def test_continuation_bad_indentation(self): self.assertRaises(IndentationError, exec, code) + @support.cpython_only + def test_disallowed_type_param_names(self): + # See gh-128632 + + self._check_error(f"class A[__classdict__]: pass", + f"reserved name '__classdict__' cannot be used for type parameter") + self._check_error(f"def f[__classdict__](): pass", + f"reserved name '__classdict__' cannot be used for type parameter") + self._check_error(f"type T[__classdict__] = tuple[__classdict__]", + f"reserved name '__classdict__' cannot be used for type parameter") + + # These compilations are here to make sure __class__, __classcell__ and __classdictcell__ + # don't break in the future like __classdict__ did in this case. + for name in ('__class__', '__classcell__', '__classdictcell__'): + compile(f""" +class A: + class B[{name}]: pass + """, "<testcase>", mode="exec") + @support.cpython_only def test_nested_named_except_blocks(self): code = "" @@ -2083,6 +2525,58 @@ def test_nested_named_except_blocks(self): code += f"{' '*4*12}pass" self._check_error(code, "too many statically nested blocks") + @support.cpython_only + def test_with_statement_many_context_managers(self): + # See gh-113297 + + def get_code(n): + code = textwrap.dedent(""" + def bug(): + with ( + a + """) + for i in range(n): + code += f" as a{i}, a\n" + code += "): yield a" + return code + + CO_MAXBLOCKS = 21 # static nesting limit of the compiler + MAX_MANAGERS = CO_MAXBLOCKS - 1 # One for the StopIteration block + + for n in range(MAX_MANAGERS): + with self.subTest(f"within range: {n=}"): + compile(get_code(n), "<string>", "exec") + + for n in range(MAX_MANAGERS, MAX_MANAGERS + 5): + with self.subTest(f"out of range: {n=}"): + self._check_error(get_code(n), "too many statically nested blocks") + + @support.cpython_only + def test_async_with_statement_many_context_managers(self): + # See gh-116767 + + def get_code(n): + code = [ textwrap.dedent(""" + async def bug(): + async with ( + a + """) ] + for i in range(n): + code.append(f" as a{i}, a\n") + code.append("): yield a") + return "".join(code) + + CO_MAXBLOCKS = 21 # static nesting limit of the compiler + MAX_MANAGERS = CO_MAXBLOCKS - 1 # One for the StopIteration block + + for n in range(MAX_MANAGERS): + with self.subTest(f"within range: {n=}"): + compile(get_code(n), "<string>", "exec") + + for n in range(MAX_MANAGERS, MAX_MANAGERS + 5): + with self.subTest(f"out of range: {n=}"): + self._check_error(get_code(n), "too many statically nested blocks") + def test_barry_as_flufl_with_syntax_errors(self): # The "barry_as_flufl" rule can produce some "bugs-at-a-distance" if # is reading the wrong token in the presence of syntax errors later @@ -2100,8 +2594,7 @@ def func2(): """ self._check_error(code, "expected ':'") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid_line_continuation_error_position(self): self._check_error(r"a = 3 \ 4", "unexpected character after line continuation character", @@ -2113,8 +2606,7 @@ def test_invalid_line_continuation_error_position(self): "unexpected character after line continuation character", lineno=3, offset=4) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid_line_continuation_left_recursive(self): # Check bpo-42218: SyntaxErrors following left-recursive rules # (t_primary_raw in this case) need to be tested explicitly @@ -2123,8 +2615,7 @@ def test_invalid_line_continuation_left_recursive(self): self._check_error("A.\u03bc\\\n", "unexpected EOF while parsing") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_error_parenthesis(self): for paren in "([{": self._check_error(paren + "1 + 2", f"\\{paren}' was never closed") @@ -2135,10 +2626,39 @@ def test_error_parenthesis(self): for paren in ")]}": self._check_error(paren + "1 + 2", f"unmatched '\\{paren}'") - # TODO: RUSTPYTHON - @unittest.expectedFailure + # Some more complex examples: + code = """\ +func( + a=["unclosed], # Need a quote in this comment: " + b=2, +) +""" + self._check_error(code, "parenthesis '\\)' does not match opening parenthesis '\\['") + + self._check_error("match y:\n case e(e=v,v,", " was never closed") + + # Examples with dencodings + s = b'# coding=latin\n(aaaaaaaaaaaaaaaaa\naaaaaaaaaaa\xb5' + self._check_error(s, r"'\(' was never closed") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_error_string_literal(self): + + self._check_error("'blech", r"unterminated string literal \(.*\)$") + self._check_error('"blech', r"unterminated string literal \(.*\)$") + self._check_error( + r'"blech\"', r"unterminated string literal \(.*\); perhaps you escaped the end quote" + ) + self._check_error( + r'r"blech\"', r"unterminated string literal \(.*\); perhaps you escaped the end quote" + ) + self._check_error("'''blech", "unterminated triple-quoted string literal") + self._check_error('"""blech', "unterminated triple-quoted string literal") + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invisible_characters(self): self._check_error('print\x17("Hello")', "invalid non-printable character") + self._check_error(b"with(0,,):\n\x01", "invalid non-printable character") def test_match_call_does_not_raise_syntax_error(self): code = """ @@ -2158,8 +2678,7 @@ def case(x): """ compile(code, "<string>", "exec") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_multiline_compiler_error_points_to_the_end(self): self._check_error( "call(\na=1,\na=1\n)", @@ -2198,7 +2717,8 @@ def test_syntax_error_on_deeply_nested_blocks(self): while 20: while 21: while 22: - break + while 23: + break """ self._check_error(source, "too many statically nested blocks") @@ -2207,7 +2727,7 @@ def test_error_on_parser_stack_overflow(self): source = "-" * 100000 + "4" for mode in ["exec", "eval", "single"]: with self.subTest(mode=mode): - with self.assertRaises(MemoryError): + with self.assertRaisesRegex(MemoryError, r"too complex"): compile(source, "<string>", mode) @support.cpython_only diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 6db1442980..da242c7df7 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -1,24 +1,35 @@ +import platform +import re import unittest import sys import os import subprocess import shutil +import json +import textwrap from copy import copy from test.support import ( - captured_stdout, PythonSymlink, requires_subprocess, is_wasi + captured_stdout, + is_apple_mobile, + is_wasi, + PythonSymlink, + requires_subprocess, ) from test.support.import_helper import import_module from test.support.os_helper import (TESTFN, unlink, skip_unless_symlink, change_cwd) -from test.support.warnings_helper import check_warnings +from test.support.venv import VirtualEnvironment import sysconfig from sysconfig import (get_paths, get_platform, get_config_vars, get_path, get_path_names, _INSTALL_SCHEMES, get_default_scheme, get_scheme_names, get_config_var, - _expand_vars, _get_preferred_schemes, _main) + _expand_vars, _get_preferred_schemes) +from sysconfig.__main__ import _main, _parse_makefile +import _imp import _osx_support +import _sysconfig HAS_USER_BASE = sysconfig._HAS_USER_BASE @@ -41,6 +52,7 @@ def setUp(self): self.name = os.name self.platform = sys.platform self.version = sys.version + self._framework = sys._framework self.sep = os.sep self.join = os.path.join self.isabs = os.path.isabs @@ -64,6 +76,7 @@ def tearDown(self): os.name = self.name sys.platform = self.platform sys.version = self.version + sys._framework = self._framework os.sep = self.sep os.path.join = self.join os.path.isabs = self.isabs @@ -91,6 +104,12 @@ def _cleanup_testfn(self): elif os.path.isdir(path): shutil.rmtree(path) + def venv(self, **venv_create_args): + return VirtualEnvironment.from_tmpdir( + prefix=f'{self.id()}-venv-', + **venv_create_args, + ) + def test_get_path_names(self): self.assertEqual(get_path_names(), sysconfig._SCHEME_KEYS) @@ -137,35 +156,37 @@ def test_get_preferred_schemes(self): # Mac, framework build. os.name = 'posix' sys.platform = 'darwin' - sys._framework = True + sys._framework = "MyPython" self.assertIsInstance(schemes, dict) self.assertEqual(set(schemes), expected_schemes) - # NOTE: RUSTPYTHON this is hardcoded to 'python', we're set up for failure. - @unittest.expectedFailure def test_posix_venv_scheme(self): # The following directories were hardcoded in the venv module # before bpo-45413, here we assert the posix_venv scheme does not regress binpath = 'bin' incpath = 'include' libpath = os.path.join('lib', - 'python%d.%d' % sys.version_info[:2], + # XXX: RUSTPYTHON + f'rustpython{sysconfig._get_python_version_abi()}', 'site-packages') - # Resolve the paths in prefix - binpath = os.path.join(sys.prefix, binpath) - incpath = os.path.join(sys.prefix, incpath) - libpath = os.path.join(sys.prefix, libpath) + # Resolve the paths in an imaginary venv/ directory + binpath = os.path.join('venv', binpath) + incpath = os.path.join('venv', incpath) + libpath = os.path.join('venv', libpath) + + # Mimic the venv module, set all bases to the venv directory + bases = ('base', 'platbase', 'installed_base', 'installed_platbase') + vars = {base: 'venv' for base in bases} - self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='posix_venv')) - self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='posix_venv')) + self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='posix_venv', vars=vars)) + self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='posix_venv', vars=vars)) # The include directory on POSIX isn't exactly the same as before, # but it is "within" - sysconfig_includedir = sysconfig.get_path('include', scheme='posix_venv') + sysconfig_includedir = sysconfig.get_path('include', scheme='posix_venv', vars=vars) self.assertTrue(sysconfig_includedir.startswith(incpath + os.sep)) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_nt_venv_scheme(self): # The following directories were hardcoded in the venv module # before bpo-45413, here we assert the posix_venv scheme does not regress @@ -173,14 +194,19 @@ def test_nt_venv_scheme(self): incpath = 'Include' libpath = os.path.join('Lib', 'site-packages') - # Resolve the paths in prefix - binpath = os.path.join(sys.prefix, binpath) - incpath = os.path.join(sys.prefix, incpath) - libpath = os.path.join(sys.prefix, libpath) + # Resolve the paths in an imaginary venv\ directory + venv = 'venv' + binpath = os.path.join(venv, binpath) + incpath = os.path.join(venv, incpath) + libpath = os.path.join(venv, libpath) - self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='nt_venv')) - self.assertEqual(incpath, sysconfig.get_path('include', scheme='nt_venv')) - self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='nt_venv')) + # Mimic the venv module, set all bases to the venv directory + bases = ('base', 'platbase', 'installed_base', 'installed_platbase') + vars = {base: 'venv' for base in bases} + + self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='nt_venv', vars=vars)) + self.assertEqual(incpath, sysconfig.get_path('include', scheme='nt_venv', vars=vars)) + self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='nt_venv', vars=vars)) def test_venv_scheme(self): if sys.platform == 'win32': @@ -216,6 +242,11 @@ def test_get_config_vars(self): self.assertTrue(cvars) def test_get_platform(self): + # Check the actual platform returns something reasonable. + actual_platform = get_platform() + self.assertIsInstance(actual_platform, str) + self.assertTrue(actual_platform) + # windows XP, 32bits os.name = 'nt' sys.version = ('2.4.4 (#71, Oct 18 2006, 08:34:43) ' @@ -331,11 +362,28 @@ def test_get_platform(self): self.assertEqual(get_platform(), 'linux-i686') + # Android + os.name = 'posix' + sys.platform = 'android' + get_config_vars()['ANDROID_API_LEVEL'] = 9 + for machine, abi in { + 'x86_64': 'x86_64', + 'i686': 'x86', + 'aarch64': 'arm64_v8a', + 'armv7l': 'armeabi_v7a', + }.items(): + with self.subTest(machine): + self._set_uname(('Linux', 'localhost', '3.18.91+', + '#1 Tue Jan 9 20:35:43 UTC 2018', machine)) + self.assertEqual(get_platform(), f'android-9-{abi}') + # XXX more platforms to tests here # TODO: RUSTPYTHON @unittest.expectedFailure @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds") + @unittest.skipIf(is_apple_mobile, + f"{sys.platform} doesn't distribute header files in the runtime environment") def test_get_config_h_filename(self): config_h = sysconfig.get_config_h_filename() self.assertTrue(os.path.isfile(config_h), config_h) @@ -381,8 +429,8 @@ def test_user_similar(self): if name == 'platlib': # Replace "/lib64/python3.11/site-packages" suffix # with "/lib/python3.11/site-packages". - py_version_short = sysconfig.get_python_version() - suffix = f'python{py_version_short}/site-packages' + py_version_abi = sysconfig._get_python_version_abi() + suffix = f'python{py_version_abi}/site-packages' expected = expected.replace(f'/{sys.platlibdir}/{suffix}', f'/lib/{suffix}') self.assertEqual(user_path, expected) @@ -402,6 +450,32 @@ def test_ldshared_value(self): self.assertIn(ldflags, ldshared) + @unittest.skipIf(not _imp.extension_suffixes(), "stub loader has no suffixes") + def test_soabi(self): + soabi = sysconfig.get_config_var('SOABI') + self.assertIn(soabi, _imp.extension_suffixes()[0]) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_library(self): + library = sysconfig.get_config_var('LIBRARY') + ldlibrary = sysconfig.get_config_var('LDLIBRARY') + major, minor = sys.version_info[:2] + if sys.platform == 'win32': + self.assertTrue(library.startswith(f'python{major}{minor}')) + self.assertTrue(library.endswith('.dll')) + self.assertEqual(library, ldlibrary) + elif is_apple_mobile: + framework = sysconfig.get_config_var('PYTHONFRAMEWORK') + self.assertEqual(ldlibrary, f"{framework}.framework/{framework}") + else: + self.assertTrue(library.startswith(f'libpython{major}.{minor}')) + self.assertTrue(library.endswith('.a')) + if sys.platform == 'darwin' and sys._framework: + self.skipTest('gh-110824: skip LDLIBRARY test for framework build') + else: + self.assertTrue(ldlibrary.startswith(f'libpython{major}.{minor}')) + @unittest.skipUnless(sys.platform == "darwin", "test only relevant on MacOSX") @requires_subprocess() def test_platform_in_subprocess(self): @@ -448,6 +522,8 @@ def test_platform_in_subprocess(self): @unittest.expectedFailureIf(sys.platform != "win32", "TODO: RUSTPYTHON") @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds") + @unittest.skipIf(is_apple_mobile, + f"{sys.platform} doesn't include config folder at runtime") def test_srcdir(self): # See Issues #15322, #15364. srcdir = sysconfig.get_config_var('srcdir') @@ -460,11 +536,15 @@ def test_srcdir(self): # should be a full source checkout. Python_h = os.path.join(srcdir, 'Include', 'Python.h') self.assertTrue(os.path.exists(Python_h), Python_h) - # <srcdir>/PC/pyconfig.h always exists even if unused on POSIX. - pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h') + # <srcdir>/PC/pyconfig.h.in always exists even if unused + pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h.in') self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h) pyconfig_h_in = os.path.join(srcdir, 'pyconfig.h.in') self.assertTrue(os.path.exists(pyconfig_h_in), pyconfig_h_in) + if os.name == 'nt': + # <executable dir>/pyconfig.h exists on Windows in a build tree + pyconfig_h = os.path.join(sys.executable, '..', 'pyconfig.h') + self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h) elif os.name == 'posix': makefile_dir = os.path.dirname(sysconfig.get_makefile_filename()) # Issue #19340: srcdir has been realpath'ed already @@ -479,23 +559,16 @@ def test_srcdir_independent_of_cwd(self): srcdir2 = sysconfig.get_config_var('srcdir') self.assertEqual(srcdir, srcdir2) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(sysconfig.get_config_var('EXT_SUFFIX') is None, 'EXT_SUFFIX required for this test') + @unittest.skipIf(not _imp.extension_suffixes(), "stub loader has no suffixes") def test_EXT_SUFFIX_in_vars(self): - import _imp - if not _imp.extension_suffixes(): - self.skipTest("stub loader has no suffixes") vars = sysconfig.get_config_vars() self.assertEqual(vars['EXT_SUFFIX'], _imp.extension_suffixes()[0]) - @unittest.skipUnless(sys.platform == 'linux' and - hasattr(sys.implementation, '_multiarch'), - 'multiarch-specific test') - def test_triplet_in_ext_suffix(self): + @unittest.skipUnless(sys.platform == 'linux', 'Linux-specific test') + def test_linux_ext_suffix(self): ctypes = import_module('ctypes') - import platform, re machine = platform.machine() suffix = sysconfig.get_config_var('EXT_SUFFIX') if re.match('(aarch64|arm|mips|ppc|powerpc|s390|sparc)', machine): @@ -510,11 +583,97 @@ def test_triplet_in_ext_suffix(self): # TODO: RUSTPYTHON @unittest.expectedFailure + @unittest.skipUnless(sys.platform == 'android', 'Android-specific test') + def test_android_ext_suffix(self): + machine = platform.machine() + suffix = sysconfig.get_config_var('EXT_SUFFIX') + expected_triplet = { + "x86_64": "x86_64-linux-android", + "i686": "i686-linux-android", + "aarch64": "aarch64-linux-android", + "armv7l": "arm-linux-androideabi", + }[machine] + self.assertTrue(suffix.endswith(f"-{expected_triplet}.so"), + f"{machine=}, {suffix=}") + + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(sys.platform == 'darwin', 'OS X-specific test') def test_osx_ext_suffix(self): suffix = sysconfig.get_config_var('EXT_SUFFIX') self.assertTrue(suffix.endswith('-darwin.so'), suffix) + # TODO: RUSTPYTHON + @unittest.expectedFailure + @requires_subprocess() + def test_config_vars_depend_on_site_initialization(self): + script = textwrap.dedent(""" + import sysconfig + + config_vars = sysconfig.get_config_vars() + + import json + print(json.dumps(config_vars, indent=2)) + """) + + with self.venv() as venv: + site_config_vars = json.loads(venv.run('-c', script).stdout) + no_site_config_vars = json.loads(venv.run('-S', '-c', script).stdout) + + self.assertNotEqual(site_config_vars, no_site_config_vars) + # With the site initialization, the virtual environment should be enabled. + self.assertEqual(site_config_vars['base'], venv.prefix) + self.assertEqual(site_config_vars['platbase'], venv.prefix) + #self.assertEqual(site_config_vars['prefix'], venv.prefix) # # FIXME: prefix gets overwriten by _init_posix + # Without the site initialization, the virtual environment should be disabled. + self.assertEqual(no_site_config_vars['base'], site_config_vars['installed_base']) + self.assertEqual(no_site_config_vars['platbase'], site_config_vars['installed_platbase']) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + @requires_subprocess() + def test_config_vars_recalculation_after_site_initialization(self): + script = textwrap.dedent(""" + import sysconfig + + before = sysconfig.get_config_vars() + + import site + site.main() + + after = sysconfig.get_config_vars() + + import json + print(json.dumps({'before': before, 'after': after}, indent=2)) + """) + + with self.venv() as venv: + config_vars = json.loads(venv.run('-S', '-c', script).stdout) + + self.assertNotEqual(config_vars['before'], config_vars['after']) + self.assertEqual(config_vars['after']['base'], venv.prefix) + #self.assertEqual(config_vars['after']['prefix'], venv.prefix) # FIXME: prefix gets overwriten by _init_posix + #self.assertEqual(config_vars['after']['exec_prefix'], venv.prefix) # FIXME: exec_prefix gets overwriten by _init_posix + + # TODO: RUSTPYTHON + @unittest.expectedFailure + @requires_subprocess() + def test_paths_depend_on_site_initialization(self): + script = textwrap.dedent(""" + import sysconfig + + paths = sysconfig.get_paths() + + import json + print(json.dumps(paths, indent=2)) + """) + + with self.venv() as venv: + site_paths = json.loads(venv.run('-c', script).stdout) + no_site_paths = json.loads(venv.run('-S', '-c', script).stdout) + + self.assertNotEqual(site_paths, no_site_paths) + + class MakefileTests(unittest.TestCase): # TODO: RUSTPYTHON @@ -522,6 +681,8 @@ class MakefileTests(unittest.TestCase): @unittest.skipIf(sys.platform.startswith('win'), 'Test is not Windows compatible') @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds") + @unittest.skipIf(is_apple_mobile, + f"{sys.platform} doesn't include config folder at runtime") def test_get_makefile_filename(self): makefile = sysconfig.get_makefile_filename() self.assertTrue(os.path.isfile(makefile), makefile) @@ -536,7 +697,7 @@ def test_parse_makefile(self): print("var5=dollar$$5", file=makefile) print("var6=${var3}/lib/python3.5/config-$(VAR2)$(var5)" "-x86_64-linux-gnu", file=makefile) - vars = sysconfig._parse_makefile(TESTFN) + vars = _parse_makefile(TESTFN) self.assertEqual(vars, { 'var1': 'ab42', 'VAR2': 'b42', @@ -547,5 +708,26 @@ def test_parse_makefile(self): }) +class DeprecationTests(unittest.TestCase): + def deprecated(self, removal_version, deprecation_msg=None, error=Exception, error_msg=None): + if sys.version_info >= removal_version: + return self.assertRaises(error, msg=error_msg) + else: + return self.assertWarns(DeprecationWarning, msg=deprecation_msg) + + def test_is_python_build_check_home(self): + with self.deprecated( + removal_version=(3, 15), + deprecation_msg=( + 'The check_home argument of sysconfig.is_python_build is ' + 'deprecated and its value is ignored. ' + 'It will be removed in Python 3.15.' + ), + error=TypeError, + error_msg="is_python_build() takes 0 positional arguments but 1 were given", + ): + sysconfig.is_python_build('foo') + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 0fed89c773..61929e537f 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1276,7 +1276,6 @@ def test_gettarinfo_pathlike_name(self): self.assertEqual(tarinfo.name, tarinfo2.name) self.assertEqual(tarinfo.size, 3) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @unittest.skipUnless(hasattr(os, "link"), "Missing hardlink implementation") def test_link_size(self): @@ -1301,7 +1300,6 @@ def test_link_size(self): os_helper.unlink(target) os_helper.unlink(link) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @os_helper.skip_unless_symlink def test_symlink_size(self): path = os.path.join(TEMPDIR, "symlink") @@ -1849,7 +1847,6 @@ def test_add_twice(self): self.assertEqual(tarinfo.type, tarfile.REGTYPE, "add file as regular failed") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_add_hardlink(self): tarinfo = self.tar.gettarinfo(self.bar) self.assertEqual(tarinfo.type, tarfile.LNKTYPE, diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 069221ae47..4aa15f6993 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -1,6 +1,7 @@ import unittest from test import support from test.support import import_helper +from test.support.script_helper import assert_python_ok import builtins import contextlib import copy @@ -32,8 +33,7 @@ def get_command_stdout(command, args): class BaseTestUUID: uuid = None - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_safe_uuid_enum(self): class CheckedSafeUUID(enum.Enum): safe = 0 @@ -775,10 +775,37 @@ def test_cli_uuid5_ouputted_with_valid_namespace_and_name(self): class TestUUIDWithoutExtModule(BaseTestUUID, unittest.TestCase): uuid = py_uuid + @unittest.skipUnless(c_uuid, 'requires the C _uuid module') class TestUUIDWithExtModule(BaseTestUUID, unittest.TestCase): uuid = c_uuid + def check_has_stable_libuuid_extractable_node(self): + if not self.uuid._has_stable_extractable_node: + self.skipTest("libuuid cannot deduce MAC address") + + @unittest.skipUnless(os.name == 'posix', 'POSIX only') + def test_unix_getnode_from_libuuid(self): + self.check_has_stable_libuuid_extractable_node() + script = 'import uuid; print(uuid._unix_getnode())' + _, n_a, _ = assert_python_ok('-c', script) + _, n_b, _ = assert_python_ok('-c', script) + n_a, n_b = n_a.decode().strip(), n_b.decode().strip() + self.assertTrue(n_a.isdigit()) + self.assertTrue(n_b.isdigit()) + self.assertEqual(n_a, n_b) + + @unittest.skipUnless(os.name == 'nt', 'Windows only') + def test_windows_getnode_from_libuuid(self): + self.check_has_stable_libuuid_extractable_node() + script = 'import uuid; print(uuid._windll_getnode())' + _, n_a, _ = assert_python_ok('-c', script) + _, n_b, _ = assert_python_ok('-c', script) + n_a, n_b = n_a.decode().strip(), n_b.decode().strip() + self.assertTrue(n_a.isdigit()) + self.assertTrue(n_b.isdigit()) + self.assertEqual(n_a, n_b) + class BaseTestInternals: _uuid = py_uuid diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 59b5515529..57b082fc58 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -339,8 +339,7 @@ def test_set_attribute(self): element.attrib = {'A': 'B', 'C': 'D'} self.assertEqual(element.attrib, {'A': 'B', 'C': 'D'}) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_simpleops(self): # Basic method sanity checks. @@ -385,8 +384,7 @@ def test_simpleops(self): self.serialize_check(element, '<tag key="value"><subtag /><subtag /></tag>') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cdata(self): # Test CDATA handling (etc). @@ -397,8 +395,7 @@ def test_cdata(self): self.serialize_check(ET.XML("<tag><![CDATA[hello]]></tag>"), '<tag>hello</tag>') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_file_init(self): stringfile = io.BytesIO(SAMPLE_XML.encode("utf-8")) tree = ET.ElementTree(file=stringfile) @@ -410,8 +407,7 @@ def test_file_init(self): self.assertEqual(tree.find("element/../empty-element").tag, 'empty-element') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_path_cache(self): # Check that the path cache behaves sanely. @@ -428,8 +424,7 @@ def test_path_cache(self): for i in range(600): ET.ElementTree(elem).find('./'+str(i)) self.assertLess(len(ElementPath._cache), 500) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_copy(self): # Test copy handling (etc). @@ -442,8 +437,7 @@ def test_copy(self): self.serialize_check(e2, '<tag>hello<bar /></tag>') self.serialize_check(e3, '<tag>hello<foo /></tag>') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_attrib(self): # Test attribute handling. @@ -520,8 +514,7 @@ def test_makeelement(self): elem[:] = tuple([subelem]) self.serialize_check(elem, '<tag><subtag key="value" /></tag>') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_parsefile(self): # Test parsing from file. @@ -567,8 +560,7 @@ def test_parsefile(self): ' <empty-element />\n' '</root>') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_parseliteral(self): element = ET.XML("<html><body>text</body></html>") self.assertEqual(ET.tostring(element, encoding='unicode'), @@ -591,8 +583,7 @@ def test_parseliteral(self): self.assertEqual(len(ids), 1) self.assertEqual(ids["body"].tag, 'body') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_iterparse(self): # Test iterparse interface. @@ -744,8 +735,7 @@ def test_iterparse(self): with self.assertRaises(FileNotFoundError): iterparse("nonexistent") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_iterparse_close(self): iterparse = ET.iterparse @@ -814,8 +804,7 @@ def test_writefile(self): elem[0] = ET.PI("key", "value") self.serialize_check(elem, 'text<?key value?><subtag>subtext</subtag>') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_custom_builder(self): # Test parser w. custom builder. @@ -877,8 +866,7 @@ def end_ns(self, prefix): ('end-ns', ''), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_custom_builder_only_end_ns(self): class Builder(list): def end_ns(self, prefix): @@ -901,8 +889,7 @@ def end_ns(self, prefix): ('end-ns', ''), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_initialize_parser_without_target(self): # Explicit None parser = ET.XMLParser(target=None) @@ -912,8 +899,7 @@ def test_initialize_parser_without_target(self): parser2 = ET.XMLParser() self.assertIsInstance(parser2.target, ET.TreeBuilder) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_children(self): # Test Element children iteration @@ -951,16 +937,14 @@ def test_children(self): elem.clear() self.assertEqual(list(elem), []) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_writestring(self): elem = ET.XML("<html><body>text</body></html>") self.assertEqual(ET.tostring(elem), b'<html><body>text</body></html>') elem = ET.fromstring("<html><body>text</body></html>") self.assertEqual(ET.tostring(elem), b'<html><body>text</body></html>') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_indent(self): elem = ET.XML("<root></root>") ET.indent(elem) @@ -1005,8 +989,7 @@ def test_indent(self): b'</html>' ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_indent_space(self): elem = ET.XML("<html><body><p>pre<br/>post</p><p>text</p></body></html>") ET.indent(elem, space='\t') @@ -1032,8 +1015,7 @@ def test_indent_space(self): b'</html>' ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_indent_space_caching(self): elem = ET.XML("<html><body><p>par</p><p>text</p><p><br/></p><p /></body></html>") ET.indent(elem) @@ -1050,8 +1032,7 @@ def test_indent_space_caching(self): len({id(el.tail) for el in elem.iter()}), ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_indent_level(self): elem = ET.XML("<html><body><p>pre<br/>post</p><p>text</p></body></html>") with self.assertRaises(ValueError): @@ -1084,8 +1065,7 @@ def test_indent_level(self): b' </html>' ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostring_default_namespace(self): elem = ET.XML('<body xmlns="http://effbot.org/ns"><tag/></body>') self.assertEqual( @@ -1097,8 +1077,7 @@ def test_tostring_default_namespace(self): '<body xmlns="http://effbot.org/ns"><tag /></body>' ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostring_default_namespace_different_namespace(self): elem = ET.XML('<body xmlns="http://effbot.org/ns"><tag/></body>') self.assertEqual( @@ -1106,16 +1085,14 @@ def test_tostring_default_namespace_different_namespace(self): '<ns1:body xmlns="foobar" xmlns:ns1="http://effbot.org/ns"><ns1:tag /></ns1:body>' ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostring_default_namespace_original_no_namespace(self): elem = ET.XML('<body><tag/></body>') EXPECTED_MSG = '^cannot use non-qualified names with default_namespace option$' with self.assertRaisesRegex(ValueError, EXPECTED_MSG): ET.tostring(elem, encoding='unicode', default_namespace='foobar') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostring_no_xml_declaration(self): elem = ET.XML('<body><tag/></body>') self.assertEqual( @@ -1123,8 +1100,7 @@ def test_tostring_no_xml_declaration(self): '<body><tag /></body>' ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostring_xml_declaration(self): elem = ET.XML('<body><tag/></body>') self.assertEqual( @@ -1132,8 +1108,7 @@ def test_tostring_xml_declaration(self): b"<?xml version='1.0' encoding='utf8'?>\n<body><tag /></body>" ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostring_xml_declaration_unicode_encoding(self): elem = ET.XML('<body><tag/></body>') self.assertEqual( @@ -1141,8 +1116,7 @@ def test_tostring_xml_declaration_unicode_encoding(self): "<?xml version='1.0' encoding='utf-8'?>\n<body><tag /></body>" ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostring_xml_declaration_cases(self): elem = ET.XML('<body><tag>ø</tag></body>') TESTCASES = [ @@ -1187,8 +1161,7 @@ def test_tostring_xml_declaration_cases(self): expected_retval ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostringlist_default_namespace(self): elem = ET.XML('<body xmlns="http://effbot.org/ns"><tag/></body>') self.assertEqual( @@ -1200,8 +1173,7 @@ def test_tostringlist_default_namespace(self): '<body xmlns="http://effbot.org/ns"><tag /></body>' ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostringlist_xml_declaration(self): elem = ET.XML('<body><tag/></body>') self.assertEqual( @@ -1221,8 +1193,7 @@ def test_tostringlist_xml_declaration(self): self.assertRegex(stringlist[0], r"^<\?xml version='1.0' encoding='.+'?>") self.assertEqual(['<body', '>', '<tag', ' />', '</body>'], stringlist[1:]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_encoding(self): def check(encoding, body=''): xml = ("<?xml version='1.0' encoding='%s'?><xml>%s</xml>" % @@ -1282,8 +1253,7 @@ def bxml(encoding): self.assertRaises(ValueError, ET.XML, xml('undefined').encode('ascii')) self.assertRaises(LookupError, ET.XML, xml('xxx').encode('ascii')) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_methods(self): # Test serialization methods. @@ -1299,8 +1269,7 @@ def test_methods(self): '<html><link><script>1 < 2</script></html>\n') self.assertEqual(serialize(e, method="text"), '1 < 2\n') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_issue18347(self): e = ET.XML('<html><CamelCase>text</CamelCase></html>') self.assertEqual(serialize(e), @@ -1308,8 +1277,7 @@ def test_issue18347(self): self.assertEqual(serialize(e, method="html"), '<html><CamelCase>text</CamelCase></html>') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_entity(self): # Test entity handling. @@ -1347,8 +1315,7 @@ def test_entity(self): self.assertEqual(str(cm.exception), 'undefined entity &entity;: line 4, column 10') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_namespace(self): # Test namespace issues. @@ -1447,8 +1414,7 @@ def test_qname(self): self.assertNotEqual(q1, 'ns:tag') self.assertEqual(q1, '{ns}tag') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_doctype_public(self): # Test PUBLIC doctype. @@ -1515,8 +1481,7 @@ def check(p, expected, namespaces=None): {'': 'http://www.w3.org/2001/XMLSchema', 'ns': 'http://www.w3.org/2001/XMLSchema'}) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_processinginstruction(self): # Test ProcessingInstruction directly @@ -1533,8 +1498,7 @@ def test_processinginstruction(self): b"<?xml version='1.0' encoding='latin-1'?>\n" b"<?test <testing&>\xe3?>") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_html_empty_elems_serialization(self): # issue 15970 # from http://www.w3.org/TR/html401/index/elements.html @@ -1565,8 +1529,7 @@ def test_tree_write_attribute_order(self): self.assertEqual(serialize(root, method='html'), '<cirriculum status="public" company="example"></cirriculum>') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_attlist_default(self): # Test default attribute values; See BPO 42151. root = ET.fromstring(ATTLIST_XML) @@ -1601,8 +1564,7 @@ def assert_event_tags(self, parser, expected, max_events=None): self.assertEqual([(action, elem.tag) for action, elem in events], expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_simple_xml(self, chunk_size=None, flush=False): parser = ET.XMLPullParser() self.assert_event_tags(parser, []) @@ -1624,23 +1586,19 @@ def test_simple_xml(self, chunk_size=None, flush=False): self.assert_event_tags(parser, [('end', 'root')]) self.assertIsNone(parser.close()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_simple_xml_chunk_1(self): self.test_simple_xml(chunk_size=1, flush=True) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_simple_xml_chunk_5(self): self.test_simple_xml(chunk_size=5, flush=True) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_simple_xml_chunk_22(self): self.test_simple_xml(chunk_size=22) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_feed_while_iterating(self): parser = ET.XMLPullParser() it = parser.read_events() @@ -1653,8 +1611,7 @@ def test_feed_while_iterating(self): with self.assertRaises(StopIteration): next(it) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_simple_xml_with_ns(self): parser = ET.XMLPullParser() self.assert_event_tags(parser, []) @@ -1676,8 +1633,7 @@ def test_simple_xml_with_ns(self): self.assert_event_tags(parser, [('end', '{namespace}root')]) self.assertIsNone(parser.close()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_ns_events(self): parser = ET.XMLPullParser(events=('start-ns', 'end-ns')) self._feed(parser, "<!-- comment -->\n") @@ -1693,8 +1649,7 @@ def test_ns_events(self): self.assertEqual(list(parser.read_events()), [('end-ns', None)]) self.assertIsNone(parser.close()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_ns_events_start(self): parser = ET.XMLPullParser(events=('start-ns', 'start', 'end')) self._feed(parser, "<tag xmlns='abc' xmlns:p='xyz'>\n") @@ -1718,8 +1673,7 @@ def test_ns_events_start(self): ('end', '{abc}tag'), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_ns_events_start_end(self): parser = ET.XMLPullParser(events=('start-ns', 'start', 'end', 'end-ns')) self._feed(parser, "<tag xmlns='abc' xmlns:p='xyz'>\n") @@ -1747,8 +1701,7 @@ def test_ns_events_start_end(self): ('end-ns', None), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_events(self): parser = ET.XMLPullParser(events=()) self._feed(parser, "<root/>\n") @@ -1795,8 +1748,7 @@ def test_events(self): self._feed(parser, "</root>") self.assertIsNone(parser.close()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_events_comment(self): parser = ET.XMLPullParser(events=('start', 'comment', 'end')) self._feed(parser, "<!-- text here -->\n") @@ -1816,8 +1768,7 @@ def test_events_comment(self): self._feed(parser, "<!-- text here -->\n") self.assert_events(parser, [('comment', (ET.Comment, ' text here '))]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_events_pi(self): parser = ET.XMLPullParser(events=('start', 'pi', 'end')) self._feed(parser, "<?pitarget?>\n") @@ -1826,8 +1777,7 @@ def test_events_pi(self): self._feed(parser, "<?pitarget some text ?>\n") self.assert_events(parser, [('pi', (ET.PI, 'pitarget some text '))]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_events_sequence(self): # Test that events can be some sequence that's not just a tuple or list eventset = {'end', 'start'} @@ -1847,14 +1797,12 @@ def __next__(self): self._feed(parser, "<foo>bar</foo>") self.assert_event_tags(parser, [('start', 'foo'), ('end', 'foo')]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_unknown_event(self): with self.assertRaises(ValueError): ET.XMLPullParser(events=('start', 'end', 'bogus')) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipIf(pyexpat.version_info < (2, 6, 0), f'Expat {pyexpat.version_info} does not ' 'support reparse deferral') @@ -1879,8 +1827,7 @@ def test_flush_reparse_deferral_enabled(self): self.assert_event_tags(parser, [('end', 'doc')]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_flush_reparse_deferral_disabled(self): parser = ET.XMLPullParser(events=('start', 'end')) @@ -2063,8 +2010,7 @@ def _my_loader(self, href, parse): else: return None - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_xinclude_default(self): from xml.etree import ElementInclude doc = self.xinclude_loader('default.xml') @@ -2079,8 +2025,7 @@ def test_xinclude_default(self): '</root>\n' '</document>') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_xinclude(self): from xml.etree import ElementInclude @@ -2145,8 +2090,7 @@ def test_xinclude(self): ' </ns0:include>\n' '</div>') # C5 - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_xinclude_repeated(self): from xml.etree import ElementInclude @@ -2154,8 +2098,7 @@ def test_xinclude_repeated(self): ElementInclude.include(document, self.xinclude_loader) self.assertEqual(1+4*2, len(document.findall(".//p"))) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_xinclude_failures(self): from xml.etree import ElementInclude @@ -2260,8 +2203,7 @@ def check(elem): elem.set("123", 123) check(elem) # attribute value - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_xmltoolkit25(self): # typo in ElementTree.findtext @@ -2270,8 +2212,7 @@ def test_bug_xmltoolkit25(self): self.assertEqual(tree.findtext("tag"), 'text') self.assertEqual(tree.findtext("section/tag"), 'subtext') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_xmltoolkit28(self): # .//tag causes exceptions @@ -2279,8 +2220,7 @@ def test_bug_xmltoolkit28(self): self.assertEqual(summarize_list(tree.findall(".//thead")), []) self.assertEqual(summarize_list(tree.findall(".//tbody")), ['tbody']) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_xmltoolkitX1(self): # dump() doesn't flush the output buffer @@ -2289,8 +2229,7 @@ def test_bug_xmltoolkitX1(self): ET.dump(tree) self.assertEqual(stdout.getvalue(), '<doc><table><tbody /></table></doc>\n') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_xmltoolkit39(self): # non-ascii element and attribute names doesn't work @@ -2316,8 +2255,7 @@ def test_bug_xmltoolkit39(self): self.assertEqual(ET.tostring(tree, "utf-8"), b'<tag \xc3\xa4ttr="v\xc3\xa4lue" />') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_xmltoolkit54(self): # problems handling internally defined entities @@ -2327,8 +2265,7 @@ def test_bug_xmltoolkit54(self): b'<doc>舰</doc>') self.assertEqual(serialize(e), '<doc>\u8230</doc>') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_xmltoolkit55(self): # make sure we're reporting the first error, not the last @@ -2338,8 +2275,7 @@ def test_bug_xmltoolkit55(self): self.assertEqual(str(cm.exception), 'undefined entity &ldots;: line 1, column 36') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_xmltoolkit60(self): # Handle crash in stream source. @@ -2349,8 +2285,7 @@ def read(self, x): self.assertRaises(OSError, ET.parse, ExceptionFile()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_xmltoolkit62(self): # Don't crash when using custom entities. @@ -2368,8 +2303,7 @@ def test_bug_xmltoolkit62(self): self.assertEqual(t.find('.//paragraph').text, 'A new cultivar of Begonia plant named \u2018BCT9801BEG\u2019.') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipIf(sys.gettrace(), "Skips under coverage.") def test_bug_xmltoolkit63(self): # Check reference leak. @@ -2385,8 +2319,7 @@ def xmltoolkit63(): xmltoolkit63() self.assertEqual(sys.getrefcount(None), count) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_200708_newline(self): # Preserve newlines in attributes. @@ -2398,8 +2331,7 @@ def test_bug_200708_newline(self): self.assertEqual(ET.tostring(ET.XML(ET.tostring(e))), b'<SomeTag text="def _f(): return 3 " />') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_200708_close(self): # Test default builder. parser = ET.XMLParser() # default @@ -2437,8 +2369,7 @@ def test_bug_200709_default_namespace(self): self.assertEqual(str(cm.exception), 'cannot use non-qualified names with default_namespace option') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_200709_register_namespace(self): e = ET.Element("{http://namespace.invalid/does/not/exist/}title") self.assertEqual(ET.tostring(e), @@ -2494,8 +2425,7 @@ def test_bug_1534630(self): e = bob.close() self.assertEqual(serialize(e), '<tag />') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_issue6233(self): e = ET.XML(b"<?xml version='1.0' encoding='utf-8'?>" b'<body>t\xc3\xa3g</body>') @@ -2508,8 +2438,7 @@ def test_issue6233(self): b"<?xml version='1.0' encoding='ascii'?>\n" b'<body>tãg</body>') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_issue6565(self): elem = ET.XML("<body><tag/></body>") self.assertEqual(summarize_list(elem), ['tag']) @@ -2555,8 +2484,7 @@ def __bool__(self): self.assertIsInstance(e[0].tail, str) self.assertEqual(e[0].tail, 'changed') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_lost_elem(self): # Issue #25902: Borrowed element can disappear class Tag: @@ -2582,8 +2510,7 @@ def check_expat224_utf8_bug(self, text): root = ET.XML(xml) self.assertEqual(root.get('b'), text.decode('utf-8')) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_expat224_utf8_bug(self): # bpo-31170: Expat 2.2.3 had a bug in its UTF-8 decoder. # Check that Expat 2.2.4 fixed the bug. @@ -2596,8 +2523,7 @@ def test_expat224_utf8_bug(self): text = b'x' + b'\xc3\xa0' * 1024 self.check_expat224_utf8_bug(text) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_expat224_utf8_bug_file(self): with open(UTF8_BUG_XMLFILE, 'rb') as fp: raw = fp.read() @@ -2747,8 +2673,7 @@ def __deepcopy__(self, memo): e[:] = [E('bar')] self.assertRaises(TypeError, copy.deepcopy, e) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cyclic_gc(self): class Dummy: pass @@ -2821,8 +2746,7 @@ def test_pickle(self): self.assertEqual(len(e2), 2) self.assertEqualElements(e, e2) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_pickle_issue18997(self): for proto in range(2, pickle.HIGHEST_PROTOCOL + 1): for dumper, loader in product(self.modules, repeat=2): @@ -3346,8 +3270,7 @@ class MyElement(ET.Element): class ElementFindTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_find_simple(self): e = ET.XML(SAMPLE_XML) self.assertEqual(e.find('tag').tag, 'tag') @@ -3371,8 +3294,7 @@ def test_find_simple(self): # Issue #16922 self.assertEqual(ET.XML('<tag><empty /></tag>').findtext('empty'), '') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_find_xpath(self): LINEAR_XML = ''' <body> @@ -3395,8 +3317,7 @@ def test_find_xpath(self): self.assertRaisesRegex(SyntaxError, 'XPath', e.find, './tag[last()-0]') self.assertRaisesRegex(SyntaxError, 'XPath', e.find, './tag[last()+1]') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_findall(self): e = ET.XML(SAMPLE_XML) e[2] = ET.XML(SAMPLE_SECTION) @@ -3509,8 +3430,7 @@ def test_findall(self): self.assertEqual(summarize_list(e.findall(".//tag[. = 'subtext']")), ['tag', 'tag']) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_test_find_with_ns(self): e = ET.XML(SAMPLE_XML_NS) self.assertEqual(summarize_list(e.findall('tag')), []) @@ -3521,8 +3441,7 @@ def test_test_find_with_ns(self): summarize_list(e.findall(".//{http://effbot.org/ns}tag")), ['{http://effbot.org/ns}tag'] * 3) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_findall_different_nsmaps(self): root = ET.XML(''' <a xmlns:x="X" xmlns:y="Y"> @@ -3540,8 +3459,7 @@ def test_findall_different_nsmaps(self): self.assertEqual(len(root.findall(".//xx:b", namespaces=nsmap)), 2) self.assertEqual(len(root.findall(".//b", namespaces=nsmap)), 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_findall_wildcard(self): root = ET.XML(''' <a xmlns:x="X" xmlns:y="Y"> @@ -3586,15 +3504,13 @@ def test_findall_wildcard(self): self.assertEqual(summarize_list(root.findall(".//{}b")), summarize_list(root.findall(".//b"))) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bad_find(self): e = ET.XML(SAMPLE_XML) with self.assertRaisesRegex(SyntaxError, 'cannot use absolute path'): e.findall('/tag') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_find_through_ElementTree(self): e = ET.XML(SAMPLE_XML) self.assertEqual(ET.ElementTree(e).find('tag').tag, 'tag') @@ -3614,8 +3530,7 @@ class ElementIterTest(unittest.TestCase): def _ilist(self, elem, tag=None): return summarize_list(elem.iter(tag)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_basic(self): doc = ET.XML("<html><body>this is a <i>paragraph</i>.</body>..</html>") self.assertEqual(self._ilist(doc), ['html', 'body', 'i']) @@ -3664,8 +3579,7 @@ def test_corners(self): del a[1] self.assertEqual(self._ilist(a), ['a', 'd']) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_iter_by_tag(self): doc = ET.XML(''' <document> @@ -3730,8 +3644,7 @@ def _check_sample1_element(self, e): self.assertEqual(child.tail, 'tail') self.assertEqual(child.attrib, {}) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_dummy_builder(self): class BaseDummyBuilder: def close(self): @@ -3779,8 +3692,7 @@ def test_treebuilder_pi(self): self.assertEqual(b.pi('target'), (len('target'), None)) self.assertEqual(b.pi('pitarget', ' text '), (len('pitarget'), ' text ')) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_late_tail(self): # Issue #37399: The tail of an ignored comment could overwrite the text before it. class TreeBuilderSubclass(ET.TreeBuilder): @@ -3805,8 +3717,7 @@ class TreeBuilderSubclass(ET.TreeBuilder): a = parser.close() self.assertEqual(a.text, "texttail") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_late_tail_mix_pi_comments(self): # Issue #37399: The tail of an ignored comment could overwrite the text before it. # Test appending tails to comments/pis. @@ -3843,16 +3754,14 @@ class TreeBuilderSubclass(ET.TreeBuilder): self.assertEqual(a[0].tail, 'tail') self.assertEqual(a.text, "text\n") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_treebuilder_elementfactory_none(self): parser = ET.XMLParser(target=ET.TreeBuilder(element_factory=None)) parser.feed(self.sample1) e = parser.close() self._check_sample1_element(e) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_subclass(self): class MyTreeBuilder(ET.TreeBuilder): def foobar(self, x): @@ -3867,8 +3776,7 @@ def foobar(self, x): e = parser.close() self._check_sample1_element(e) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_subclass_comment_pi(self): class MyTreeBuilder(ET.TreeBuilder): def foobar(self, x): @@ -3884,8 +3792,7 @@ def foobar(self, x): e = parser.close() self._check_sample1_element(e) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_element_factory(self): lst = [] def myfactory(tag, attrib): @@ -3909,15 +3816,13 @@ def _check_element_factory_class(self, cls): self.assertIsInstance(e, cls) self._check_sample1_element(e) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_element_factory_subclass(self): class MyElement(ET.Element): pass self._check_element_factory_class(MyElement) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_element_factory_pure_python_subclass(self): # Mimic SimpleTAL's behaviour (issue #16089): both versions of # TreeBuilder should be able to cope with a subclass of the @@ -3931,8 +3836,7 @@ class MyElement(base, ValueError): pass self._check_element_factory_class(MyElement) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_doctype(self): class DoctypeParser: _doctype = None @@ -3950,8 +3854,7 @@ def close(self): ('html', '-//W3C//DTD XHTML 1.0 Transitional//EN', 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd')) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_builder_lookup_errors(self): class RaisingBuilder: def __init__(self, raise_in=None, what=ValueError): @@ -3992,16 +3895,14 @@ def _check_sample_element(self, e): self.assertEqual(e[0].tag, 'line') self.assertEqual(e[0].text, '22') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_constructor_args(self): parser2 = ET.XMLParser(encoding='utf-8', target=ET.TreeBuilder()) parser2.feed(self.sample1) self._check_sample_element(parser2.close()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_subclass(self): class MyParser(ET.XMLParser): pass @@ -4009,8 +3910,7 @@ class MyParser(ET.XMLParser): parser.feed(self.sample1) self._check_sample_element(parser.close()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_doctype_warning(self): with warnings.catch_warnings(): warnings.simplefilter('error', DeprecationWarning) @@ -4018,8 +3918,7 @@ def test_doctype_warning(self): parser.feed(self.sample2) parser.close() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_subclass_doctype(self): _doctype = None class MyParserWithDoctype(ET.XMLParser): @@ -4050,8 +3949,7 @@ def doctype(self, name, pubid, system): ('html', '-//W3C//DTD XHTML 1.0 Transitional//EN', 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd')) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_inherited_doctype(self): '''Ensure that ordinary usage is not deprecated (Issue 19176)''' with warnings.catch_warnings(): @@ -4063,8 +3961,7 @@ class MyParserWithoutDoctype(ET.XMLParser): parser.feed(self.sample2) parser.close() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_parse_string(self): parser = ET.XMLParser(target=ET.TreeBuilder()) parser.feed(self.sample3) @@ -4075,8 +3972,7 @@ def test_parse_string(self): class NamespaceParseTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_find_with_namespace(self): nsmap = {'h': 'hello', 'f': 'foo'} doc = ET.fromstring(SAMPLE_XML_NS_ELEMS) @@ -4253,8 +4149,7 @@ def f(): e[:1] = (f() for i in range(2)) class IOTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_encoding(self): # Test encoding issues. elem = ET.Element("tag") @@ -4324,8 +4219,7 @@ def test_encoding(self): ("<?xml version='1.0' encoding='%s'?>\n" "<tag key=\"åöö<>\" />" % enc).encode(enc)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_filename(self): self.addCleanup(os_helper.unlink, TESTFN) tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) @@ -4333,8 +4227,7 @@ def test_write_to_filename(self): with open(TESTFN, 'rb') as f: self.assertEqual(f.read(), b'''<site>ø</site>''') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_filename_with_encoding(self): self.addCleanup(os_helper.unlink, TESTFN) tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) @@ -4348,8 +4241,7 @@ def test_write_to_filename_with_encoding(self): b'''<?xml version='1.0' encoding='ISO-8859-1'?>\n''' b'''<site>\xf8</site>''')) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_filename_as_unicode(self): self.addCleanup(os_helper.unlink, TESTFN) with open(TESTFN, 'w') as f: @@ -4361,8 +4253,7 @@ def test_write_to_filename_as_unicode(self): with open(TESTFN, 'rb') as f: self.assertEqual(f.read(), b"<site>\xc3\xb8</site>") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_text_file(self): self.addCleanup(os_helper.unlink, TESTFN) tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) @@ -4384,8 +4275,7 @@ def test_write_to_text_file(self): with open(TESTFN, 'rb') as f: self.assertEqual(f.read(), b'''<site>\xf8</site>''') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_binary_file(self): self.addCleanup(os_helper.unlink, TESTFN) tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) @@ -4395,8 +4285,7 @@ def test_write_to_binary_file(self): with open(TESTFN, 'rb') as f: self.assertEqual(f.read(), b'''<site>ø</site>''') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_binary_file_with_encoding(self): self.addCleanup(os_helper.unlink, TESTFN) tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) @@ -4414,8 +4303,7 @@ def test_write_to_binary_file_with_encoding(self): b'''<?xml version='1.0' encoding='ISO-8859-1'?>\n''' b'''<site>\xf8</site>''') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_binary_file_with_bom(self): self.addCleanup(os_helper.unlink, TESTFN) tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) @@ -4436,32 +4324,28 @@ def test_write_to_binary_file_with_bom(self): '''<?xml version='1.0' encoding='utf-16'?>\n''' '''<site>\xf8</site>'''.encode("utf-16")) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_from_stringio(self): tree = ET.ElementTree() stream = io.StringIO('''<?xml version="1.0"?><site></site>''') tree.parse(stream) self.assertEqual(tree.getroot().tag, 'site') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_stringio(self): tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) stream = io.StringIO() tree.write(stream, encoding='unicode') self.assertEqual(stream.getvalue(), '''<site>\xf8</site>''') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_from_bytesio(self): tree = ET.ElementTree() raw = io.BytesIO(b'''<?xml version="1.0"?><site></site>''') tree.parse(raw) self.assertEqual(tree.getroot().tag, 'site') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_bytesio(self): tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) raw = io.BytesIO() @@ -4471,8 +4355,7 @@ def test_write_to_bytesio(self): class dummy: pass - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_from_user_text_reader(self): stream = io.StringIO('''<?xml version="1.0"?><site></site>''') reader = self.dummy() @@ -4481,8 +4364,7 @@ def test_read_from_user_text_reader(self): tree.parse(reader) self.assertEqual(tree.getroot().tag, 'site') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_user_text_writer(self): tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) stream = io.StringIO() @@ -4491,8 +4373,7 @@ def test_write_to_user_text_writer(self): tree.write(writer, encoding='unicode') self.assertEqual(stream.getvalue(), '''<site>\xf8</site>''') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_from_user_binary_reader(self): raw = io.BytesIO(b'''<?xml version="1.0"?><site></site>''') reader = self.dummy() @@ -4502,8 +4383,7 @@ def test_read_from_user_binary_reader(self): self.assertEqual(tree.getroot().tag, 'site') tree = ET.ElementTree() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_user_binary_writer(self): tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) raw = io.BytesIO() @@ -4512,8 +4392,7 @@ def test_write_to_user_binary_writer(self): tree.write(writer) self.assertEqual(raw.getvalue(), b'''<site>ø</site>''') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_user_binary_writer_with_bom(self): tree = ET.ElementTree(ET.XML('''<site />''')) raw = io.BytesIO() @@ -4526,8 +4405,7 @@ def test_write_to_user_binary_writer_with_bom(self): '''<?xml version='1.0' encoding='utf-16'?>\n''' '''<site />'''.encode("utf-16")) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostringlist_invariant(self): root = ET.fromstring('<tag>foo</tag>') self.assertEqual( @@ -4537,8 +4415,7 @@ def test_tostringlist_invariant(self): ET.tostring(root, 'utf-16'), b''.join(ET.tostringlist(root, 'utf-16'))) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_short_empty_elements(self): root = ET.fromstring('<tag>a<x />b<y></y>c</tag>') self.assertEqual( @@ -4562,15 +4439,13 @@ def _get_error(self, s): except ET.ParseError as e: return e - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_error_position(self): self.assertEqual(self._get_error('foo').position, (1, 0)) self.assertEqual(self._get_error('<tag>&foo;</tag>').position, (1, 5)) self.assertEqual(self._get_error('foobar<').position, (1, 6)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_error_code(self): import xml.parsers.expat.errors as ERRORS self.assertEqual(self._get_error('foo').code, @@ -4580,8 +4455,7 @@ def test_error_code(self): class KeywordArgsTest(unittest.TestCase): # Test various issues with keyword arguments passed to ET.Element # constructor and methods - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_issue14818(self): x = ET.XML("<a>foo</a>") self.assertEqual(x.find('a', None), @@ -4632,8 +4506,7 @@ def test_correct_import_pyET(self): # -------------------------------------------------------------------- class BoolTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_warning(self): e = ET.fromstring('<a style="new"></a>') msg = ( @@ -4663,8 +4536,7 @@ class C14NTest(unittest.TestCase): # # simple roundtrip tests (from c14n.py) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_simple_roundtrip(self): # Basics self.assertEqual(c14n_roundtrip("<doc/>"), '<doc></doc>') @@ -4705,8 +4577,7 @@ def test_simple_roundtrip(self): xml = '<X xmlns="http://nps/a"><Y xmlns:b="http://nsp/b" b:targets="abc,xyz"></Y></X>' self.assertEqual(c14n_roundtrip(xml), xml) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_c14n_exclusion(self): xml = textwrap.dedent("""\ <root xmlns:x="http://example.com/x"> @@ -4787,8 +4658,7 @@ def test_c14n_exclusion(self): # note that this uses generated C14N versions of the standard ET.write # output, not roundtripped C14N (see above). - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_xml_c14n2(self): datadir = findfile("c14n-20", subdir="xmltestdata") full_path = partial(os.path.join, datadir) diff --git a/Lib/test/test_yield_from.py b/Lib/test/test_yield_from.py index 97acfd5413..88fa1b88c9 100644 --- a/Lib/test/test_yield_from.py +++ b/Lib/test/test_yield_from.py @@ -538,8 +538,7 @@ def g(): "finishing g", ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_broken_getattr_handling(self): """ Test subiterator with a broken getattr implementation @@ -787,8 +786,7 @@ def outer(): repr(value), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_throwing_GeneratorExit_into_subgen_that_returns(self): """ Test throwing GeneratorExit into a subgenerator that @@ -819,8 +817,7 @@ def g(): "Enter f", ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_throwing_GeneratorExit_into_subgenerator_that_yields(self): """ Test throwing GeneratorExit into a subgenerator that @@ -887,8 +884,7 @@ def g(): yield from () self.assertRaises(StopIteration, next, g()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_delegating_generators_claim_to_be_running(self): # Check with basic iteration def one(): @@ -904,6 +900,7 @@ def two(): yield 2 g1 = one() self.assertEqual(list(g1), [0, 1, 2, 3]) + # Check with send g1 = one() res = [next(g1)] @@ -913,6 +910,9 @@ def two(): except StopIteration: pass self.assertEqual(res, [0, 1, 2, 3]) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: Lists differ: [0, 1, 2] != [0, 1, 2, 3] + def test_delegating_generators_claim_to_be_running_with_throw(self): # Check with throw class MyErr(Exception): pass @@ -949,8 +949,10 @@ def two(): except: self.assertEqual(res, [0, 1, 2, 3]) raise + + def test_delegating_generators_claim_to_be_running_with_close(self): # Check with close - class MyIt(object): + class MyIt: def __iter__(self): return self def __next__(self): @@ -1057,6 +1059,538 @@ def outer(): g.send((1, 2, 3, 4)) self.assertEqual(v, (1, 2, 3, 4)) +class TestInterestingEdgeCases(unittest.TestCase): + + def assert_stop_iteration(self, iterator): + with self.assertRaises(StopIteration) as caught: + next(iterator) + self.assertIsNone(caught.exception.value) + self.assertIsNone(caught.exception.__context__) + + def assert_generator_raised_stop_iteration(self): + return self.assertRaisesRegex(RuntimeError, r"^generator raised StopIteration$") + + def assert_generator_ignored_generator_exit(self): + return self.assertRaisesRegex(RuntimeError, r"^generator ignored GeneratorExit$") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_close_and_throw_work(self): + + yielded_first = object() + yielded_second = object() + returned = object() + + def inner(): + yield yielded_first + yield yielded_second + return returned + + def outer(): + return (yield from inner()) + + with self.subTest("close"): + g = outer() + self.assertIs(next(g), yielded_first) + g.close() + self.assert_stop_iteration(g) + + with self.subTest("throw GeneratorExit"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = GeneratorExit() + with self.assertRaises(GeneratorExit) as caught: + g.throw(thrown) + self.assertIs(caught.exception, thrown) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw StopIteration"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = StopIteration() + # PEP 479: + with self.assert_generator_raised_stop_iteration() as caught: + g.throw(thrown) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw BaseException"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = BaseException() + with self.assertRaises(BaseException) as caught: + g.throw(thrown) + self.assertIs(caught.exception, thrown) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw Exception"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = Exception() + with self.assertRaises(Exception) as caught: + g.throw(thrown) + self.assertIs(caught.exception, thrown) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: GeneratorExit() is not GeneratorExit() + def test_close_and_throw_raise_generator_exit(self): + + yielded_first = object() + yielded_second = object() + returned = object() + + def inner(): + try: + yield yielded_first + yield yielded_second + return returned + finally: + raise raised + + def outer(): + return (yield from inner()) + + with self.subTest("close"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = GeneratorExit() + # GeneratorExit is suppressed. This is consistent with PEP 342: + # https://peps.python.org/pep-0342/#new-generator-method-close + g.close() + self.assert_stop_iteration(g) + + with self.subTest("throw GeneratorExit"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = GeneratorExit() + thrown = GeneratorExit() + with self.assertRaises(GeneratorExit) as caught: + g.throw(thrown) + # The raised GeneratorExit is suppressed, but the thrown one + # propagates. This is consistent with PEP 380: + # https://peps.python.org/pep-0380/#proposal + self.assertIs(caught.exception, thrown) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw StopIteration"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = GeneratorExit() + thrown = StopIteration() + with self.assertRaises(GeneratorExit) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw BaseException"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = GeneratorExit() + thrown = BaseException() + with self.assertRaises(GeneratorExit) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw Exception"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = GeneratorExit() + thrown = Exception() + with self.assertRaises(GeneratorExit) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: RuntimeError not raised + def test_close_and_throw_raise_stop_iteration(self): + + yielded_first = object() + yielded_second = object() + returned = object() + + def inner(): + try: + yield yielded_first + yield yielded_second + return returned + finally: + raise raised + + def outer(): + return (yield from inner()) + + with self.subTest("close"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = StopIteration() + # PEP 479: + with self.assert_generator_raised_stop_iteration() as caught: + g.close() + self.assertIs(caught.exception.__context__, raised) + self.assertIsInstance(caught.exception.__context__.__context__, GeneratorExit) + self.assertIsNone(caught.exception.__context__.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw GeneratorExit"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = StopIteration() + thrown = GeneratorExit() + # PEP 479: + with self.assert_generator_raised_stop_iteration() as caught: + g.throw(thrown) + self.assertIs(caught.exception.__context__, raised) + # This isn't the same GeneratorExit as thrown! It's the one created + # by calling inner.close(): + self.assertIsInstance(caught.exception.__context__.__context__, GeneratorExit) + self.assertIsNone(caught.exception.__context__.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw StopIteration"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = StopIteration() + thrown = StopIteration() + # PEP 479: + with self.assert_generator_raised_stop_iteration() as caught: + g.throw(thrown) + self.assertIs(caught.exception.__context__, raised) + self.assertIs(caught.exception.__context__.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw BaseException"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = StopIteration() + thrown = BaseException() + # PEP 479: + with self.assert_generator_raised_stop_iteration() as caught: + g.throw(thrown) + self.assertIs(caught.exception.__context__, raised) + self.assertIs(caught.exception.__context__.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw Exception"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = StopIteration() + thrown = Exception() + # PEP 479: + with self.assert_generator_raised_stop_iteration() as caught: + g.throw(thrown) + self.assertIs(caught.exception.__context__, raised) + self.assertIs(caught.exception.__context__.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__.__context__) + self.assert_stop_iteration(g) + + def test_close_and_throw_raise_base_exception(self): + + yielded_first = object() + yielded_second = object() + returned = object() + + def inner(): + try: + yield yielded_first + yield yielded_second + return returned + finally: + raise raised + + def outer(): + return (yield from inner()) + + with self.subTest("close"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = BaseException() + with self.assertRaises(BaseException) as caught: + g.close() + self.assertIs(caught.exception, raised) + self.assertIsInstance(caught.exception.__context__, GeneratorExit) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw GeneratorExit"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = BaseException() + thrown = GeneratorExit() + with self.assertRaises(BaseException) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + # This isn't the same GeneratorExit as thrown! It's the one created + # by calling inner.close(): + self.assertIsInstance(caught.exception.__context__, GeneratorExit) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw StopIteration"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = BaseException() + thrown = StopIteration() + with self.assertRaises(BaseException) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw BaseException"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = BaseException() + thrown = BaseException() + with self.assertRaises(BaseException) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw Exception"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = BaseException() + thrown = Exception() + with self.assertRaises(BaseException) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + def test_close_and_throw_raise_exception(self): + + yielded_first = object() + yielded_second = object() + returned = object() + + def inner(): + try: + yield yielded_first + yield yielded_second + return returned + finally: + raise raised + + def outer(): + return (yield from inner()) + + with self.subTest("close"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = Exception() + with self.assertRaises(Exception) as caught: + g.close() + self.assertIs(caught.exception, raised) + self.assertIsInstance(caught.exception.__context__, GeneratorExit) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw GeneratorExit"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = Exception() + thrown = GeneratorExit() + with self.assertRaises(Exception) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + # This isn't the same GeneratorExit as thrown! It's the one created + # by calling inner.close(): + self.assertIsInstance(caught.exception.__context__, GeneratorExit) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw StopIteration"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = Exception() + thrown = StopIteration() + with self.assertRaises(Exception) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw BaseException"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = Exception() + thrown = BaseException() + with self.assertRaises(Exception) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw Exception"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = Exception() + thrown = Exception() + with self.assertRaises(Exception) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: None is not StopIteration() + def test_close_and_throw_yield(self): + + yielded_first = object() + yielded_second = object() + returned = object() + + def inner(): + try: + yield yielded_first + finally: + yield yielded_second + return returned + + def outer(): + return (yield from inner()) + + with self.subTest("close"): + g = outer() + self.assertIs(next(g), yielded_first) + # No chaining happens. This is consistent with PEP 342: + # https://peps.python.org/pep-0342/#new-generator-method-close + with self.assert_generator_ignored_generator_exit() as caught: + g.close() + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw GeneratorExit"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = GeneratorExit() + # No chaining happens. This is consistent with PEP 342: + # https://peps.python.org/pep-0342/#new-generator-method-close + with self.assert_generator_ignored_generator_exit() as caught: + g.throw(thrown) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw StopIteration"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = StopIteration() + self.assertEqual(g.throw(thrown), yielded_second) + # PEP 479: + with self.assert_generator_raised_stop_iteration() as caught: + next(g) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw BaseException"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = BaseException() + self.assertEqual(g.throw(thrown), yielded_second) + with self.assertRaises(BaseException) as caught: + next(g) + self.assertIs(caught.exception, thrown) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw Exception"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = Exception() + self.assertEqual(g.throw(thrown), yielded_second) + with self.assertRaises(Exception) as caught: + next(g) + self.assertIs(caught.exception, thrown) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_close_and_throw_return(self): + + yielded_first = object() + yielded_second = object() + returned = object() + + def inner(): + try: + yield yielded_first + yield yielded_second + finally: + return returned + + def outer(): + return (yield from inner()) + + with self.subTest("close"): + g = outer() + self.assertIs(next(g), yielded_first) + # StopIteration is suppressed. This is consistent with PEP 342: + # https://peps.python.org/pep-0342/#new-generator-method-close + g.close() + self.assert_stop_iteration(g) + + with self.subTest("throw GeneratorExit"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = GeneratorExit() + # StopIteration is suppressed. This is consistent with PEP 342: + # https://peps.python.org/pep-0342/#new-generator-method-close + with self.assertRaises(GeneratorExit) as caught: + g.throw(thrown) + self.assertIs(caught.exception, thrown) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw StopIteration"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = StopIteration() + with self.assertRaises(StopIteration) as caught: + g.throw(thrown) + self.assertIs(caught.exception.value, returned) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw BaseException"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = BaseException() + with self.assertRaises(StopIteration) as caught: + g.throw(thrown) + self.assertIs(caught.exception.value, returned) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw Exception"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = Exception() + with self.assertRaises(StopIteration) as caught: + g.throw(thrown) + self.assertIs(caught.exception.value, returned) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + if __name__ == '__main__': unittest.main() diff --git a/Lib/uuid.py b/Lib/uuid.py index c286eac38e..55f46eb510 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -572,39 +572,43 @@ def _netstat_getnode(): try: import _uuid _generate_time_safe = getattr(_uuid, "generate_time_safe", None) + _has_stable_extractable_node = getattr(_uuid, "has_stable_extractable_node", False) _UuidCreate = getattr(_uuid, "UuidCreate", None) except ImportError: _uuid = None _generate_time_safe = None + _has_stable_extractable_node = False _UuidCreate = None def _unix_getnode(): """Get the hardware address on Unix using the _uuid extension module.""" - if _generate_time_safe: + if _generate_time_safe and _has_stable_extractable_node: uuid_time, _ = _generate_time_safe() return UUID(bytes=uuid_time).node def _windll_getnode(): """Get the hardware address on Windows using the _uuid extension module.""" - if _UuidCreate: + if _UuidCreate and _has_stable_extractable_node: uuid_bytes = _UuidCreate() return UUID(bytes_le=uuid_bytes).node def _random_getnode(): """Get a random node ID.""" - # RFC 4122, $4.1.6 says "For systems with no IEEE address, a randomly or - # pseudo-randomly generated value may be used; see Section 4.5. The - # multicast bit must be set in such addresses, in order that they will - # never conflict with addresses obtained from network cards." + # RFC 9562, §6.10-3 says that + # + # Implementations MAY elect to obtain a 48-bit cryptographic-quality + # random number as per Section 6.9 to use as the Node ID. [...] [and] + # implementations MUST set the least significant bit of the first octet + # of the Node ID to 1. This bit is the unicast or multicast bit, which + # will never be set in IEEE 802 addresses obtained from network cards. # # The "multicast bit" of a MAC address is defined to be "the least # significant bit of the first octet". This works out to be the 41st bit # counting from 1 being the least significant bit, or 1<<40. # # See https://en.wikipedia.org/w/index.php?title=MAC_address&oldid=1128764812#Universal_vs._local_(U/L_bit) - import random - return random.getrandbits(48) | (1 << 40) + return int.from_bytes(os.urandom(6)) | (1 << 40) # _OS_GETTERS, when known, are targeted for a specific OS or platform. diff --git a/Lib/zoneinfo/_common.py b/Lib/zoneinfo/_common.py index 98cdfe37ca..03cc42149f 100644 --- a/Lib/zoneinfo/_common.py +++ b/Lib/zoneinfo/_common.py @@ -9,9 +9,13 @@ def load_tzdata(key): resource_name = components[-1] try: - return resources.files(package_name).joinpath(resource_name).open("rb") - except (ImportError, FileNotFoundError, UnicodeEncodeError): - # There are three types of exception that can be raised that all amount + path = resources.files(package_name).joinpath(resource_name) + # gh-85702: Prevent PermissionError on Windows + if path.is_dir(): + raise IsADirectoryError + return path.open("rb") + except (ImportError, FileNotFoundError, UnicodeEncodeError, IsADirectoryError): + # There are four types of exception that can be raised that all amount # to "we cannot find this key": # # ImportError: If package_name doesn't exist (e.g. if tzdata is not @@ -21,6 +25,7 @@ def load_tzdata(key): # (e.g. Europe/Krasnoy) # UnicodeEncodeError: If package_name or resource_name are not UTF-8, # such as keys containing a surrogate character. + # IsADirectoryError: If package_name without a resource_name specified. raise ZoneInfoNotFoundError(f"No time zone found with key {key}") diff --git a/Lib/zoneinfo/_zoneinfo.py b/Lib/zoneinfo/_zoneinfo.py index b77dc0ed39..3ffdb4c837 100644 --- a/Lib/zoneinfo/_zoneinfo.py +++ b/Lib/zoneinfo/_zoneinfo.py @@ -75,12 +75,12 @@ def _new_instance(cls, key): return obj @classmethod - def from_file(cls, fobj, /, key=None): + def from_file(cls, file_obj, /, key=None): obj = super().__new__(cls) obj._key = key obj._file_path = None - obj._load_file(fobj) - obj._file_repr = repr(fobj) + obj._load_file(file_obj) + obj._file_repr = repr(file_obj) # Disable pickling for objects created from files obj.__reduce__ = obj._file_reduce diff --git a/benches/execution.rs b/benches/execution.rs index 7a7ba247e5..c2239b59d1 100644 --- a/benches/execution.rs +++ b/benches/execution.rs @@ -1,17 +1,15 @@ -use criterion::measurement::WallTime; use criterion::{ - Bencher, BenchmarkGroup, BenchmarkId, Criterion, Throughput, black_box, criterion_group, - criterion_main, + Bencher, BenchmarkGroup, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main, + measurement::WallTime, }; use rustpython_compiler::Mode; use rustpython_vm::{Interpreter, PyResult, Settings}; -use std::collections::HashMap; -use std::path::Path; +use std::{collections::HashMap, hint::black_box, path::Path}; fn bench_cpython_code(b: &mut Bencher, source: &str) { let c_str_source_head = std::ffi::CString::new(source).unwrap(); let c_str_source = c_str_source_head.as_c_str(); - pyo3::Python::with_gil(|py| { + pyo3::Python::attach(|py| { b.iter(|| { let module = pyo3::types::PyModule::from_code(py, c_str_source, c"", c"") .expect("Error running source"); @@ -54,7 +52,7 @@ pub fn benchmark_file_parsing(group: &mut BenchmarkGroup<WallTime>, name: &str, }); group.bench_function(BenchmarkId::new("cpython", name), |b| { use pyo3::types::PyAnyMethods; - pyo3::Python::with_gil(|py| { + pyo3::Python::attach(|py| { let builtins = pyo3::types::PyModule::import(py, "builtins").expect("Failed to import builtins"); let compile = builtins.getattr("compile").expect("no compile in builtins"); diff --git a/benches/microbenchmarks.rs b/benches/microbenchmarks.rs index 5f04f4bbf8..f70be59522 100644 --- a/benches/microbenchmarks.rs +++ b/benches/microbenchmarks.rs @@ -37,7 +37,7 @@ pub struct MicroBenchmark { } fn bench_cpython_code(group: &mut BenchmarkGroup<WallTime>, bench: &MicroBenchmark) { - pyo3::Python::with_gil(|py| { + pyo3::Python::attach(|py| { let setup_name = format!("{}_setup", bench.name); let setup_code = cpy_compile_code(py, &bench.setup, &setup_name).unwrap(); diff --git a/common/Cargo.toml b/common/Cargo.toml index 94704d18c9..c75f33a7b7 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -10,6 +10,7 @@ license.workspace = true [features] threading = ["parking_lot"] +wasm_js = ["getrandom/wasm_js"] [dependencies] rustpython-literal = { workspace = true } @@ -17,7 +18,6 @@ rustpython-wtf8 = { workspace = true } ascii = { workspace = true } bitflags = { workspace = true } -bstr = { workspace = true } cfg-if = { workspace = true } getrandom = { workspace = true } itertools = { workspace = true } @@ -25,7 +25,6 @@ libc = { workspace = true } malachite-bigint = { workspace = true } malachite-q = { workspace = true } malachite-base = { workspace = true } -memchr = { workspace = true } num-traits = { workspace = true } once_cell = { workspace = true } parking_lot = { workspace = true, optional = true } diff --git a/common/src/crt_fd.rs b/common/src/crt_fd.rs index b4c1ea3294..4fda2f3dcf 100644 --- a/common/src/crt_fd.rs +++ b/common/src/crt_fd.rs @@ -189,6 +189,10 @@ impl Owned { pub fn into_raw(self) -> Raw { self.inner.into_raw_fd() } + + pub fn leak<'fd>(self) -> Borrowed<'fd> { + unsafe { Borrowed::borrow_raw(self.into_raw()) } + } } #[cfg(unix)] diff --git a/common/src/fileutils.rs b/common/src/fileutils.rs index f40a0f8069..6fa8e62a65 100644 --- a/common/src/fileutils.rs +++ b/common/src/fileutils.rs @@ -81,14 +81,13 @@ pub mod windows { .next_back() .and_then(|s| String::from_utf16(s).ok()); - if let Some(file_extension) = file_extension { - if file_extension.eq_ignore_ascii_case("exe") + if let Some(file_extension) = file_extension + && (file_extension.eq_ignore_ascii_case("exe") || file_extension.eq_ignore_ascii_case("bat") || file_extension.eq_ignore_ascii_case("cmd") - || file_extension.eq_ignore_ascii_case("com") - { - self.st_mode |= 0o111; - } + || file_extension.eq_ignore_ascii_case("com")) + { + self.st_mode |= 0o111; } } } diff --git a/compiler/Cargo.toml b/compiler/Cargo.toml index 2ecc06d59a..5c79324111 100644 --- a/compiler/Cargo.toml +++ b/compiler/Cargo.toml @@ -18,7 +18,6 @@ ruff_text_size = { workspace = true } thiserror = { workspace = true } [dev-dependencies] -rand = { workspace = true } [lints] workspace = true diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index a0c3a3b92e..d35f3c6e3b 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -23,24 +23,24 @@ use ruff_python_ast::{ Alias, Arguments, BoolOp, CmpOp, Comprehension, ConversionFlag, DebugText, Decorator, DictItem, ExceptHandler, ExceptHandlerExceptHandler, Expr, ExprAttribute, ExprBoolOp, ExprContext, ExprFString, ExprList, ExprName, ExprSlice, ExprStarred, ExprSubscript, ExprTuple, ExprUnaryOp, - FString, FStringElement, FStringElements, FStringFlags, FStringPart, Identifier, Int, Keyword, - MatchCase, ModExpression, ModModule, Operator, Parameters, Pattern, PatternMatchAs, - PatternMatchClass, PatternMatchMapping, PatternMatchOr, PatternMatchSequence, - PatternMatchSingleton, PatternMatchStar, PatternMatchValue, Singleton, Stmt, StmtExpr, - TypeParam, TypeParamParamSpec, TypeParamTypeVar, TypeParamTypeVarTuple, TypeParams, UnaryOp, - WithItem, + FString, FStringFlags, FStringPart, Identifier, Int, InterpolatedElement, + InterpolatedStringElement, InterpolatedStringElements, Keyword, MatchCase, ModExpression, + ModModule, Operator, Parameters, Pattern, PatternMatchAs, PatternMatchClass, + PatternMatchMapping, PatternMatchOr, PatternMatchSequence, PatternMatchSingleton, + PatternMatchStar, PatternMatchValue, Singleton, Stmt, StmtExpr, TypeParam, TypeParamParamSpec, + TypeParamTypeVar, TypeParamTypeVarTuple, TypeParams, UnaryOp, WithItem, }; use ruff_source_file::LineEnding; use ruff_text_size::{Ranged, TextRange}; use rustpython_compiler_core::{ - Mode, OneIndexed, SourceFile, SourceLocation, + Mode, OneIndexed, PositionEncoding, SourceFile, SourceLocation, bytecode::{ self, Arg as OpArgMarker, BinaryOperator, CodeObject, ComparisonOperator, ConstantData, Instruction, OpArg, OpArgType, UnpackExArgs, }, }; use rustpython_wtf8::Wtf8Buf; -use std::borrow::Cow; +use std::{borrow::Cow, collections::HashSet}; const MAXBLOCKS: usize = 20; @@ -160,6 +160,20 @@ fn unparse_expr(expr: &Expr) -> String { ruff_python_codegen::Generator::new(&indentation, LineEnding::default()).expr(expr) } +fn validate_duplicate_params(params: &Parameters) -> Result<(), CodegenErrorType> { + let mut seen_params = HashSet::new(); + for param in params { + let param_name = param.name().as_str(); + if !seen_params.insert(param_name) { + return Err(CodegenErrorType::SyntaxError(format!( + r#"Duplicate parameter "{param_name}""# + ))); + } + } + + Ok(()) +} + /// Compile an Mod produced from ruff parser pub fn compile_top( ast: ruff_python_ast::Mod, @@ -253,18 +267,18 @@ fn eprint_location(zelf: &Compiler) { let start = zelf .source_file .to_source_code() - .source_location(zelf.current_source_range.start()); + .source_location(zelf.current_source_range.start(), PositionEncoding::Utf8); let end = zelf .source_file .to_source_code() - .source_location(zelf.current_source_range.end()); + .source_location(zelf.current_source_range.end(), PositionEncoding::Utf8); eprintln!( "LOCATION: {} from {}:{} to {}:{}", zelf.source_file.name(), - start.row, - start.column, - end.row, - end.column + start.line, + start.character_offset, + end.line, + end.character_offset ); } @@ -544,7 +558,7 @@ impl Compiler { let location = self .source_file .to_source_code() - .source_location(range.start()); + .source_location(range.start(), PositionEncoding::Utf8); CodegenError { error, location: Some(location), @@ -644,8 +658,8 @@ impl Compiler { ) -> CompileResult<()> { // Create location let location = SourceLocation { - row: OneIndexed::new(lineno as usize).unwrap_or(OneIndexed::MIN), - column: OneIndexed::new(1).unwrap(), + line: OneIndexed::new(lineno as usize).unwrap_or(OneIndexed::MIN), + character_offset: OneIndexed::MIN, }; // Allocate a new compiler unit @@ -782,8 +796,8 @@ impl Compiler { let _resume_loc = if scope_type == CompilerScope::Module { // Module scope starts with lineno 0 SourceLocation { - row: OneIndexed::MIN, - column: OneIndexed::MIN, + line: OneIndexed::MIN, + character_offset: OneIndexed::MIN, } } else { location @@ -955,12 +969,11 @@ impl Compiler { if stack_size > self.symbol_table_stack.len() { // We might be in a situation where symbol table isn't pushed yet // In this case, check the parent symbol table - if let Some(parent_table) = self.symbol_table_stack.last() { - if let Some(symbol) = parent_table.lookup(¤t_obj_name) { - if symbol.scope == SymbolScope::GlobalExplicit { - force_global = true; - } - } + if let Some(parent_table) = self.symbol_table_stack.last() + && let Some(symbol) = parent_table.lookup(¤t_obj_name) + && symbol.scope == SymbolScope::GlobalExplicit + { + force_global = true; } } else if let Some(_current_table) = self.symbol_table_stack.last() { // Mangle the name if necessary (for private names in classes) @@ -969,10 +982,10 @@ impl Compiler { // Look up in parent symbol table to check scope if self.symbol_table_stack.len() >= 2 { let parent_table = &self.symbol_table_stack[self.symbol_table_stack.len() - 2]; - if let Some(symbol) = parent_table.lookup(&mangled_name) { - if symbol.scope == SymbolScope::GlobalExplicit { - force_global = true; - } + if let Some(symbol) = parent_table.lookup(&mangled_name) + && symbol.scope == SymbolScope::GlobalExplicit + { + force_global = true; } } } @@ -1514,15 +1527,19 @@ impl Compiler { type_params, is_async, .. - }) => self.compile_function_def( - name.as_str(), - parameters, - body, - decorator_list, - returns.as_deref(), - *is_async, - type_params.as_deref(), - )?, + }) => { + validate_duplicate_params(parameters).map_err(|e| self.error(e))?; + + self.compile_function_def( + name.as_str(), + parameters, + body, + decorator_list, + returns.as_deref(), + *is_async, + type_params.as_deref(), + )? + } Stmt::ClassDef(StmtClassDef { name, body, @@ -3541,10 +3558,10 @@ impl Compiler { } // Validate rest pattern: '_' cannot be used as a rest target - if let Some(rest) = star_target { - if rest.as_str() == "_" { - return Err(self.error(CodegenErrorType::SyntaxError("invalid syntax".to_string()))); - } + if let Some(rest) = star_target + && rest.as_str() == "_" + { + return Err(self.error(CodegenErrorType::SyntaxError("invalid syntax".to_string()))); } // Step 1: Check if subject is a mapping @@ -3593,7 +3610,7 @@ impl Compiler { // Step 2: If we have keys to match if size > 0 { // Validate and compile keys - let mut seen = std::collections::HashSet::new(); + let mut seen = HashSet::new(); for key in keys { let is_attribute = matches!(key, Expr::Attribute(_)); let is_literal = matches!( @@ -4668,10 +4685,12 @@ impl Compiler { Expr::Lambda(ExprLambda { parameters, body, .. }) => { - let prev_ctx = self.ctx; - let name = "<lambda>".to_owned(); let default_params = Parameters::default(); let params = parameters.as_deref().unwrap_or(&default_params); + validate_duplicate_params(params).map_err(|e| self.error(e))?; + + let prev_ctx = self.ctx; + let name = "<lambda>".to_owned(); // Prepare defaults before entering function let defaults: Vec<_> = std::iter::empty() @@ -4884,6 +4903,7 @@ impl Compiler { Expr::Named(ExprNamed { target, value, + node_index: _, range: _, }) => { self.compile_expression(value)?; @@ -4893,6 +4913,9 @@ impl Compiler { Expr::FString(fstring) => { self.compile_expr_fstring(fstring)?; } + Expr::TString(_) => { + return Err(self.error(CodegenErrorType::NotImplementedYet)); + } Expr::StringLiteral(string) => { let value = string.value.to_str(); if value.contains(char::REPLACEMENT_CHARACTER) { @@ -5346,7 +5369,7 @@ impl Compiler { let location = self .source_file .to_source_code() - .source_location(range.start()); + .source_location(range.start(), PositionEncoding::Utf8); // TODO: insert source filename self.current_block().instructions.push(ir::InstructionInfo { instr, @@ -5388,11 +5411,11 @@ impl Compiler { } fn emit_return_value(&mut self) { - if let Some(inst) = self.current_block().instructions.last_mut() { - if let Instruction::LoadConst { idx } = inst.instr { - inst.instr = Instruction::ReturnConst { idx }; - return; - } + if let Some(inst) = self.current_block().instructions.last_mut() + && let Instruction::LoadConst { idx } = inst.instr + { + inst.instr = Instruction::ReturnConst { idx }; + return; } emit!(self, Instruction::ReturnValue) } @@ -5536,27 +5559,14 @@ impl Compiler { Expr::Named(ExprNamed { target, value, + node_index: _, range: _, }) => Self::contains_await(target) || Self::contains_await(value), - Expr::FString(ExprFString { value, range: _ }) => { - fn expr_element_contains_await<F: Copy + Fn(&Expr) -> bool>( - expr_element: &FStringExpressionElement, - contains_await: F, - ) -> bool { - contains_await(&expr_element.expression) - || expr_element - .format_spec - .iter() - .flat_map(|spec| spec.elements.expressions()) - .any(|element| expr_element_contains_await(element, contains_await)) - } - - value.elements().any(|element| match element { - FStringElement::Expression(expr_element) => { - expr_element_contains_await(expr_element, Self::contains_await) - } - FStringElement::Literal(_) => false, - }) + Expr::FString(fstring) => { + Self::interpolated_string_contains_await(fstring.value.elements()) + } + Expr::TString(tstring) => { + Self::interpolated_string_contains_await(tstring.value.elements()) } Expr::StringLiteral(_) | Expr::BytesLiteral(_) @@ -5568,6 +5578,29 @@ impl Compiler { } } + fn interpolated_string_contains_await<'a>( + mut elements: impl Iterator<Item = &'a InterpolatedStringElement>, + ) -> bool { + fn interpolated_element_contains_await<F: Copy + Fn(&Expr) -> bool>( + expr_element: &InterpolatedElement, + contains_await: F, + ) -> bool { + contains_await(&expr_element.expression) + || expr_element + .format_spec + .iter() + .flat_map(|spec| spec.elements.interpolations()) + .any(|element| interpolated_element_contains_await(element, contains_await)) + } + + elements.any(|element| match element { + InterpolatedStringElement::Interpolation(expr_element) => { + interpolated_element_contains_await(expr_element, Self::contains_await) + } + InterpolatedStringElement::Literal(_) => false, + }) + } + fn compile_expr_fstring(&mut self, fstring: &ExprFString) -> CompileResult<()> { let fstring = &fstring.value; for part in fstring { @@ -5614,13 +5647,13 @@ impl Compiler { fn compile_fstring_elements( &mut self, flags: FStringFlags, - fstring_elements: &FStringElements, + fstring_elements: &InterpolatedStringElements, ) -> CompileResult<()> { let mut element_count = 0; for element in fstring_elements { element_count += 1; match element { - FStringElement::Literal(string) => { + InterpolatedStringElement::Literal(string) => { if string.value.contains(char::REPLACEMENT_CHARACTER) { // might have a surrogate literal; should reparse to be sure let source = self.source_file.slice(string.range); @@ -5637,7 +5670,7 @@ impl Compiler { }); } } - FStringElement::Expression(fstring_expr) => { + InterpolatedStringElement::Interpolation(fstring_expr) => { let mut conversion = fstring_expr.conversion; if let Some(DebugText { leading, trailing }) = &fstring_expr.debug_text { @@ -5870,21 +5903,27 @@ mod ruff_tests { // f'{x}' let expr_x = Expr::Name(ExprName { + node_index: AtomicNodeIndex::NONE, range, id: Name::new("x"), ctx: ExprContext::Load, }); let not_present = &Expr::FString(ExprFString { + node_index: AtomicNodeIndex::NONE, range, value: FStringValue::single(FString { + node_index: AtomicNodeIndex::NONE, range, - elements: vec![FStringElement::Expression(FStringExpressionElement { - range, - expression: Box::new(expr_x), - debug_text: None, - conversion: ConversionFlag::None, - format_spec: None, - })] + elements: vec![InterpolatedStringElement::Interpolation( + InterpolatedElement { + node_index: AtomicNodeIndex::NONE, + range, + expression: Box::new(expr_x), + debug_text: None, + conversion: ConversionFlag::None, + format_spec: None, + }, + )] .into(), flags, }), @@ -5893,24 +5932,31 @@ mod ruff_tests { // f'{await x}' let expr_await_x = Expr::Await(ExprAwait { + node_index: AtomicNodeIndex::NONE, range, value: Box::new(Expr::Name(ExprName { + node_index: AtomicNodeIndex::NONE, range, id: Name::new("x"), ctx: ExprContext::Load, })), }); let present = &Expr::FString(ExprFString { + node_index: AtomicNodeIndex::NONE, range, value: FStringValue::single(FString { + node_index: AtomicNodeIndex::NONE, range, - elements: vec![FStringElement::Expression(FStringExpressionElement { - range, - expression: Box::new(expr_await_x), - debug_text: None, - conversion: ConversionFlag::None, - format_spec: None, - })] + elements: vec![InterpolatedStringElement::Interpolation( + InterpolatedElement { + node_index: AtomicNodeIndex::NONE, + range, + expression: Box::new(expr_await_x), + debug_text: None, + conversion: ConversionFlag::None, + format_spec: None, + }, + )] .into(), flags, }), @@ -5919,39 +5965,51 @@ mod ruff_tests { // f'{x:{await y}}' let expr_x = Expr::Name(ExprName { + node_index: AtomicNodeIndex::NONE, range, id: Name::new("x"), ctx: ExprContext::Load, }); let expr_await_y = Expr::Await(ExprAwait { + node_index: AtomicNodeIndex::NONE, range, value: Box::new(Expr::Name(ExprName { + node_index: AtomicNodeIndex::NONE, range, id: Name::new("y"), ctx: ExprContext::Load, })), }); let present = &Expr::FString(ExprFString { + node_index: AtomicNodeIndex::NONE, range, value: FStringValue::single(FString { + node_index: AtomicNodeIndex::NONE, range, - elements: vec![FStringElement::Expression(FStringExpressionElement { - range, - expression: Box::new(expr_x), - debug_text: None, - conversion: ConversionFlag::None, - format_spec: Some(Box::new(FStringFormatSpec { + elements: vec![InterpolatedStringElement::Interpolation( + InterpolatedElement { + node_index: AtomicNodeIndex::NONE, range, - elements: vec![FStringElement::Expression(FStringExpressionElement { + expression: Box::new(expr_x), + debug_text: None, + conversion: ConversionFlag::None, + format_spec: Some(Box::new(InterpolatedStringFormatSpec { + node_index: AtomicNodeIndex::NONE, range, - expression: Box::new(expr_await_y), - debug_text: None, - conversion: ConversionFlag::None, - format_spec: None, - })] - .into(), - })), - })] + elements: vec![InterpolatedStringElement::Interpolation( + InterpolatedElement { + node_index: AtomicNodeIndex::NONE, + range, + expression: Box::new(expr_await_y), + debug_text: None, + conversion: ConversionFlag::None, + format_spec: None, + }, + )] + .into(), + })), + }, + )] .into(), flags, }), diff --git a/compiler/codegen/src/ir.rs b/compiler/codegen/src/ir.rs index 1cc59dd656..7cd173c821 100644 --- a/compiler/codegen/src/ir.rs +++ b/compiler/codegen/src/ir.rs @@ -5,7 +5,7 @@ use rustpython_compiler_core::{ OneIndexed, SourceLocation, bytecode::{ CodeFlags, CodeObject, CodeUnit, ConstantData, InstrDisplayContext, Instruction, Label, - OpArg, + OpArg, PyCodeLocationInfoKind, }, }; @@ -72,6 +72,7 @@ pub struct InstructionInfo { pub target: BlockIdx, // pub range: TextRange, pub location: SourceLocation, + // TODO: end_location for debug ranges } // spell-checker:ignore petgraph @@ -181,7 +182,7 @@ impl CodeInfo { *arg = new_arg; } let (extras, lo_arg) = arg.split(); - locations.extend(std::iter::repeat_n(info.location.clone(), arg.instr_size())); + locations.extend(std::iter::repeat_n(info.location, arg.instr_size())); instructions.extend( extras .map(|byte| CodeUnit::new(Instruction::ExtendedArg, byte)) @@ -199,6 +200,9 @@ impl CodeInfo { locations.clear() } + // Generate linetable from locations + let linetable = generate_linetable(&locations, first_line_number.get() as i32); + Ok(CodeObject { flags, posonlyarg_count, @@ -218,6 +222,8 @@ impl CodeInfo { cellvars: cellvar_cache.into_iter().collect(), freevars: freevar_cache.into_iter().collect(), cell2arg, + linetable, + exceptiontable: Box::new([]), // TODO: Generate actual exception table }) } @@ -388,3 +394,134 @@ fn iter_blocks(blocks: &[Block]) -> impl Iterator<Item = (BlockIdx, &Block)> + ' Some((idx, b)) }) } + +/// Generate CPython 3.11+ format linetable from source locations +fn generate_linetable(locations: &[SourceLocation], first_line: i32) -> Box<[u8]> { + if locations.is_empty() { + return Box::new([]); + } + + let mut linetable = Vec::new(); + // Initialize prev_line to first_line + // The first entry's delta is relative to co_firstlineno + let mut prev_line = first_line; + let mut i = 0; + + while i < locations.len() { + let loc = &locations[i]; + + // Count consecutive instructions with the same location + let mut length = 1; + while i + length < locations.len() && locations[i + length] == locations[i] { + length += 1; + } + + // Process in chunks of up to 8 instructions + while length > 0 { + let entry_length = length.min(8); + + // Get line and column information + // SourceLocation always has row and column (both are OneIndexed) + let line = loc.line.get() as i32; + let col = loc.character_offset.to_zero_indexed() as i32; + + let line_delta = line - prev_line; + + // Choose the appropriate encoding based on line delta and column info + // Note: SourceLocation always has valid column, so we never get NO_COLUMNS case + if line_delta == 0 { + let end_col = col; // Use same column for end (no range info available) + + if col < 80 && end_col - col < 16 && end_col >= col { + // Short form (codes 0-9) for common cases + let code = (col / 8).min(9) as u8; // Short0 to Short9 + linetable.push(0x80 | (code << 3) | ((entry_length - 1) as u8)); + let col_byte = (((col % 8) as u8) << 4) | ((end_col - col) as u8 & 0xf); + linetable.push(col_byte); + } else if col < 128 && end_col < 128 { + // One-line form (code 10) for same line + linetable.push( + 0x80 | ((PyCodeLocationInfoKind::OneLine0 as u8) << 3) + | ((entry_length - 1) as u8), + ); + linetable.push(col as u8); + linetable.push(end_col as u8); + } else { + // Long form for columns >= 128 + linetable.push( + 0x80 | ((PyCodeLocationInfoKind::Long as u8) << 3) + | ((entry_length - 1) as u8), + ); + write_signed_varint(&mut linetable, 0); // line_delta = 0 + write_varint(&mut linetable, 0); // end_line delta = 0 + write_varint(&mut linetable, (col as u32) + 1); // column + 1 for encoding + write_varint(&mut linetable, (end_col as u32) + 1); // end_col + 1 + } + } else if line_delta > 0 && line_delta < 3 + /* && column.is_some() */ + { + // One-line form (codes 11-12) for line deltas 1-2 + let end_col = col; // Use same column for end + + if col < 128 && end_col < 128 { + let code = (PyCodeLocationInfoKind::OneLine0 as u8) + (line_delta as u8); // 11 for delta=1, 12 for delta=2 + linetable.push(0x80 | (code << 3) | ((entry_length - 1) as u8)); + linetable.push(col as u8); + linetable.push(end_col as u8); + } else { + // Long form for columns >= 128 or negative line delta + linetable.push( + 0x80 | ((PyCodeLocationInfoKind::Long as u8) << 3) + | ((entry_length - 1) as u8), + ); + write_signed_varint(&mut linetable, line_delta); + write_varint(&mut linetable, 0); // end_line delta = 0 + write_varint(&mut linetable, (col as u32) + 1); // column + 1 for encoding + write_varint(&mut linetable, (end_col as u32) + 1); // end_col + 1 + } + } else { + // Long form (code 14) for all other cases + // This handles: line_delta < 0, line_delta >= 3, or columns >= 128 + let end_col = col; // Use same column for end + linetable.push( + 0x80 | ((PyCodeLocationInfoKind::Long as u8) << 3) | ((entry_length - 1) as u8), + ); + write_signed_varint(&mut linetable, line_delta); + write_varint(&mut linetable, 0); // end_line delta = 0 + write_varint(&mut linetable, (col as u32) + 1); // column + 1 for encoding + write_varint(&mut linetable, (end_col as u32) + 1); // end_col + 1 + } + + prev_line = line; + length -= entry_length; + i += entry_length; + } + } + + linetable.into_boxed_slice() +} + +/// Write a variable-length unsigned integer (6-bit chunks) +/// Returns the number of bytes written +fn write_varint(buf: &mut Vec<u8>, mut val: u32) -> usize { + let start_len = buf.len(); + while val >= 64 { + buf.push(0x40 | (val & 0x3f) as u8); + val >>= 6; + } + buf.push(val as u8); + buf.len() - start_len +} + +/// Write a variable-length signed integer +/// Returns the number of bytes written +fn write_signed_varint(buf: &mut Vec<u8>, val: i32) -> usize { + let uval = if val < 0 { + // (unsigned int)(-val) has an undefined behavior for INT_MIN + // So we use (0 - val as u32) to handle it correctly + ((0u32.wrapping_sub(val as u32)) << 1) | 1 + } else { + (val as u32) << 1 + }; + write_varint(buf, uval) +} diff --git a/compiler/codegen/src/lib.rs b/compiler/codegen/src/lib.rs index b936ffbd1a..e794407998 100644 --- a/compiler/codegen/src/lib.rs +++ b/compiler/codegen/src/lib.rs @@ -55,6 +55,7 @@ impl ToPythonName for Expr { Self::Starred { .. } => "starred", Self::Slice { .. } => "slice", Self::FString { .. } => "f-string expression", + Self::TString { .. } => "t-string expression", Self::Name { .. } => "name", Self::Lambda { .. } => "lambda", Self::If { .. } => "conditional expression", diff --git a/compiler/codegen/src/symboltable.rs b/compiler/codegen/src/symboltable.rs index 7bed678aaf..0464e09c13 100644 --- a/compiler/codegen/src/symboltable.rs +++ b/compiler/codegen/src/symboltable.rs @@ -19,7 +19,7 @@ use ruff_python_ast::{ Stmt, TypeParam, TypeParamParamSpec, TypeParamTypeVar, TypeParamTypeVarTuple, TypeParams, }; use ruff_text_size::{Ranged, TextRange}; -use rustpython_compiler_core::{SourceFile, SourceLocation}; +use rustpython_compiler_core::{PositionEncoding, SourceFile, SourceLocation}; use std::{borrow::Cow, fmt}; /// Captures all symbols in the current scope, and has a list of sub-scopes in this scope. @@ -240,21 +240,21 @@ fn analyze_symbol_table(symbol_table: &mut SymbolTable) -> SymbolTableResult { */ fn drop_class_free(symbol_table: &mut SymbolTable) { // Check if __class__ is used as a free variable - if let Some(class_symbol) = symbol_table.symbols.get("__class__") { - if class_symbol.scope == SymbolScope::Free { - symbol_table.needs_class_closure = true; - // Note: In CPython, the symbol is removed from the free set, - // but in RustPython we handle this differently during code generation - } + if let Some(class_symbol) = symbol_table.symbols.get("__class__") + && class_symbol.scope == SymbolScope::Free + { + symbol_table.needs_class_closure = true; + // Note: In CPython, the symbol is removed from the free set, + // but in RustPython we handle this differently during code generation } // Check if __classdict__ is used as a free variable - if let Some(classdict_symbol) = symbol_table.symbols.get("__classdict__") { - if classdict_symbol.scope == SymbolScope::Free { - symbol_table.needs_classdict = true; - // Note: In CPython, the symbol is removed from the free set, - // but in RustPython we handle this differently during code generation - } + if let Some(classdict_symbol) = symbol_table.symbols.get("__classdict__") + && classdict_symbol.scope == SymbolScope::Free + { + symbol_table.needs_classdict = true; + // Note: In CPython, the symbol is removed from the free set, + // but in RustPython we handle this differently during code generation } } @@ -733,12 +733,12 @@ impl SymbolTableBuilder { fn scan_statement(&mut self, statement: &Stmt) -> SymbolTableResult { use ruff_python_ast::*; - if let Stmt::ImportFrom(StmtImportFrom { module, names, .. }) = &statement { - if module.as_ref().map(|id| id.as_str()) == Some("__future__") { - for feature in names { - if &feature.name == "annotations" { - self.future_annotations = true; - } + if let Stmt::ImportFrom(StmtImportFrom { module, names, .. }) = &statement + && module.as_ref().map(|id| id.as_str()) == Some("__future__") + { + for feature in names { + if &feature.name == "annotations" { + self.future_annotations = true; } } } @@ -793,6 +793,7 @@ impl SymbolTableBuilder { decorator_list, type_params, range, + node_index: _, }) => { if let Some(type_params) = type_params { self.enter_type_param_block( @@ -910,6 +911,7 @@ impl SymbolTableBuilder { value, simple, range, + node_index: _, }) => { // https://github.com/python/cpython/blob/main/Python/symtable.c#L1233 match &**target { @@ -1032,26 +1034,23 @@ impl SymbolTableBuilder { use ruff_python_ast::*; // Check for expressions not allowed in type parameters scope - if let Some(table) = self.tables.last() { - if table.typ == CompilerScope::TypeParams { - if let Some(keyword) = match expression { - Expr::Yield(_) | Expr::YieldFrom(_) => Some("yield"), - Expr::Await(_) => Some("await"), - Expr::Named(_) => Some("named"), - _ => None, - } { - return Err(SymbolTableError { - error: format!( - "{keyword} expression cannot be used within a type parameter" - ), - location: Some( - self.source_file - .to_source_code() - .source_location(expression.range().start()), - ), - }); - } + if let Some(table) = self.tables.last() + && table.typ == CompilerScope::TypeParams + && let Some(keyword) = match expression { + Expr::Yield(_) | Expr::YieldFrom(_) => Some("yield"), + Expr::Await(_) => Some("await"), + Expr::Named(_) => Some("named"), + _ => None, } + { + return Err(SymbolTableError { + error: format!("{keyword} expression cannot be used within a type parameter"), + location: Some( + self.source_file + .to_source_code() + .source_location(expression.range().start(), PositionEncoding::Utf8), + ), + }); } match expression { @@ -1092,7 +1091,11 @@ impl SymbolTableBuilder { }) => { self.scan_expression(value, ExpressionContext::Load)?; } - Expr::Dict(ExprDict { items, range: _ }) => { + Expr::Dict(ExprDict { + items, + node_index: _, + range: _, + }) => { for item in items { if let Some(key) = &item.key { self.scan_expression(key, context)?; @@ -1100,15 +1103,27 @@ impl SymbolTableBuilder { self.scan_expression(&item.value, context)?; } } - Expr::Await(ExprAwait { value, range: _ }) => { + Expr::Await(ExprAwait { + value, + node_index: _, + range: _, + }) => { self.scan_expression(value, context)?; } - Expr::Yield(ExprYield { value, range: _ }) => { + Expr::Yield(ExprYield { + value, + node_index: _, + range: _, + }) => { if let Some(expression) = value { self.scan_expression(expression, context)?; } } - Expr::YieldFrom(ExprYieldFrom { value, range: _ }) => { + Expr::YieldFrom(ExprYieldFrom { + value, + node_index: _, + range: _, + }) => { self.scan_expression(value, context)?; } Expr::UnaryOp(ExprUnaryOp { @@ -1130,6 +1145,7 @@ impl SymbolTableBuilder { lower, upper, step, + node_index: _, range: _, }) => { if let Some(lower) = lower { @@ -1154,6 +1170,7 @@ impl SymbolTableBuilder { elt, generators, range, + node_index: _, }) => { self.scan_comprehension("genexpr", elt, None, generators, *range)?; } @@ -1161,6 +1178,7 @@ impl SymbolTableBuilder { elt, generators, range, + node_index: _, }) => { self.scan_comprehension("genexpr", elt, None, generators, *range)?; } @@ -1169,12 +1187,14 @@ impl SymbolTableBuilder { value, generators, range, + node_index: _, }) => { self.scan_comprehension("genexpr", key, Some(value), generators, *range)?; } Expr::Call(ExprCall { func, arguments, + node_index: _, range: _, }) => { match context { @@ -1221,6 +1241,7 @@ impl SymbolTableBuilder { Expr::Lambda(ExprLambda { body, parameters, + node_index: _, range: _, }) => { if let Some(parameters) = parameters { @@ -1247,15 +1268,25 @@ impl SymbolTableBuilder { self.leave_scope(); } Expr::FString(ExprFString { value, .. }) => { - for expr in value.elements().filter_map(|x| x.as_expression()) { + for expr in value.elements().filter_map(|x| x.as_interpolation()) { self.scan_expression(&expr.expression, ExpressionContext::Load)?; if let Some(format_spec) = &expr.format_spec { - for element in format_spec.elements.expressions() { + for element in format_spec.elements.interpolations() { self.scan_expression(&element.expression, ExpressionContext::Load)? } } } } + Expr::TString(tstring) => { + return Err(SymbolTableError { + error: "not yet implemented".into(), + location: Some( + self.source_file + .to_source_code() + .source_location(tstring.range.start(), PositionEncoding::Utf8), + ), + }); + } // Constants Expr::StringLiteral(_) | Expr::BytesLiteral(_) @@ -1268,6 +1299,7 @@ impl SymbolTableBuilder { test, body, orelse, + node_index: _, range: _, }) => { self.scan_expression(test, ExpressionContext::Load)?; @@ -1279,13 +1311,14 @@ impl SymbolTableBuilder { target, value, range, + node_index: _, }) => { // named expressions are not allowed in the definition of // comprehension iterator definitions if let ExpressionContext::IterDefinitionExp = context { return Err(SymbolTableError { error: "assignment expression cannot be used in a comprehension iterable expression".to_string(), - location: Some(self.source_file.to_source_code().source_location(target.range().start())), + location: Some(self.source_file.to_source_code().source_location(target.range().start(), PositionEncoding::Utf8)), }); } @@ -1396,6 +1429,7 @@ impl SymbolTableBuilder { bound, range: type_var_range, default, + node_index: _, }) => { self.register_name(name.as_str(), SymbolUsage::TypeParam, *type_var_range)?; @@ -1419,6 +1453,7 @@ impl SymbolTableBuilder { name, range: param_spec_range, default, + node_index: _, }) => { self.register_name(name, SymbolUsage::TypeParam, *param_spec_range)?; @@ -1432,6 +1467,7 @@ impl SymbolTableBuilder { name, range: type_var_tuple_range, default, + node_index: _, }) => { self.register_name(name, SymbolUsage::TypeParam, *type_var_tuple_range)?; @@ -1568,7 +1604,7 @@ impl SymbolTableBuilder { let location = self .source_file .to_source_code() - .source_location(range.start()); + .source_location(range.start(), PositionEncoding::Utf8); let location = Some(location); let scope_depth = self.tables.len(); let table = self.tables.last_mut().unwrap(); diff --git a/compiler/core/Cargo.toml b/compiler/core/Cargo.toml index 837f9e4866..e49c73eb14 100644 --- a/compiler/core/Cargo.toml +++ b/compiler/core/Cargo.toml @@ -17,9 +17,8 @@ bitflags = { workspace = true } itertools = { workspace = true } malachite-bigint = { workspace = true } num-complex = { workspace = true } -serde = { workspace = true, optional = true, default-features = false, features = ["derive"] } lz4_flex = "0.11" [lints] -workspace = true \ No newline at end of file +workspace = true diff --git a/compiler/core/src/bytecode.rs b/compiler/core/src/bytecode.rs index b38c599508..4ab666da99 100644 --- a/compiler/core/src/bytecode.rs +++ b/compiler/core/src/bytecode.rs @@ -33,6 +33,75 @@ pub enum ResumeType { AfterAwait = 3, } +/// CPython 3.11+ linetable location info codes +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(u8)] +pub enum PyCodeLocationInfoKind { + // Short forms are 0 to 9 + Short0 = 0, + Short1 = 1, + Short2 = 2, + Short3 = 3, + Short4 = 4, + Short5 = 5, + Short6 = 6, + Short7 = 7, + Short8 = 8, + Short9 = 9, + // One line forms are 10 to 12 + OneLine0 = 10, + OneLine1 = 11, + OneLine2 = 12, + NoColumns = 13, + Long = 14, + None = 15, +} + +impl PyCodeLocationInfoKind { + pub fn from_code(code: u8) -> Option<Self> { + match code { + 0 => Some(Self::Short0), + 1 => Some(Self::Short1), + 2 => Some(Self::Short2), + 3 => Some(Self::Short3), + 4 => Some(Self::Short4), + 5 => Some(Self::Short5), + 6 => Some(Self::Short6), + 7 => Some(Self::Short7), + 8 => Some(Self::Short8), + 9 => Some(Self::Short9), + 10 => Some(Self::OneLine0), + 11 => Some(Self::OneLine1), + 12 => Some(Self::OneLine2), + 13 => Some(Self::NoColumns), + 14 => Some(Self::Long), + 15 => Some(Self::None), + _ => Option::None, + } + } + + pub fn is_short(&self) -> bool { + (*self as u8) <= 9 + } + + pub fn short_column_group(&self) -> Option<u8> { + if self.is_short() { + Some(*self as u8) + } else { + Option::None + } + } + + pub fn one_line_delta(&self) -> Option<i32> { + match self { + Self::OneLine0 => Some(0), + Self::OneLine1 => Some(1), + Self::OneLine2 => Some(2), + _ => Option::None, + } + } +} + pub trait Constant: Sized { type Name: AsRef<str>; @@ -129,23 +198,27 @@ pub struct CodeObject<C: Constant = ConstantData> { pub instructions: Box<[CodeUnit]>, pub locations: Box<[SourceLocation]>, pub flags: CodeFlags, + /// Number of positional-only arguments pub posonlyarg_count: u32, - // Number of positional-only arguments pub arg_count: u32, pub kwonlyarg_count: u32, pub source_path: C::Name, pub first_line_number: Option<OneIndexed>, pub max_stackdepth: u32, + /// Name of the object that created this code object pub obj_name: C::Name, - // Name of the object that created this code object + /// Qualified name of the object (like CPython's co_qualname) pub qualname: C::Name, - // Qualified name of the object (like CPython's co_qualname) pub cell2arg: Option<Box<[i32]>>, pub constants: Box<[C]>, pub names: Box<[C::Name]>, pub varnames: Box<[C::Name]>, pub cellvars: Box<[C::Name]>, pub freevars: Box<[C::Name]>, + /// Line number table (CPython 3.11+ format) + pub linetable: Box<[u8]>, + /// Exception handling table + pub exceptiontable: Box<[u8]>, } bitflags! { @@ -221,6 +294,12 @@ impl OpArg { } } +impl From<u32> for OpArg { + fn from(raw: u32) -> Self { + Self(raw) + } +} + #[derive(Default, Copy, Clone)] #[repr(transparent)] pub struct OpArgState { @@ -1117,14 +1196,14 @@ impl<C: Constant> CodeObject<C> { level: usize, ) -> fmt::Result { let label_targets = self.label_targets(); - let line_digits = (3).max(self.locations.last().unwrap().row.to_string().len()); - let offset_digits = (4).max(self.instructions.len().to_string().len()); + let line_digits = (3).max(self.locations.last().unwrap().line.digits().get()); + let offset_digits = (4).max(1 + self.instructions.len().ilog10() as usize); let mut last_line = OneIndexed::MAX; let mut arg_state = OpArgState::default(); for (offset, &instruction) in self.instructions.iter().enumerate() { let (instruction, arg) = arg_state.get(instruction); // optional line number - let line = self.locations[offset].row; + let line = self.locations[offset].line; if line != last_line { if last_line != OneIndexed::MAX { writeln!(f)?; @@ -1202,6 +1281,8 @@ impl<C: Constant> CodeObject<C> { first_line_number: self.first_line_number, max_stackdepth: self.max_stackdepth, cell2arg: self.cell2arg, + linetable: self.linetable, + exceptiontable: self.exceptiontable, } } @@ -1232,6 +1313,8 @@ impl<C: Constant> CodeObject<C> { first_line_number: self.first_line_number, max_stackdepth: self.max_stackdepth, cell2arg: self.cell2arg.clone(), + linetable: self.linetable.clone(), + exceptiontable: self.exceptiontable.clone(), } } } diff --git a/compiler/core/src/lib.rs b/compiler/core/src/lib.rs index 0ce4a9defb..08cdc0ec21 100644 --- a/compiler/core/src/lib.rs +++ b/compiler/core/src/lib.rs @@ -8,4 +8,6 @@ mod mode; pub use mode::Mode; -pub use ruff_source_file::{LineIndex, OneIndexed, SourceFile, SourceFileBuilder, SourceLocation}; +pub use ruff_source_file::{ + LineIndex, OneIndexed, PositionEncoding, SourceFile, SourceFileBuilder, SourceLocation, +}; diff --git a/compiler/core/src/marshal.rs b/compiler/core/src/marshal.rs index ff82340c0e..5e16e59102 100644 --- a/compiler/core/src/marshal.rs +++ b/compiler/core/src/marshal.rs @@ -198,8 +198,8 @@ pub fn deserialize_code<R: Read, Bag: ConstantBag>( let locations = (0..len) .map(|_| { Ok(SourceLocation { - row: OneIndexed::new(rdr.read_u32()? as _).ok_or(MarshalError::InvalidLocation)?, - column: OneIndexed::from_zero_indexed(rdr.read_u32()? as _), + line: OneIndexed::new(rdr.read_u32()? as _).ok_or(MarshalError::InvalidLocation)?, + character_offset: OneIndexed::from_zero_indexed(rdr.read_u32()? as _), }) }) .collect::<Result<Box<[SourceLocation]>>>()?; @@ -251,6 +251,16 @@ pub fn deserialize_code<R: Read, Bag: ConstantBag>( let cellvars = read_names()?; let freevars = read_names()?; + // Read linetable and exceptiontable + let linetable_len = rdr.read_u32()?; + let linetable = rdr.read_slice(linetable_len)?.to_vec().into_boxed_slice(); + + let exceptiontable_len = rdr.read_u32()?; + let exceptiontable = rdr + .read_slice(exceptiontable_len)? + .to_vec() + .into_boxed_slice(); + Ok(CodeObject { instructions, locations, @@ -269,6 +279,8 @@ pub fn deserialize_code<R: Read, Bag: ConstantBag>( varnames, cellvars, freevars, + linetable, + exceptiontable, }) } @@ -644,8 +656,8 @@ pub fn serialize_code<W: Write, C: Constant>(buf: &mut W, code: &CodeObject<C>) write_len(buf, code.locations.len()); for loc in &*code.locations { - buf.write_u32(loc.row.get() as _); - buf.write_u32(loc.column.to_zero_indexed() as _); + buf.write_u32(loc.line.get() as _); + buf.write_u32(loc.character_offset.to_zero_indexed() as _); } buf.write_u16(code.flags.bits()); @@ -684,4 +696,8 @@ pub fn serialize_code<W: Write, C: Constant>(buf: &mut W, code: &CodeObject<C>) write_names(&code.varnames); write_names(&code.cellvars); write_names(&code.freevars); + + // Serialize linetable and exceptiontable + write_vec(buf, &code.linetable); + write_vec(buf, &code.exceptiontable); } diff --git a/compiler/literal/Cargo.toml b/compiler/literal/Cargo.toml index da55d107b3..bd6a269974 100644 --- a/compiler/literal/Cargo.toml +++ b/compiler/literal/Cargo.toml @@ -13,7 +13,7 @@ rustpython-wtf8 = { workspace = true } hexf-parse = "0.2.1" is-macro.workspace = true -lexical-parse-float = { version = "1.0.4", features = ["format"] } +lexical-parse-float = { version = "1.0.6", features = ["format"] } num-traits = { workspace = true } unic-ucd-category = { workspace = true } diff --git a/compiler/literal/src/float.rs b/compiler/literal/src/float.rs index b1fc4607b5..e2bc54a8f1 100644 --- a/compiler/literal/src/float.rs +++ b/compiler/literal/src/float.rs @@ -18,7 +18,7 @@ fn parse_inner(literal: &[u8]) -> Option<f64> { // lexical-core's format::PYTHON_STRING is inaccurate const PYTHON_STRING: u128 = NumberFormatBuilder::rebuild(PYTHON3_LITERAL) .no_special(false) - .build(); + .build_unchecked(); f64::from_lexical_with_options::<PYTHON_STRING>(literal, &Options::new()).ok() } diff --git a/compiler/src/lib.rs b/compiler/src/lib.rs index 111b9aec2e..84e64f3c27 100644 --- a/compiler/src/lib.rs +++ b/compiler/src/lib.rs @@ -1,4 +1,4 @@ -use ruff_source_file::{SourceFile, SourceFileBuilder, SourceLocation}; +use ruff_source_file::{PositionEncoding, SourceFile, SourceFileBuilder, SourceLocation}; use rustpython_codegen::{compile, symboltable}; pub use rustpython_codegen::compile::CompileOpts; @@ -46,7 +46,7 @@ impl CompileError { pub fn from_ruff_parse_error(error: parser::ParseError, source_file: &SourceFile) -> Self { let location = source_file .to_source_code() - .source_location(error.location.start()); + .source_location(error.location.start(), PositionEncoding::Utf8); Self::Parse(ParseError { error: error.error, raw_location: error.location, @@ -55,26 +55,18 @@ impl CompileError { }) } - pub fn location(&self) -> Option<SourceLocation> { + pub const fn location(&self) -> Option<SourceLocation> { match self { - Self::Codegen(codegen_error) => codegen_error.location.clone(), - Self::Parse(parse_error) => Some(parse_error.location.clone()), + Self::Codegen(codegen_error) => codegen_error.location, + Self::Parse(parse_error) => Some(parse_error.location), } } pub const fn python_location(&self) -> (usize, usize) { - match self { - Self::Codegen(codegen_error) => { - if let Some(location) = &codegen_error.location { - (location.row.get(), location.column.get()) - } else { - (0, 0) - } - } - Self::Parse(parse_error) => ( - parse_error.location.row.get(), - parse_error.location.column.get(), - ), + if let Some(location) = self.location() { + (location.line.get(), location.character_offset.get()) + } else { + (0, 0) } } diff --git a/derive-impl/src/compile_bytecode.rs b/derive-impl/src/compile_bytecode.rs index edd746f1e1..cdcc89b998 100644 --- a/derive-impl/src/compile_bytecode.rs +++ b/derive-impl/src/compile_bytecode.rs @@ -140,10 +140,10 @@ impl CompilationSource { let mut code_map = HashMap::new(); let paths = fs::read_dir(path) .or_else(|e| { - if cfg!(windows) { - if let Ok(real_path) = fs::read_to_string(path.canonicalize().unwrap()) { - return fs::read_dir(real_path.trim()); - } + if cfg!(windows) + && let Ok(real_path) = fs::read_to_string(path.canonicalize().unwrap()) + { + return fs::read_dir(real_path.trim()); } Err(e) }) @@ -195,14 +195,14 @@ impl CompilationSource { }) }; let code = compile_path(&path).or_else(|e| { - if cfg!(windows) { - if let Ok(real_path) = fs::read_to_string(path.canonicalize().unwrap()) { - let joined = path.parent().unwrap().join(real_path.trim()); - if joined.exists() { - return compile_path(&joined); - } else { - return Err(e); - } + if cfg!(windows) + && let Ok(real_path) = fs::read_to_string(path.canonicalize().unwrap()) + { + let joined = path.parent().unwrap().join(real_path.trim()); + if joined.exists() { + return compile_path(&joined); + } else { + return Err(e); } } Err(e) diff --git a/derive/Cargo.toml b/derive/Cargo.toml index 17c0810884..5ecd490cf9 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -14,8 +14,7 @@ proc-macro = true [dependencies] rustpython-compiler = { workspace = true } rustpython-derive-impl = { workspace = true } -proc-macro2 = { workspace = true } syn = { workspace = true } [lints] -workspace = true \ No newline at end of file +workspace = true diff --git a/jit/src/instructions.rs b/jit/src/instructions.rs index b207521c64..5acf4a53ed 100644 --- a/jit/src/instructions.rs +++ b/jit/src/instructions.rs @@ -175,10 +175,11 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { let target_block = self.get_or_create_block(label); // If the current block isn't terminated, jump: - if let Some(cur) = self.builder.current_block() { - if cur != target_block && self.builder.func.layout.last_inst(cur).is_none() { - self.builder.ins().jump(target_block, &[]); - } + if let Some(cur) = self.builder.current_block() + && cur != target_block + && self.builder.func.layout.last_inst(cur).is_none() + { + self.builder.ins().jump(target_block, &[]); } // Switch to the target block if self.builder.current_block() != Some(target_block) { @@ -207,10 +208,10 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { } // After processing, if the current block is unterminated, insert a trap or fallthrough - if let Some(cur) = self.builder.current_block() { - if self.builder.func.layout.last_inst(cur).is_none() { - self.builder.ins().trap(TrapCode::user(0).unwrap()); - } + if let Some(cur) = self.builder.current_block() + && self.builder.func.layout.last_inst(cur).is_none() + { + self.builder.ins().trap(TrapCode::user(0).unwrap()); } Ok(()) } diff --git a/pylib/build.rs b/pylib/build.rs index 553b97202a..f85c4b5cfb 100644 --- a/pylib/build.rs +++ b/pylib/build.rs @@ -9,15 +9,15 @@ fn main() { process_python_libs("./Lib/**/*"); } - if cfg!(windows) { - if let Ok(real_path) = std::fs::read_to_string("Lib") { - let canonicalized_path = std::fs::canonicalize(real_path) - .expect("failed to resolve RUSTPYTHONPATH during build time"); - println!( - "cargo:rustc-env=win_lib_path={}", - canonicalized_path.to_str().unwrap() - ); - } + if cfg!(windows) + && let Ok(real_path) = std::fs::read_to_string("Lib") + { + let canonicalized_path = std::fs::canonicalize(real_path) + .expect("failed to resolve RUSTPYTHONPATH during build time"); + println!( + "cargo:rustc-env=win_lib_path={}", + canonicalized_path.to_str().unwrap() + ); } } diff --git a/scripts/lib_updater.py b/scripts/lib_updater.py new file mode 100755 index 0000000000..8573705dd1 --- /dev/null +++ b/scripts/lib_updater.py @@ -0,0 +1,352 @@ +#!/usr/bin/env python +__doc__ = """ +This tool helps with updating test files from CPython. + +Examples +-------- +To move the patches found in `Lib/test/foo.py` to ` ~/cpython/Lib/test/foo.py` then write the contents back to `Lib/test/foo.py` + +>>> ./{fname} --from Lib/test/foo.py --to ~/cpython/Lib/test/foo.py -o Lib/test/foo.py + +You can run the same command without `-o` to override the `--from` path: + +>>> ./{fname} --from Lib/test/foo.py --to ~/cpython/Lib/test/foo.py + +To get a baseline of patches, you can alter the patches file with your favorite tool/script/etc and then reapply it with: + +>>> ./{fname} --from Lib/test/foo.py --show-patches -o my_patches.json + +(By default the output is set to print to stdout). + +When you want to apply your own patches: + +>>> ./{fname} -p my_patches.json --to Lib/test/foo.py +""".format(fname=__import__("os").path.basename(__file__)) + + +import argparse +import ast +import collections +import enum +import json +import pathlib +import re +import sys +import textwrap +import typing + +if typing.TYPE_CHECKING: + from collections.abc import Iterator + +type Patches = dict[str, dict[str, list["PatchSpec"]]] + +DEFAULT_INDENT = " " * 4 +COMMENT = "TODO: RUSTPYTHON" +UT = "unittest" + + +@enum.unique +class UtMethod(enum.StrEnum): + """ + UnitTest Method. + """ + + def _generate_next_value_(name, start, count, last_values) -> str: + return name[0].lower() + name[1:] + + def has_args(self) -> bool: + return self != self.ExpectedFailure + + def has_cond(self) -> bool: + return self.endswith(("If", "Unless")) + + ExpectedFailure = enum.auto() + ExpectedFailureIf = enum.auto() + ExpectedFailureIfWindows = enum.auto() + Skip = enum.auto() + SkipIf = enum.auto() + SkipUnless = enum.auto() + + +class PatchSpec(typing.NamedTuple): + """ + Attributes + ---------- + ut_method : UtMethod + unittest method. + cond : str, optional + `ut_method` condition. Relevant only for some of `ut_method` types. + reason : str, optional + Reason for why the test is patched in this way. + """ + + ut_method: UtMethod + cond: str | None = None + reason: str = "" + + @property + def _reason(self) -> str: + return f"{COMMENT}; {self.reason}".strip(" ;") + + @property + def _attr_node(self) -> ast.Attribute: + return ast.Attribute(value=ast.Name(id=UT), attr=self.ut_method) + + def as_ast_node(self) -> ast.Attribute | ast.Call: + if not self.ut_method.has_args(): + return self._attr_node + + args = [] + if self.cond: + args.append(ast.parse(self.cond).body[0].value) + args.append(ast.Constant(value=self._reason)) + + return ast.Call(func=self._attr_node, args=args, keywords=[]) + + def as_decorator(self) -> str: + unparsed = ast.unparse(self.as_ast_node()) + + if not self.ut_method.has_args(): + unparsed = f"{unparsed} # {self._reason}" + + return f"@{unparsed}" + + +class PatchEntry(typing.NamedTuple): + """ + Stores patch metadata. + + Attributes + ---------- + parent_class : str + Parent class of test. + test_name : str + Test name. + spec : PatchSpec + Patch spec. + """ + + parent_class: str + test_name: str + spec: PatchSpec + + @classmethod + def iter_patch_entires( + cls, tree: ast.Module, lines: list[str] + ) -> "Iterator[typing.Self]": + for cls_node, fn_node in iter_tests(tree): + parent_class = cls_node.name + for dec_node in fn_node.decorator_list: + if not isinstance(dec_node, (ast.Attribute, ast.Call)): + continue + + attr_node = ( + dec_node if isinstance(dec_node, ast.Attribute) else dec_node.func + ) + + if ( + isinstance(attr_node, ast.Name) + or getattr(attr_node.value, "id", None) != UT + ): + continue + + cond = None + try: + ut_method = UtMethod(attr_node.attr) + except ValueError: + continue + + # If our ut_method has args then, + # we need to search for a constant that contains our `COMMENT`. + # Otherwise we need to search it in the raw source code :/ + if ut_method.has_args(): + reason = next( + ( + node.value + for node in ast.walk(dec_node) + if isinstance(node, ast.Constant) + and isinstance(node.value, str) + and COMMENT in node.value + ), + None, + ) + + # If we didn't find a constant containing <COMMENT>, + # then we didn't put this decorator + if not reason: + continue + + if ut_method.has_cond(): + cond = ast.unparse(dec_node.args[0]) + else: + # Search first on decorator line, then in the line before + for line in lines[dec_node.lineno - 1 : dec_node.lineno - 3 : -1]: + if found := re.search(rf"{COMMENT}.?(.*)", line): + reason = found.group() + break + else: + # Didn't find our `COMMENT` :) + continue + + reason = reason.removeprefix(COMMENT).strip(";:, ") + spec = PatchSpec(ut_method, cond, reason) + yield cls(parent_class, fn_node.name, spec) + + +def iter_tests( + tree: ast.Module, +) -> "Iterator[tuple[ast.ClassDef, ast.FunctionDef | ast.AsyncFunctionDef]]": + for key, nodes in ast.iter_fields(tree): + if key != "body": + continue + + for cls_node in nodes: + if not isinstance(cls_node, ast.ClassDef): + continue + + for fn_node in cls_node.body: + if not isinstance(fn_node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + + yield (cls_node, fn_node) + + +def iter_patches(contents: str) -> "Iterator[PatchEntry]": + lines = contents.splitlines() + tree = ast.parse(contents) + yield from PatchEntry.iter_patch_entires(tree, lines) + + +def build_patch_dict(it: "Iterator[PatchEntry]") -> Patches: + patches = collections.defaultdict(lambda: collections.defaultdict(list)) + for entry in it: + patches[entry.parent_class][entry.test_name].append(entry.spec) + + return {k: dict(v) for k, v in patches.items()} + + +def iter_patch_lines(tree: ast.Module, patches: Patches) -> "Iterator[tuple[int, str]]": + cache = {} # Used in phase 2. Stores the end line location of a class name. + + # Phase 1: Iterate and mark existing tests + for cls_node, fn_node in iter_tests(tree): + cache[cls_node.name] = cls_node.end_lineno + specs = patches.get(cls_node.name, {}).pop(fn_node.name, None) + if not specs: + continue + + lineno = min( + (dec_node.lineno for dec_node in fn_node.decorator_list), + default=fn_node.lineno, + ) + indent = " " * fn_node.col_offset + patch_lines = "\n".join(spec.as_decorator() for spec in specs) + yield (lineno - 1, textwrap.indent(patch_lines, indent)) + + # Phase 2: Iterate and mark inhereted tests + for cls_name, tests in patches.items(): + lineno = cache.get(cls_name) + if not lineno: + print(f"WARNING: {cls_name} does not exist in remote file", file=sys.stderr) + continue + + for test_name, specs in tests.items(): + decorators = "\n".join(spec.as_decorator() for spec in specs) + patch_lines = f""" +{decorators} +def {test_name}(self): +{DEFAULT_INDENT}return super().{test_name}() +""".rstrip() + yield (lineno, textwrap.indent(patch_lines, DEFAULT_INDENT)) + + +def apply_patches(contents: str, patches: Patches) -> str: + tree = ast.parse(contents) + lines = contents.splitlines() + + modifications = list(iter_patch_lines(tree, patches)) + # Going in reverse to not distrupt the line offset + for lineno, patch in sorted(modifications, reverse=True): + lines.insert(lineno, patch) + + joined = "\n".join(lines) + return f"{joined}\n" + + +def write_output(data: str, dest: str) -> None: + if dest == "-": + print(data, end="") + return + + with open(dest, "w") as fd: + fd.write(data) + + +def build_argparse() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + + patches_group = parser.add_mutually_exclusive_group(required=True) + patches_group.add_argument( + "-p", + "--patches", + help="File path to file containing patches in a JSON format", + type=pathlib.Path, + ) + patches_group.add_argument( + "--from", + help="File to gather patches from", + dest="gather_from", + type=pathlib.Path, + ) + + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument( + "--to", + help="File to apply patches to", + type=pathlib.Path, + ) + group.add_argument( + "--show-patches", action="store_true", help="Show the patches and exit" + ) + + parser.add_argument( + "-o", "--output", default="-", help="Output file. Set to '-' for stdout" + ) + + return parser + + +if __name__ == "__main__": + parser = build_argparse() + args = parser.parse_args() + + if args.patches: + patches = { + cls_name: { + test_name: [ + PatchSpec(**spec)._replace(ut_method=UtMethod(spec["ut_method"])) + for spec in specs + ] + for test_name, specs in tests.items() + } + for cls_name, tests in json.loads(args.patches.read_text()).items() + } + else: + patches = build_patch_dict(iter_patches(args.gather_from.read_text())) + + if args.show_patches: + patches = { + cls_name: { + test_name: [spec._asdict() for spec in specs] + for test_name, specs in tests.items() + } + for cls_name, tests in patches.items() + } + output = json.dumps(patches, indent=4) + "\n" + write_output(output, args.output) + sys.exit(0) + + patched = apply_patches(args.to.read_text(), patches) + write_output(patched, args.output) diff --git a/src/shell.rs b/src/shell.rs index 358b4e6df7..d2e54c490a 100644 --- a/src/shell.rs +++ b/src/shell.rs @@ -1,7 +1,7 @@ mod helper; use rustpython_compiler::{ - CompileError, ParseError, parser::FStringErrorType, parser::LexicalErrorType, + CompileError, ParseError, parser::InterpolatedStringErrorType, parser::LexicalErrorType, parser::ParseErrorType, }; use rustpython_vm::{ @@ -52,7 +52,7 @@ fn shell_exec( Err(CompileError::Parse(ParseError { error: ParseErrorType::Lexical(LexicalErrorType::FStringError( - FStringErrorType::UnterminatedTripleQuotedString, + InterpolatedStringErrorType::UnterminatedTripleQuotedString, )), .. })) => ShellExecResult::ContinueLine, @@ -67,10 +67,11 @@ fn shell_exec( { let loc = raw_location.start().to_usize(); let mut iter = source.chars(); - if let Some(quote) = iter.nth(loc) { - if iter.next() == Some(quote) && iter.next() == Some(quote) { - return ShellExecResult::ContinueLine; - } + if let Some(quote) = iter.nth(loc) + && iter.next() == Some(quote) + && iter.next() == Some(quote) + { + return ShellExecResult::ContinueLine; } }; @@ -208,6 +209,13 @@ pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> { ReadlineResult::Eof => { break; } + #[cfg(unix)] + ReadlineResult::OsError(num) => { + let os_error = + vm.new_exception_msg(vm.ctx.exceptions.os_error.to_owned(), format!("{num:?}")); + vm.print_exception(os_error); + break; + } ReadlineResult::Other(err) => { eprintln!("Readline error: {err:?}"); break; diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index 5f8182ea15..fd6d3e8a59 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -22,7 +22,7 @@ tkinter = ["dep:tk-sys", "dep:tcl-sys"] [dependencies] # rustpython crates rustpython-derive = { workspace = true } -rustpython-vm = { workspace = true, default-features = false } +rustpython-vm = { workspace = true, default-features = false, features = ["compiler"]} rustpython-common = { workspace = true } ahash = { workspace = true } @@ -117,7 +117,6 @@ lzma-sys = "0.1" xz2 = "0.1" [target.'cfg(windows)'.dependencies] -junction = { workspace = true } paste = { workspace = true } schannel = { workspace = true } widestring = { workspace = true } diff --git a/stdlib/src/contextvars.rs b/stdlib/src/contextvars.rs index 62f7dfc73d..cb66b8511f 100644 --- a/stdlib/src/contextvars.rs +++ b/stdlib/src/contextvars.rs @@ -378,12 +378,11 @@ mod _contextvars { let ctx = ctxs.last()?; let cached_ptr = zelf.cached.as_ptr(); debug_assert!(!cached_ptr.is_null()); - if let Some(cached) = unsafe { &*cached_ptr } { - if zelf.cached_id.load(Ordering::SeqCst) == ctx.get_id() - && cached.idx + 1 == ctxs.len() - { - return Some(cached.object.clone()); - } + if let Some(cached) = unsafe { &*cached_ptr } + && zelf.cached_id.load(Ordering::SeqCst) == ctx.get_id() + && cached.idx + 1 == ctxs.len() + { + return Some(cached.object.clone()); } let vars = ctx.borrow_vars(); let obj = vars.get(zelf)?; diff --git a/stdlib/src/faulthandler.rs b/stdlib/src/faulthandler.rs index f358129c87..5c9196ad33 100644 --- a/stdlib/src/faulthandler.rs +++ b/stdlib/src/faulthandler.rs @@ -10,7 +10,7 @@ mod decl { stderr, " File \"{}\", line {} in {}", frame.code.source_path, - frame.current_location().row, + frame.current_location().line, frame.code.obj_name ) } diff --git a/stdlib/src/lib.rs b/stdlib/src/lib.rs index c22a95d16c..706ce0ef21 100644 --- a/stdlib/src/lib.rs +++ b/stdlib/src/lib.rs @@ -38,6 +38,7 @@ mod locale; mod math; #[cfg(unix)] mod mmap; +mod opcode; mod pyexpat; mod pystruct; mod random; @@ -135,6 +136,7 @@ pub fn get_module_inits() -> impl Iterator<Item = (Cow<'static, str>, StdlibInit "_json" => json::make_module, "math" => math::make_module, "pyexpat" => pyexpat::make_module, + "_opcode" => opcode::make_module, "_random" => random::make_module, "_statistics" => statistics::make_module, "_struct" => pystruct::make_module, diff --git a/stdlib/src/math.rs b/stdlib/src/math.rs index 7860a343b4..aa0097c11e 100644 --- a/stdlib/src/math.rs +++ b/stdlib/src/math.rs @@ -518,11 +518,11 @@ mod math { #[pyfunction] fn ceil(x: PyObjectRef, vm: &VirtualMachine) -> PyResult { let result_or_err = try_magic_method(identifier!(vm, __ceil__), vm, &x); - if result_or_err.is_err() { - if let Some(v) = x.try_float_opt(vm) { - let v = try_f64_to_bigint(v?.to_f64().ceil(), vm)?; - return Ok(vm.ctx.new_int(v).into()); - } + if result_or_err.is_err() + && let Some(v) = x.try_float_opt(vm) + { + let v = try_f64_to_bigint(v?.to_f64().ceil(), vm)?; + return Ok(vm.ctx.new_int(v).into()); } result_or_err } @@ -530,11 +530,11 @@ mod math { #[pyfunction] fn floor(x: PyObjectRef, vm: &VirtualMachine) -> PyResult { let result_or_err = try_magic_method(identifier!(vm, __floor__), vm, &x); - if result_or_err.is_err() { - if let Some(v) = x.try_float_opt(vm) { - let v = try_f64_to_bigint(v?.to_f64().floor(), vm)?; - return Ok(vm.ctx.new_int(v).into()); - } + if result_or_err.is_err() + && let Some(v) = x.try_float_opt(vm) + { + let v = try_f64_to_bigint(v?.to_f64().floor(), vm)?; + return Ok(vm.ctx.new_int(v).into()); } result_or_err } diff --git a/stdlib/src/mmap.rs b/stdlib/src/mmap.rs index ed92b74d2f..143bdcb43d 100644 --- a/stdlib/src/mmap.rs +++ b/stdlib/src/mmap.rs @@ -27,11 +27,9 @@ mod mmap { use nix::sys::stat::fstat; use nix::unistd; use num_traits::Signed; - use std::fs::File; + use rustpython_common::crt_fd; use std::io::{self, Write}; use std::ops::{Deref, DerefMut}; - #[cfg(unix)] - use std::os::unix::io::{FromRawFd, RawFd}; fn advice_try_from_i32(vm: &VirtualMachine, i: i32) -> PyResult<Advice> { Ok(match i { @@ -179,7 +177,7 @@ mod mmap { struct PyMmap { closed: AtomicCell<bool>, mmap: PyMutex<Option<MmapObj>>, - fd: RawFd, + fd: AtomicCell<i32>, offset: libc::off_t, size: AtomicCell<usize>, pos: AtomicCell<usize>, // relative to offset @@ -190,7 +188,7 @@ mod mmap { #[derive(FromArgs)] struct MmapNewArgs { #[pyarg(any)] - fileno: RawFd, + fileno: i32, #[pyarg(any)] length: isize, #[pyarg(any, default = MAP_SHARED)] @@ -340,7 +338,8 @@ mod mmap { } }; - if fd != -1 { + let fd = unsafe { crt_fd::Borrowed::try_borrow_raw(fd) }; + if let Ok(fd) = fd { let metadata = fstat(fd) .map_err(|err| io::Error::from_raw_os_error(err as i32).to_pyexception(vm))?; let file_len = metadata.st_size; @@ -366,19 +365,19 @@ mod mmap { let mmap_opt = mmap_opt.offset(offset.try_into().unwrap()).len(map_size); let (fd, mmap) = || -> std::io::Result<_> { - if fd == -1 { - let mmap = MmapObj::Write(mmap_opt.map_anon()?); - Ok((fd, mmap)) - } else { - let new_fd = unistd::dup(fd)?; + if let Ok(fd) = fd { + let new_fd: crt_fd::Owned = unistd::dup(fd)?.into(); let mmap = match access { AccessMode::Default | AccessMode::Write => { - MmapObj::Write(unsafe { mmap_opt.map_mut(fd) }?) + MmapObj::Write(unsafe { mmap_opt.map_mut(&new_fd) }?) } - AccessMode::Read => MmapObj::Read(unsafe { mmap_opt.map(fd) }?), - AccessMode::Copy => MmapObj::Write(unsafe { mmap_opt.map_copy(fd) }?), + AccessMode::Read => MmapObj::Read(unsafe { mmap_opt.map(&new_fd) }?), + AccessMode::Copy => MmapObj::Write(unsafe { mmap_opt.map_copy(&new_fd) }?), }; - Ok((new_fd, mmap)) + Ok((Some(new_fd), mmap)) + } else { + let mmap = MmapObj::Write(mmap_opt.map_anon()?); + Ok((None, mmap)) } }() .map_err(|e| e.to_pyexception(vm))?; @@ -386,7 +385,7 @@ mod mmap { let m_obj = Self { closed: AtomicCell::new(false), mmap: PyMutex::new(Some(mmap)), - fd, + fd: AtomicCell::new(fd.map_or(-1, |fd| fd.into_raw())), offset, size: AtomicCell::new(map_size), pos: AtomicCell::new(0), @@ -841,9 +840,8 @@ mod mmap { #[pymethod] fn size(&self, vm: &VirtualMachine) -> std::io::Result<PyIntRef> { - let new_fd = unistd::dup(self.fd)?; - let file = unsafe { File::from_raw_fd(new_fd) }; - let file_len = file.metadata()?.len(); + let fd = unsafe { crt_fd::Borrowed::try_borrow_raw(self.fd.load())? }; + let file_len = fstat(fd)?.st_size; Ok(PyInt::from(file_len).into_ref(&vm.ctx)) } diff --git a/stdlib/src/opcode.rs b/stdlib/src/opcode.rs new file mode 100644 index 0000000000..c355b59df9 --- /dev/null +++ b/stdlib/src/opcode.rs @@ -0,0 +1,282 @@ +pub(crate) use opcode::make_module; + +#[pymodule] +mod opcode { + use crate::vm::{ + AsObject, PyObjectRef, PyResult, VirtualMachine, + builtins::{PyBool, PyInt, PyIntRef, PyNone}, + bytecode::Instruction, + match_class, + }; + use std::ops::Deref; + + struct Opcode(Instruction); + + impl Deref for Opcode { + type Target = Instruction; + + fn deref(&self) -> &Self::Target { + &self.0 + } + } + + impl Opcode { + // https://github.com/python/cpython/blob/bcee1c322115c581da27600f2ae55e5439c027eb/Include/opcode_ids.h#L238 + const HAVE_ARGUMENT: i32 = 44; + + pub fn try_from_pyint(raw: PyIntRef, vm: &VirtualMachine) -> PyResult<Self> { + let instruction = raw + .try_to_primitive::<u8>(vm) + .and_then(|v| { + Instruction::try_from(v).map_err(|_| { + vm.new_exception_empty(vm.ctx.exceptions.value_error.to_owned()) + }) + }) + .map_err(|_| vm.new_value_error("invalid opcode or oparg"))?; + + Ok(Self(instruction)) + } + + /// https://github.com/python/cpython/blob/bcee1c322115c581da27600f2ae55e5439c027eb/Include/internal/pycore_opcode_metadata.h#L914-L916 + #[must_use] + pub const fn is_valid(opcode: i32) -> bool { + opcode >= 0 && opcode < 268 && opcode != 255 + } + + // All `has_*` methods below mimics + // https://github.com/python/cpython/blob/bcee1c322115c581da27600f2ae55e5439c027eb/Include/internal/pycore_opcode_metadata.h#L966-L1190 + + #[must_use] + pub const fn has_arg(opcode: i32) -> bool { + Self::is_valid(opcode) && opcode > Self::HAVE_ARGUMENT + } + + #[must_use] + pub const fn has_const(opcode: i32) -> bool { + Self::is_valid(opcode) && matches!(opcode, 83 | 103 | 240) + } + + #[must_use] + pub const fn has_name(opcode: i32) -> bool { + Self::is_valid(opcode) + && matches!( + opcode, + 63 | 66 + | 67 + | 74 + | 75 + | 82 + | 90 + | 91 + | 92 + | 93 + | 108 + | 113 + | 114 + | 259 + | 260 + | 261 + | 262 + ) + } + + #[must_use] + pub const fn has_jump(opcode: i32) -> bool { + Self::is_valid(opcode) + && matches!( + opcode, + 72 | 77 | 78 | 79 | 97 | 98 | 99 | 100 | 104 | 256 | 257 + ) + } + + #[must_use] + pub const fn has_free(opcode: i32) -> bool { + Self::is_valid(opcode) && matches!(opcode, 64 | 84 | 89 | 94 | 109) + } + + #[must_use] + pub const fn has_local(opcode: i32) -> bool { + Self::is_valid(opcode) + && matches!(opcode, 65 | 85 | 86 | 87 | 88 | 110 | 111 | 112 | 258 | 267) + } + + #[must_use] + pub const fn has_exc(opcode: i32) -> bool { + Self::is_valid(opcode) && matches!(opcode, 264..=266) + } + } + + #[pyattr] + const ENABLE_SPECIALIZATION: i8 = 1; + + #[derive(FromArgs)] + struct StackEffectArgs { + #[pyarg(positional)] + opcode: PyIntRef, + #[pyarg(positional, optional)] + oparg: Option<PyObjectRef>, + #[pyarg(named, optional)] + jump: Option<PyObjectRef>, + } + + #[pyfunction] + fn stack_effect(args: StackEffectArgs, vm: &VirtualMachine) -> PyResult<i32> { + let oparg = args + .oparg + .map(|v| { + if !v.fast_isinstance(vm.ctx.types.int_type) { + return Err(vm.new_type_error(format!( + "'{}' object cannot be interpreted as an integer", + v.class().name() + ))); + } + v.downcast_ref::<PyInt>() + .ok_or_else(|| vm.new_type_error(""))? + .try_to_primitive::<u32>(vm) + }) + .unwrap_or(Ok(0))?; + + let jump = args + .jump + .map(|v| { + match_class!(match v { + b @ PyBool => Ok(b.is(&vm.ctx.true_value)), + _n @ PyNone => Ok(false), + _ => { + Err(vm.new_value_error("stack_effect: jump must be False, True or None")) + } + }) + }) + .unwrap_or(Ok(false))?; + + let opcode = Opcode::try_from_pyint(args.opcode, vm)?; + + Ok(opcode.stack_effect(oparg.into(), jump)) + } + + #[pyfunction] + fn is_valid(opcode: i32) -> bool { + Opcode::is_valid(opcode) + } + + #[pyfunction] + fn has_arg(opcode: i32) -> bool { + Opcode::has_arg(opcode) + } + + #[pyfunction] + fn has_const(opcode: i32) -> bool { + Opcode::has_const(opcode) + } + + #[pyfunction] + fn has_name(opcode: i32) -> bool { + Opcode::has_name(opcode) + } + + #[pyfunction] + fn has_jump(opcode: i32) -> bool { + Opcode::has_jump(opcode) + } + + #[pyfunction] + fn has_free(opcode: i32) -> bool { + Opcode::has_free(opcode) + } + + #[pyfunction] + fn has_local(opcode: i32) -> bool { + Opcode::has_local(opcode) + } + + #[pyfunction] + fn has_exc(opcode: i32) -> bool { + Opcode::has_exc(opcode) + } + + #[pyfunction] + fn get_intrinsic1_descs(vm: &VirtualMachine) -> Vec<PyObjectRef> { + [ + "INTRINSIC_1_INVALID", + "INTRINSIC_PRINT", + "INTRINSIC_IMPORT_STAR", + "INTRINSIC_STOPITERATION_ERROR", + "INTRINSIC_ASYNC_GEN_WRAP", + "INTRINSIC_UNARY_POSITIVE", + "INTRINSIC_LIST_TO_TUPLE", + "INTRINSIC_TYPEVAR", + "INTRINSIC_PARAMSPEC", + "INTRINSIC_TYPEVARTUPLE", + "INTRINSIC_SUBSCRIPT_GENERIC", + "INTRINSIC_TYPEALIAS", + ] + .into_iter() + .map(|x| vm.ctx.new_str(x).into()) + .collect() + } + + #[pyfunction] + fn get_intrinsic2_descs(vm: &VirtualMachine) -> Vec<PyObjectRef> { + [ + "INTRINSIC_2_INVALID", + "INTRINSIC_PREP_RERAISE_STAR", + "INTRINSIC_TYPEVAR_WITH_BOUND", + "INTRINSIC_TYPEVAR_WITH_CONSTRAINTS", + "INTRINSIC_SET_FUNCTION_TYPE_PARAMS", + "INTRINSIC_SET_TYPEPARAM_DEFAULT", + ] + .into_iter() + .map(|x| vm.ctx.new_str(x).into()) + .collect() + } + + #[pyfunction] + fn get_nb_ops(vm: &VirtualMachine) -> Vec<PyObjectRef> { + [ + ("NB_ADD", "+"), + ("NB_AND", "&"), + ("NB_FLOOR_DIVIDE", "//"), + ("NB_LSHIFT", "<<"), + ("NB_MATRIX_MULTIPLY", "@"), + ("NB_MULTIPLY", "*"), + ("NB_REMAINDER", "%"), + ("NB_OR", "|"), + ("NB_POWER", "**"), + ("NB_RSHIFT", ">>"), + ("NB_SUBTRACT", "-"), + ("NB_TRUE_DIVIDE", "/"), + ("NB_XOR", "^"), + ("NB_INPLACE_ADD", "+="), + ("NB_INPLACE_AND", "&="), + ("NB_INPLACE_FLOOR_DIVIDE", "//="), + ("NB_INPLACE_LSHIFT", "<<="), + ("NB_INPLACE_MATRIX_MULTIPLY", "@="), + ("NB_INPLACE_MULTIPLY", "*="), + ("NB_INPLACE_REMAINDER", "%="), + ("NB_INPLACE_OR", "|="), + ("NB_INPLACE_POWER", "**="), + ("NB_INPLACE_RSHIFT", ">>="), + ("NB_INPLACE_SUBTRACT", "-="), + ("NB_INPLACE_TRUE_DIVIDE", "/="), + ("NB_INPLACE_XOR", "^="), + ] + .into_iter() + .map(|(a, b)| { + vm.ctx + .new_tuple(vec![vm.ctx.new_str(a).into(), vm.ctx.new_str(b).into()]) + .into() + }) + .collect() + } + + #[pyfunction] + fn get_executor(_code: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // TODO + Ok(vm.ctx.none()) + } + + #[pyfunction] + fn get_specialization_stats(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.none() + } +} diff --git a/stdlib/src/posixsubprocess.rs b/stdlib/src/posixsubprocess.rs index 744024e21f..7f418c8993 100644 --- a/stdlib/src/posixsubprocess.rs +++ b/stdlib/src/posixsubprocess.rs @@ -18,9 +18,8 @@ use std::{ io::prelude::*, marker::PhantomData, ops::Deref, - os::fd::FromRawFd, + os::fd::{AsFd, AsRawFd, BorrowedFd, IntoRawFd, OwnedFd, RawFd}, }; -use std::{fs::File, os::unix::io::AsRawFd}; use unistd::{Gid, Uid}; pub(crate) use _posixsubprocess::make_module; @@ -33,7 +32,7 @@ mod _posixsubprocess { use crate::vm::{PyResult, VirtualMachine, convert::IntoPyException}; #[pyfunction] - fn fork_exec(args: ForkExecArgs, vm: &VirtualMachine) -> PyResult<libc::pid_t> { + fn fork_exec(args: ForkExecArgs<'_>, vm: &VirtualMachine) -> PyResult<libc::pid_t> { if args.preexec_fn.is_some() { return Err(vm.new_not_implemented_error("preexec_fn not supported yet")); } @@ -59,7 +58,7 @@ mod _posixsubprocess { macro_rules! gen_args { ($($field:ident: $t:ty),*$(,)?) => { #[derive(FromArgs)] - struct ForkExecArgs { + struct ForkExecArgs<'fd> { $(#[pyarg(positional)] $field: $t,)* } }; @@ -121,21 +120,95 @@ impl CharPtrSlice<'_> { } } +#[derive(Copy, Clone)] +struct Fd(BorrowedFd<'static>); + +impl TryFromObject for Fd { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + match MaybeFd::try_from_object(vm, obj)? { + MaybeFd::Valid(fd) => Ok(fd), + MaybeFd::Invalid => Err(vm.new_value_error("invalid fd")), + } + } +} + +impl Write for Fd { + fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> { + Ok(unistd::write(self, buf)?) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +impl AsRawFd for Fd { + fn as_raw_fd(&self) -> RawFd { + self.0.as_raw_fd() + } +} + +impl IntoRawFd for Fd { + fn into_raw_fd(self) -> RawFd { + self.0.as_raw_fd() + } +} + +impl AsFd for Fd { + fn as_fd(&self) -> BorrowedFd<'_> { + self.0.as_fd() + } +} + +impl From<OwnedFd> for Fd { + fn from(fd: OwnedFd) -> Self { + Self(unsafe { BorrowedFd::borrow_raw(fd.into_raw_fd()) }) + } +} + +#[derive(Copy, Clone)] +enum MaybeFd { + Valid(Fd), + Invalid, +} + +impl TryFromObject for MaybeFd { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + let fd = i32::try_from_object(vm, obj)?; + Ok(if fd == -1 { + MaybeFd::Invalid + } else { + MaybeFd::Valid(Fd(unsafe { BorrowedFd::borrow_raw(fd) })) + }) + } +} + +impl AsRawFd for MaybeFd { + fn as_raw_fd(&self) -> RawFd { + match self { + MaybeFd::Valid(fd) => fd.as_raw_fd(), + MaybeFd::Invalid => -1, + } + } +} + +// impl + gen_args! { args: ArgSequence<CStrPathLike> /* list */, exec_list: ArgSequence<CStrPathLike> /* list */, close_fds: bool, - fds_to_keep: ArgSequence<i32>, + fds_to_keep: ArgSequence<BorrowedFd<'fd>>, cwd: Option<CStrPathLike>, env_list: Option<ArgSequence<CStrPathLike>>, - p2cread: i32, - p2cwrite: i32, - c2pread: i32, - c2pwrite: i32, - errread: i32, - errwrite: i32, - errpipe_read: i32, - errpipe_write: i32, + p2cread: MaybeFd, + p2cwrite: MaybeFd, + c2pread: MaybeFd, + c2pwrite: MaybeFd, + errread: MaybeFd, + errwrite: MaybeFd, + errpipe_read: Fd, + errpipe_write: Fd, restore_signals: bool, call_setsid: bool, pgid_to_set: libc::pid_t, @@ -154,13 +227,12 @@ struct ProcArgs<'a> { extra_groups: Option<&'a [Gid]>, } -fn exec(args: &ForkExecArgs, procargs: ProcArgs<'_>) -> ! { +fn exec(args: &ForkExecArgs<'_>, procargs: ProcArgs<'_>) -> ! { let mut ctx = ExecErrorContext::NoExec; match exec_inner(args, procargs, &mut ctx) { Ok(x) => match x {}, Err(e) => { - let mut pipe = - std::mem::ManuallyDrop::new(unsafe { File::from_raw_fd(args.errpipe_write) }); + let mut pipe = args.errpipe_write; let _ = write!(pipe, "OSError:{}:{}", e as i32, ctx.as_msg()); std::process::exit(255) } @@ -184,49 +256,59 @@ impl ExecErrorContext { } fn exec_inner( - args: &ForkExecArgs, + args: &ForkExecArgs<'_>, procargs: ProcArgs<'_>, ctx: &mut ExecErrorContext, ) -> nix::Result<Never> { for &fd in args.fds_to_keep.as_slice() { - if fd != args.errpipe_write { - posix::raw_set_inheritable(fd, true)? + if fd.as_raw_fd() != args.errpipe_write.as_raw_fd() { + posix::set_inheritable(fd, true)? } } for &fd in &[args.p2cwrite, args.c2pread, args.errread] { - if fd != -1 { + if let MaybeFd::Valid(fd) = fd { unistd::close(fd)?; } } unistd::close(args.errpipe_read)?; - let c2pwrite = if args.c2pwrite == 0 { - let fd = unistd::dup(args.c2pwrite)?; - posix::raw_set_inheritable(fd, true)?; - fd - } else { - args.c2pwrite + let c2pwrite = match args.c2pwrite { + MaybeFd::Valid(c2pwrite) if c2pwrite.as_raw_fd() == 0 => { + let fd = unistd::dup(c2pwrite)?; + posix::set_inheritable(fd.as_fd(), true)?; + MaybeFd::Valid(fd.into()) + } + fd => fd, }; let mut errwrite = args.errwrite; - while errwrite == 0 || errwrite == 1 { - errwrite = unistd::dup(errwrite)?; - posix::raw_set_inheritable(errwrite, true)?; + loop { + match errwrite { + MaybeFd::Valid(fd) if fd.as_raw_fd() == 0 || fd.as_raw_fd() == 1 => { + let fd = unistd::dup(fd)?; + posix::set_inheritable(fd.as_fd(), true)?; + errwrite = MaybeFd::Valid(fd.into()); + } + _ => break, + } } - let dup_into_stdio = |fd, io_fd| { - if fd == io_fd { - posix::raw_set_inheritable(fd, true) - } else if fd != -1 { - unistd::dup2(fd, io_fd).map(drop) - } else { - Ok(()) + fn dup_into_stdio<F>(fd: MaybeFd, io_fd: i32, dup2_stdio: F) -> nix::Result<()> + where + F: Fn(Fd) -> nix::Result<()>, + { + match fd { + MaybeFd::Valid(fd) if fd.as_raw_fd() == io_fd => { + posix::set_inheritable(fd.as_fd(), true) + } + MaybeFd::Valid(fd) => dup2_stdio(fd), + MaybeFd::Invalid => Ok(()), } - }; - dup_into_stdio(args.p2cread, 0)?; - dup_into_stdio(c2pwrite, 1)?; - dup_into_stdio(errwrite, 2)?; + } + dup_into_stdio(args.p2cread, 0, unistd::dup2_stdin)?; + dup_into_stdio(c2pwrite, 1, unistd::dup2_stdout)?; + dup_into_stdio(errwrite, 2, unistd::dup2_stderr)?; if let Some(ref cwd) = args.cwd { unistd::chdir(cwd.s.as_c_str()).inspect_err(|_| *ctx = ExecErrorContext::ChDir)? @@ -292,12 +374,16 @@ fn exec_inner( #[derive(Copy, Clone)] struct KeepFds<'a> { above: i32, - keep: &'a [i32], + keep: &'a [BorrowedFd<'a>], } impl KeepFds<'_> { fn should_keep(self, fd: i32) -> bool { - fd > self.above && self.keep.binary_search(&fd).is_err() + fd > self.above + && self + .keep + .binary_search_by_key(&fd, BorrowedFd::as_raw_fd) + .is_err() } } @@ -394,9 +480,13 @@ fn close_fds_brute_force(keep: KeepFds<'_>) { .ok() .flatten() .unwrap_or(256) as i32; - let fds = itertools::chain![Some(keep.above), keep.keep.iter().copied(), Some(max_fd)]; + let fds = itertools::chain![ + Some(keep.above), + keep.keep.iter().map(BorrowedFd::as_raw_fd), + Some(max_fd) + ]; for fd in fds.tuple_windows().flat_map(|(start, end)| start + 1..end) { - let _ = unistd::close(fd); + unsafe { libc::close(fd) }; } } diff --git a/stdlib/src/scproxy.rs b/stdlib/src/scproxy.rs index c63acf5343..f49b6890a6 100644 --- a/stdlib/src/scproxy.rs +++ b/stdlib/src/scproxy.rs @@ -86,23 +86,22 @@ mod _scproxy { .and_then(|v| v.downcast::<CFNumber>()) .and_then(|v| v.to_i32()) .unwrap_or(0); - if enabled { - if let Some(host) = proxy_dict + if enabled + && let Some(host) = proxy_dict .find(host_key) .and_then(|v| v.downcast::<CFString>()) + { + let h = std::borrow::Cow::<str>::from(&host); + let v = if let Some(port) = proxy_dict + .find(port_key) + .and_then(|v| v.downcast::<CFNumber>()) + .and_then(|v| v.to_i32()) { - let h = std::borrow::Cow::<str>::from(&host); - let v = if let Some(port) = proxy_dict - .find(port_key) - .and_then(|v| v.downcast::<CFNumber>()) - .and_then(|v| v.to_i32()) - { - format!("http://{h}:{port}") - } else { - format!("http://{h}") - }; - result.set_item(proto, vm.new_pyobj(v), vm)?; - } + format!("http://{h}:{port}") + } else { + format!("http://{h}") + }; + result.set_item(proto, vm.new_pyobj(v), vm)?; } Ok(()) }; diff --git a/stdlib/src/select.rs b/stdlib/src/select.rs index b19fecc9fb..c19052965b 100644 --- a/stdlib/src/select.rs +++ b/stdlib/src/select.rs @@ -244,10 +244,10 @@ mod decl { Either::A(f) => f, Either::B(i) => i as f64, }); - if let Some(timeout) = timeout { - if timeout < 0.0 { - return Err(vm.new_value_error("timeout must be positive")); - } + if let Some(timeout) = timeout + && timeout < 0.0 + { + return Err(vm.new_value_error("timeout must be positive")); } let deadline = timeout.map(|s| time::time(vm).unwrap() + s); diff --git a/stdlib/src/socket.rs b/stdlib/src/socket.rs index 50ee2b96fc..f3ae72b63f 100644 --- a/stdlib/src/socket.rs +++ b/stdlib/src/socket.rs @@ -2277,15 +2277,16 @@ mod _socket { 0, ))); } - if let c::AF_INET | c::AF_UNSPEC = af { - if let Ok(addr) = name.parse::<Ipv4Addr>() { - return Ok(SocketAddr::V4(net::SocketAddrV4::new(addr, 0))); - } + if let c::AF_INET | c::AF_UNSPEC = af + && let Ok(addr) = name.parse::<Ipv4Addr>() + { + return Ok(SocketAddr::V4(net::SocketAddrV4::new(addr, 0))); } - if matches!(af, c::AF_INET | c::AF_UNSPEC) && !name.contains('%') { - if let Ok(addr) = name.parse::<Ipv6Addr>() { - return Ok(SocketAddr::V6(net::SocketAddrV6::new(addr, 0, 0, 0))); - } + if matches!(af, c::AF_INET | c::AF_UNSPEC) + && !name.contains('%') + && let Ok(addr) = name.parse::<Ipv6Addr>() + { + return Ok(SocketAddr::V6(net::SocketAddrV6::new(addr, 0, 0, 0))); } let hints = dns_lookup::AddrInfoHints { address: af, diff --git a/stdlib/src/sqlite.rs b/stdlib/src/sqlite.rs index 15d3c80894..e19aca1f6d 100644 --- a/stdlib/src/sqlite.rs +++ b/stdlib/src/sqlite.rs @@ -74,8 +74,8 @@ mod _sqlite { }, sliceable::{SaturatedSliceIter, SliceableSequenceOp}, types::{ - AsMapping, AsNumber, AsSequence, Callable, Comparable, Constructor, Hashable, IterNext, - Iterable, PyComparisonOp, SelfIter, Unconstructible, + AsMapping, AsNumber, AsSequence, Callable, Comparable, Constructor, Hashable, + Initializer, IterNext, Iterable, PyComparisonOp, SelfIter, Unconstructible, }, utils::ToCString, }; @@ -571,10 +571,10 @@ mod _sqlite { unsafe extern "C" fn progress_callback(data: *mut c_void) -> c_int { let (callable, vm) = unsafe { (*data.cast::<Self>()).retrieve() }; - if let Ok(val) = callable.call((), vm) { - if let Ok(val) = val.is_true(vm) { - return val as c_int; - } + if let Ok(val) = callable.call((), vm) + && let Ok(val) = val.is_true(vm) + { + return val as c_int; } -1 } @@ -851,7 +851,31 @@ mod _sqlite { type Args = ConnectArgs; fn py_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult { - Ok(Self::new(args, vm)?.into_ref_with_type(vm, cls)?.into()) + let text_factory = PyStr::class(&vm.ctx).to_owned().into_object(); + + // For non-subclassed Connection, initialize in __new__ + // For subclassed Connection, leave db as None and require __init__ to be called + let is_base_class = cls.is(Connection::class(&vm.ctx).as_object()); + + let db = if is_base_class { + // Initialize immediately for base class + Some(Connection::initialize_db(&args, vm)?) + } else { + // For subclasses, require __init__ to be called + None + }; + + let conn = Self { + db: PyMutex::new(db), + detect_types: args.detect_types, + isolation_level: PyAtomicRef::from(args.isolation_level), + check_same_thread: args.check_same_thread, + thread_ident: std::thread::current().id(), + row_factory: PyAtomicRef::from(None), + text_factory: PyAtomicRef::from(text_factory), + }; + + Ok(conn.into_ref_with_type(vm, cls)?.into()) } } @@ -871,9 +895,25 @@ mod _sqlite { } } - #[pyclass(with(Constructor, Callable), flags(BASETYPE))] + impl Initializer for Connection { + type Args = ConnectArgs; + + fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + let mut guard = zelf.db.lock(); + if guard.is_some() { + // Already initialized + return Ok(()); + } + + let db = Self::initialize_db(&args, vm)?; + *guard = Some(db); + Ok(()) + } + } + + #[pyclass(with(Constructor, Callable, Initializer), flags(BASETYPE))] impl Connection { - fn new(args: ConnectArgs, vm: &VirtualMachine) -> PyResult<Self> { + fn initialize_db(args: &ConnectArgs, vm: &VirtualMachine) -> PyResult<Sqlite> { let path = args.database.to_cstring(vm)?; let db = Sqlite::from(SqliteRaw::open(path.as_ptr(), args.uri, vm)?); let timeout = (args.timeout * 1000.0) as c_int; @@ -881,17 +921,7 @@ mod _sqlite { if let Some(isolation_level) = &args.isolation_level { begin_statement_ptr_from_isolation_level(isolation_level, vm)?; } - let text_factory = PyStr::class(&vm.ctx).to_owned().into_object(); - - Ok(Self { - db: PyMutex::new(Some(db)), - detect_types: args.detect_types, - isolation_level: PyAtomicRef::from(args.isolation_level), - check_same_thread: args.check_same_thread, - thread_ident: std::thread::current().id(), - row_factory: PyAtomicRef::from(None), - text_factory: PyAtomicRef::from(text_factory), - }) + Ok(db) } fn db_lock(&self, vm: &VirtualMachine) -> PyResult<PyMappedMutexGuard<'_, Sqlite>> { @@ -908,7 +938,7 @@ mod _sqlite { } else { Err(new_programming_error( vm, - "Cannot operate on a closed database.".to_owned(), + "Base Connection.__init__ not called.".to_owned(), )) } } @@ -1459,6 +1489,8 @@ mod _sqlite { #[pytraverse(skip)] rowcount: i64, statement: Option<PyRef<Statement>>, + #[pytraverse(skip)] + closed: bool, } #[derive(FromArgs)] @@ -1484,20 +1516,54 @@ mod _sqlite { lastrowid: -1, rowcount: -1, statement: None, + closed: false, })), } } + fn new_uninitialized(connection: PyRef<Connection>, _vm: &VirtualMachine) -> Self { + Self { + connection, + arraysize: Radium::new(1), + row_factory: PyAtomicRef::from(None), + inner: PyMutex::from(None), + } + } + + #[pymethod] + fn __init__(&self, _connection: PyRef<Connection>, _vm: &VirtualMachine) -> PyResult<()> { + let mut guard = self.inner.lock(); + if guard.is_some() { + // Already initialized (e.g., from a call to super().__init__) + return Ok(()); + } + *guard = Some(CursorInner { + description: None, + row_cast_map: vec![], + lastrowid: -1, + rowcount: -1, + statement: None, + closed: false, + }); + Ok(()) + } + fn inner(&self, vm: &VirtualMachine) -> PyResult<PyMappedMutexGuard<'_, CursorInner>> { let guard = self.inner.lock(); if guard.is_some() { - Ok(PyMutexGuard::map(guard, |x| unsafe { - x.as_mut().unwrap_unchecked() - })) + let inner_guard = + PyMutexGuard::map(guard, |x| unsafe { x.as_mut().unwrap_unchecked() }); + if inner_guard.closed { + return Err(new_programming_error( + vm, + "Cannot operate on a closed cursor.".to_owned(), + )); + } + Ok(inner_guard) } else { Err(new_programming_error( vm, - "Cannot operate on a closed cursor.".to_owned(), + "Base Cursor.__init__ not called.".to_owned(), )) } } @@ -1717,12 +1783,23 @@ mod _sqlite { } #[pymethod] - fn close(&self) { - if let Some(inner) = self.inner.lock().take() { - if let Some(stmt) = inner.statement { + fn close(&self, vm: &VirtualMachine) -> PyResult<()> { + // Check if __init__ was called + let mut guard = self.inner.lock(); + if guard.is_none() { + return Err(new_programming_error( + vm, + "Base Cursor.__init__ not called.".to_owned(), + )); + } + + if let Some(inner) = guard.as_mut() { + if let Some(stmt) = &inner.statement { stmt.lock().reset(); } + inner.closed = true; } + Ok(()) } #[pymethod] @@ -1809,7 +1886,7 @@ mod _sqlite { type Args = (PyRef<Connection>,); fn py_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult { - Self::new(args.0, None, vm) + Self::new_uninitialized(args.0, vm) .into_ref_with_type(vm, cls) .map(Into::into) } diff --git a/stdlib/src/syslog.rs b/stdlib/src/syslog.rs index 205fd85c44..adba6f297c 100644 --- a/stdlib/src/syslog.rs +++ b/stdlib/src/syslog.rs @@ -26,16 +26,16 @@ mod syslog { use libc::{LOG_AUTHPRIV, LOG_CRON, LOG_PERROR}; fn get_argv(vm: &VirtualMachine) -> Option<PyStrRef> { - if let Some(argv) = vm.state.settings.argv.first() { - if !argv.is_empty() { - return Some( - PyStr::from(match argv.find('\\') { - Some(value) => &argv[value..], - None => argv, - }) - .into_ref(&vm.ctx), - ); - } + if let Some(argv) = vm.state.settings.argv.first() + && !argv.is_empty() + { + return Some( + PyStr::from(match argv.find('\\') { + Some(value) => &argv[value..], + None => argv, + }) + .into_ref(&vm.ctx), + ); } None } diff --git a/stdlib/src/unicodedata.rs b/stdlib/src/unicodedata.rs index 6273b2b47d..46e1835726 100644 --- a/stdlib/src/unicodedata.rs +++ b/stdlib/src/unicodedata.rs @@ -121,10 +121,10 @@ mod unicodedata { #[pymethod] fn lookup(&self, name: PyStrRef, vm: &VirtualMachine) -> PyResult<String> { - if let Some(character) = unicode_names2::character(name.as_str()) { - if self.check_age(character.into()) { - return Ok(character.to_string()); - } + if let Some(character) = unicode_names2::character(name.as_str()) + && self.check_age(character.into()) + { + return Ok(character.to_string()); } Err(vm.new_lookup_error(format!("undefined character name '{name}'"))) } @@ -138,12 +138,11 @@ mod unicodedata { ) -> PyResult { let c = self.extract_char(character, vm)?; - if let Some(c) = c { - if self.check_age(c) { - if let Some(name) = c.to_char().and_then(unicode_names2::name) { - return Ok(vm.ctx.new_str(name.to_string()).into()); - } - } + if let Some(c) = c + && self.check_age(c) + && let Some(name) = c.to_char().and_then(unicode_names2::name) + { + return Ok(vm.ctx.new_str(name.to_string()).into()); } default.ok_or_else(|| vm.new_value_error("no such name")) } diff --git a/stdlib/src/uuid.rs b/stdlib/src/uuid.rs index 9b0e23a81c..3f75db402c 100644 --- a/stdlib/src/uuid.rs +++ b/stdlib/src/uuid.rs @@ -30,4 +30,7 @@ mod _uuid { fn has_uuid_generate_time_safe(_vm: &VirtualMachine) -> u32 { 0 } + + #[pyattr(name = "has_stable_extractable_node")] + const HAS_STABLE_EXTRACTABLE_NODE: bool = false; } diff --git a/stdlib/src/zlib.rs b/stdlib/src/zlib.rs index 6034b4a2c5..cb444ec8c0 100644 --- a/stdlib/src/zlib.rs +++ b/stdlib/src/zlib.rs @@ -193,11 +193,11 @@ mod zlib { fn decompressobj(args: DecompressobjArgs, vm: &VirtualMachine) -> PyResult<PyDecompress> { let mut decompress = InitOptions::new(args.wbits.value, vm)?.decompress(); let zdict = args.zdict.into_option(); - if let Some(dict) = &zdict { - if args.wbits.value < 0 { - dict.with_ref(|d| decompress.set_dictionary(d)) - .map_err(|_| new_zlib_error("failed to set dictionary", vm))?; - } + if let Some(dict) = &zdict + && args.wbits.value < 0 + { + dict.with_ref(|d| decompress.set_dictionary(d)) + .map_err(|_| new_zlib_error("failed to set dictionary", vm))?; } let inner = PyDecompressInner { decompress: Some(DecompressWithDict { decompress, zdict }), @@ -573,11 +573,11 @@ mod zlib { fn py_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult { let mut decompress = InitOptions::new(args.wbits.value, vm)?.decompress(); let zdict = args.zdict.into_option(); - if let Some(dict) = &zdict { - if args.wbits.value < 0 { - dict.with_ref(|d| decompress.set_dictionary(d)) - .map_err(|_| new_zlib_error("failed to set dictionary", vm))?; - } + if let Some(dict) = &zdict + && args.wbits.value < 0 + { + dict.with_ref(|d| decompress.set_dictionary(d)) + .map_err(|_| new_zlib_error("failed to set dictionary", vm))?; } let inner = DecompressState::new(DecompressWithDict { decompress, zdict }, vm); Self { diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 803eae014e..ffd5836178 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -24,7 +24,7 @@ ast = ["ruff_python_ast", "ruff_text_size"] codegen = ["rustpython-codegen", "ast"] parser = ["ast"] serde = ["dep:serde"] -wasmbind = ["chrono/wasmbind", "getrandom/wasm_js", "wasm-bindgen"] +wasmbind = ["rustpython-common/wasm_js", "chrono/wasmbind", "wasm-bindgen"] [dependencies] rustpython-compiler = { workspace = true, optional = true } @@ -56,7 +56,6 @@ itertools = { workspace = true } is-macro = { workspace = true } libc = { workspace = true } log = { workspace = true } -nix = { workspace = true } malachite-bigint = { workspace = true } num-complex = { workspace = true } num-integer = { workspace = true } @@ -76,15 +75,13 @@ thread_local = { workspace = true } memchr = { workspace = true } caseless = "0.2.2" -flamer = { version = "0.4", optional = true } +flamer = { version = "0.5", optional = true } half = "2" -memoffset = "0.9.1" optional = { workspace = true } result-like = "0.5.0" timsort = "0.1.2" ## unicode stuff -unicode_names2 = { workspace = true } # TODO: use unic for this; needed for title case: # https://github.com/RustPython/RustPython/pull/832#discussion_r275428939 unicode-casing = { workspace = true } @@ -95,6 +92,7 @@ unic-ucd-ident = { workspace = true } [target.'cfg(unix)'.dependencies] rustix = { workspace = true } +nix = { workspace = true } exitcode = "1.1.2" uname = "0.1.1" @@ -113,7 +111,6 @@ num_cpus = "1.17.0" [target.'cfg(windows)'.dependencies] junction = { workspace = true } -schannel = { workspace = true } winreg = "0.55" [target.'cfg(windows)'.dependencies.windows] diff --git a/vm/src/builtins/code.rs b/vm/src/builtins/code.rs index 1ce0f3b3e0..79ad896aaf 100644 --- a/vm/src/builtins/code.rs +++ b/vm/src/builtins/code.rs @@ -16,8 +16,125 @@ use crate::{ use malachite_bigint::BigInt; use num_traits::Zero; use rustpython_compiler_core::OneIndexed; +use rustpython_compiler_core::bytecode::PyCodeLocationInfoKind; use std::{borrow::Borrow, fmt, ops::Deref}; +/// State for iterating through code address ranges +struct PyCodeAddressRange<'a> { + ar_start: i32, + ar_end: i32, + ar_line: i32, + computed_line: i32, + reader: LineTableReader<'a>, +} + +impl<'a> PyCodeAddressRange<'a> { + fn new(linetable: &'a [u8], first_line: i32) -> Self { + PyCodeAddressRange { + ar_start: 0, + ar_end: 0, + ar_line: -1, + computed_line: first_line, + reader: LineTableReader::new(linetable), + } + } + + /// Check if this is a NO_LINE marker (code 15) + fn is_no_line_marker(byte: u8) -> bool { + (byte >> 3) == 0x1f + } + + /// Advance to next address range + fn advance(&mut self) -> bool { + if self.reader.at_end() { + return false; + } + + let first_byte = match self.reader.read_byte() { + Some(b) => b, + None => return false, + }; + + if (first_byte & 0x80) == 0 { + return false; // Invalid linetable + } + + let code = (first_byte >> 3) & 0x0f; + let length = ((first_byte & 0x07) + 1) as i32; + + // Get line delta for this entry + let line_delta = self.get_line_delta(code); + + // Update computed line + self.computed_line += line_delta; + + // Check for NO_LINE marker + if Self::is_no_line_marker(first_byte) { + self.ar_line = -1; + } else { + self.ar_line = self.computed_line; + } + + // Update address range + self.ar_start = self.ar_end; + self.ar_end += length * 2; // sizeof(_Py_CODEUNIT) = 2 + + // Skip remaining bytes for this entry + while !self.reader.at_end() { + if let Some(b) = self.reader.peek_byte() { + if (b & 0x80) != 0 { + break; + } + self.reader.read_byte(); + } else { + break; + } + } + + true + } + + fn get_line_delta(&mut self, code: u8) -> i32 { + let kind = match PyCodeLocationInfoKind::from_code(code) { + Some(k) => k, + None => return 0, + }; + + match kind { + PyCodeLocationInfoKind::None => 0, // NO_LINE marker + PyCodeLocationInfoKind::Long => { + let delta = self.reader.read_signed_varint(); + // Skip end_line, col, end_col + self.reader.read_varint(); + self.reader.read_varint(); + self.reader.read_varint(); + delta + } + PyCodeLocationInfoKind::NoColumns => self.reader.read_signed_varint(), + PyCodeLocationInfoKind::OneLine0 => { + self.reader.read_byte(); // Skip column + self.reader.read_byte(); // Skip end column + 0 + } + PyCodeLocationInfoKind::OneLine1 => { + self.reader.read_byte(); // Skip column + self.reader.read_byte(); // Skip end column + 1 + } + PyCodeLocationInfoKind::OneLine2 => { + self.reader.read_byte(); // Skip column + self.reader.read_byte(); // Skip end column + 2 + } + _ if kind.is_short() => { + self.reader.read_byte(); // Skip column byte + 0 + } + _ => 0, + } + } +} + #[derive(FromArgs)] pub struct ReplaceArgs { #[pyarg(named, optional)] @@ -40,6 +157,22 @@ pub struct ReplaceArgs { co_flags: OptionalArg<u16>, #[pyarg(named, optional)] co_varnames: OptionalArg<Vec<PyObjectRef>>, + #[pyarg(named, optional)] + co_nlocals: OptionalArg<u32>, + #[pyarg(named, optional)] + co_stacksize: OptionalArg<u32>, + #[pyarg(named, optional)] + co_code: OptionalArg<crate::builtins::PyBytesRef>, + #[pyarg(named, optional)] + co_linetable: OptionalArg<crate::builtins::PyBytesRef>, + #[pyarg(named, optional)] + co_exceptiontable: OptionalArg<crate::builtins::PyBytesRef>, + #[pyarg(named, optional)] + co_freevars: OptionalArg<Vec<PyObjectRef>>, + #[pyarg(named, optional)] + co_cellvars: OptionalArg<Vec<PyObjectRef>>, + #[pyarg(named, optional)] + co_qualname: OptionalArg<PyStrRef>, } #[derive(Clone)] @@ -350,6 +483,211 @@ impl PyCode { vm.ctx.new_tuple(names) } + #[pygetset] + pub fn co_linetable(&self, vm: &VirtualMachine) -> crate::builtins::PyBytesRef { + // Return the actual linetable from the code object + vm.ctx.new_bytes(self.code.linetable.to_vec()) + } + + #[pygetset] + pub fn co_exceptiontable(&self, vm: &VirtualMachine) -> crate::builtins::PyBytesRef { + // Return the actual exception table from the code object + vm.ctx.new_bytes(self.code.exceptiontable.to_vec()) + } + + #[pymethod] + pub fn co_lines(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // TODO: Implement lazy iterator (lineiterator) like CPython for better performance + // Currently returns eager list for simplicity + + // Return an iterator over (start_offset, end_offset, lineno) tuples + let linetable = self.code.linetable.as_ref(); + let mut lines = Vec::new(); + + if !linetable.is_empty() { + let first_line = self.code.first_line_number.map_or(0, |n| n.get() as i32); + let mut range = PyCodeAddressRange::new(linetable, first_line); + + // Process all address ranges and merge consecutive entries with same line + let mut pending_entry: Option<(i32, i32, i32)> = None; + + while range.advance() { + let start = range.ar_start; + let end = range.ar_end; + let line = range.ar_line; + + if let Some((prev_start, _, prev_line)) = pending_entry { + if prev_line == line { + // Same line, extend the range + pending_entry = Some((prev_start, end, prev_line)); + } else { + // Different line, emit the previous entry + let tuple = if prev_line == -1 { + vm.ctx.new_tuple(vec![ + vm.ctx.new_int(prev_start).into(), + vm.ctx.new_int(start).into(), + vm.ctx.none(), + ]) + } else { + vm.ctx.new_tuple(vec![ + vm.ctx.new_int(prev_start).into(), + vm.ctx.new_int(start).into(), + vm.ctx.new_int(prev_line).into(), + ]) + }; + lines.push(tuple.into()); + pending_entry = Some((start, end, line)); + } + } else { + // First entry + pending_entry = Some((start, end, line)); + } + } + + // Emit the last pending entry + if let Some((start, end, line)) = pending_entry { + let tuple = if line == -1 { + vm.ctx.new_tuple(vec![ + vm.ctx.new_int(start).into(), + vm.ctx.new_int(end).into(), + vm.ctx.none(), + ]) + } else { + vm.ctx.new_tuple(vec![ + vm.ctx.new_int(start).into(), + vm.ctx.new_int(end).into(), + vm.ctx.new_int(line).into(), + ]) + }; + lines.push(tuple.into()); + } + } + + let list = vm.ctx.new_list(lines); + vm.call_method(list.as_object(), "__iter__", ()) + } + + #[pymethod] + pub fn co_positions(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // Return an iterator over (line, end_line, column, end_column) tuples for each instruction + let linetable = self.code.linetable.as_ref(); + let mut positions = Vec::new(); + + if !linetable.is_empty() { + let mut reader = LineTableReader::new(linetable); + let mut line = self.code.first_line_number.map_or(0, |n| n.get() as i32); + + while !reader.at_end() { + let first_byte = match reader.read_byte() { + Some(b) => b, + None => break, + }; + + if (first_byte & 0x80) == 0 { + break; // Invalid linetable + } + + let code = (first_byte >> 3) & 0x0f; + let length = ((first_byte & 0x07) + 1) as i32; + + let kind = match PyCodeLocationInfoKind::from_code(code) { + Some(k) => k, + None => break, // Invalid code + }; + + let (line_delta, end_line_delta, column, end_column): ( + i32, + i32, + Option<i32>, + Option<i32>, + ) = match kind { + PyCodeLocationInfoKind::None => { + // No location - all values are None + (0, 0, None, None) + } + PyCodeLocationInfoKind::Long => { + // Long form + let delta = reader.read_signed_varint(); + let end_line_delta = reader.read_varint() as i32; + + let col = reader.read_varint(); + let column = if col == 0 { + None + } else { + Some((col - 1) as i32) + }; + + let end_col = reader.read_varint(); + let end_column = if end_col == 0 { + None + } else { + Some((end_col - 1) as i32) + }; + + // endline = line + end_line_delta (will be computed after line update) + (delta, end_line_delta, column, end_column) + } + PyCodeLocationInfoKind::NoColumns => { + // No column form + let delta = reader.read_signed_varint(); + (delta, 0, None, None) // endline will be same as line (delta = 0) + } + PyCodeLocationInfoKind::OneLine0 + | PyCodeLocationInfoKind::OneLine1 + | PyCodeLocationInfoKind::OneLine2 => { + // One-line form - endline = line + let col = reader.read_byte().unwrap_or(0) as i32; + let end_col = reader.read_byte().unwrap_or(0) as i32; + let delta = kind.one_line_delta().unwrap_or(0); + (delta, 0, Some(col), Some(end_col)) // endline = line (delta = 0) + } + _ if kind.is_short() => { + // Short form - endline = line + let col_data = reader.read_byte().unwrap_or(0); + let col_group = kind.short_column_group().unwrap_or(0); + let col = ((col_group as i32) << 3) | ((col_data >> 4) as i32); + let end_col = col + (col_data & 0x0f) as i32; + (0, 0, Some(col), Some(end_col)) // endline = line (delta = 0) + } + _ => (0, 0, None, None), + }; + + // Update line number + line += line_delta; + + // Generate position tuples for each instruction covered by this entry + for _ in 0..length { + // Handle special case for no location (code 15) + let final_line = if kind == PyCodeLocationInfoKind::None { + None + } else { + Some(line) + }; + + let final_endline = if kind == PyCodeLocationInfoKind::None { + None + } else { + Some(line + end_line_delta) + }; + + // Convert Option to PyObject (None or int) + let line_obj = final_line.to_pyobject(vm); + let end_line_obj = final_endline.to_pyobject(vm); + let column_obj = column.to_pyobject(vm); + let end_column_obj = end_column.to_pyobject(vm); + + let tuple = + vm.ctx + .new_tuple(vec![line_obj, end_line_obj, column_obj, end_column_obj]); + positions.push(tuple.into()); + } + } + } + + let list = vm.ctx.new_list(positions); + vm.call_method(list.as_object(), "__iter__", ()) + } + #[pymethod] pub fn replace(&self, args: ReplaceArgs, vm: &VirtualMachine) -> PyResult<Self> { let posonlyarg_count = match args.co_posonlyargcount { @@ -408,6 +746,66 @@ impl PyCode { OptionalArg::Missing => self.code.varnames.iter().map(|s| s.to_object()).collect(), }; + let qualname = match args.co_qualname { + OptionalArg::Present(qualname) => qualname, + OptionalArg::Missing => self.code.qualname.to_owned(), + }; + + let max_stackdepth = match args.co_stacksize { + OptionalArg::Present(stacksize) => stacksize, + OptionalArg::Missing => self.code.max_stackdepth, + }; + + let instructions = match args.co_code { + OptionalArg::Present(_code_bytes) => { + // Convert bytes back to instructions + // For now, keep the original instructions + // TODO: Properly parse bytecode from bytes + self.code.instructions.clone() + } + OptionalArg::Missing => self.code.instructions.clone(), + }; + + let cellvars = match args.co_cellvars { + OptionalArg::Present(cellvars) => cellvars + .into_iter() + .map(|o| o.as_interned_str(vm).unwrap()) + .collect(), + OptionalArg::Missing => self.code.cellvars.clone(), + }; + + let freevars = match args.co_freevars { + OptionalArg::Present(freevars) => freevars + .into_iter() + .map(|o| o.as_interned_str(vm).unwrap()) + .collect(), + OptionalArg::Missing => self.code.freevars.clone(), + }; + + // Validate co_nlocals if provided + if let OptionalArg::Present(nlocals) = args.co_nlocals + && nlocals as usize != varnames.len() + { + return Err(vm.new_value_error(format!( + "co_nlocals ({}) != len(co_varnames) ({})", + nlocals, + varnames.len() + ))); + } + + // Handle linetable and exceptiontable + let linetable = match args.co_linetable { + OptionalArg::Present(linetable) => linetable.as_bytes().to_vec().into_boxed_slice(), + OptionalArg::Missing => self.code.linetable.clone(), + }; + + let exceptiontable = match args.co_exceptiontable { + OptionalArg::Present(exceptiontable) => { + exceptiontable.as_bytes().to_vec().into_boxed_slice() + } + OptionalArg::Missing => self.code.exceptiontable.clone(), + }; + Ok(Self { code: CodeObject { flags: CodeFlags::from_bits_truncate(flags), @@ -417,10 +815,10 @@ impl PyCode { source_path: source_path.as_object().as_interned_str(vm).unwrap(), first_line_number, obj_name: obj_name.as_object().as_interned_str(vm).unwrap(), - qualname: self.code.qualname, + qualname: qualname.as_object().as_interned_str(vm).unwrap(), - max_stackdepth: self.code.max_stackdepth, - instructions: self.code.instructions.clone(), + max_stackdepth, + instructions, locations: self.code.locations.clone(), constants: constants.into_iter().map(Literal).collect(), names: names @@ -431,12 +829,23 @@ impl PyCode { .into_iter() .map(|o| o.as_interned_str(vm).unwrap()) .collect(), - cellvars: self.code.cellvars.clone(), - freevars: self.code.freevars.clone(), + cellvars, + freevars, cell2arg: self.code.cell2arg.clone(), + linetable, + exceptiontable, }, }) } + + #[pymethod] + fn _varname_from_oparg(&self, opcode: i32, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let idx_err = |vm: &VirtualMachine| vm.new_index_error("tuple index out of range"); + + let idx = usize::try_from(opcode).map_err(|_| idx_err(vm))?; + let name = self.code.varnames.get(idx).ok_or_else(|| idx_err(vm))?; + Ok(name.to_object()) + } } impl fmt::Display for PyCode { @@ -457,6 +866,69 @@ impl ToPyObject for bytecode::CodeObject { } } +// Helper struct for reading linetable +struct LineTableReader<'a> { + data: &'a [u8], + pos: usize, +} + +impl<'a> LineTableReader<'a> { + fn new(data: &'a [u8]) -> Self { + Self { data, pos: 0 } + } + + fn read_byte(&mut self) -> Option<u8> { + if self.pos < self.data.len() { + let byte = self.data[self.pos]; + self.pos += 1; + Some(byte) + } else { + None + } + } + + fn peek_byte(&self) -> Option<u8> { + if self.pos < self.data.len() { + Some(self.data[self.pos]) + } else { + None + } + } + + fn read_varint(&mut self) -> u32 { + if let Some(first) = self.read_byte() { + let mut val = (first & 0x3f) as u32; + let mut shift = 0; + let mut byte = first; + while (byte & 0x40) != 0 { + if let Some(next) = self.read_byte() { + shift += 6; + val |= ((next & 0x3f) as u32) << shift; + byte = next; + } else { + break; + } + } + val + } else { + 0 + } + } + + fn read_signed_varint(&mut self) -> i32 { + let uval = self.read_varint(); + if uval & 1 != 0 { + -((uval >> 1) as i32) + } else { + (uval >> 1) as i32 + } + } + + fn at_end(&self) -> bool { + self.pos >= self.data.len() + } +} + pub fn init(ctx: &Context) { PyCode::extend_class(ctx, ctx.types.code_type); } diff --git a/vm/src/builtins/descriptor.rs b/vm/src/builtins/descriptor.rs index 977a442529..bc3aded325 100644 --- a/vm/src/builtins/descriptor.rs +++ b/vm/src/builtins/descriptor.rs @@ -367,14 +367,12 @@ impl GetDescriptor for PyMemberDescriptor { // return the class's docstring if available // When accessed from class (not instance), check if the class has // an attribute with the same name as this member descriptor - if let Some(cls) = cls { - if let Ok(cls_type) = cls.downcast::<PyType>() { - if let Some(interned) = vm.ctx.interned_str(descr.member.name.as_str()) { - if let Some(attr) = cls_type.attributes.read().get(&interned) { - return Ok(attr.clone()); - } - } - } + if let Some(cls) = cls + && let Ok(cls_type) = cls.downcast::<PyType>() + && let Some(interned) = vm.ctx.interned_str(descr.member.name.as_str()) + && let Some(attr) = cls_type.attributes.read().get(&interned) + { + return Ok(attr.clone()); } Ok(zelf) } diff --git a/vm/src/builtins/enumerate.rs b/vm/src/builtins/enumerate.rs index a83404c734..9d163899a3 100644 --- a/vm/src/builtins/enumerate.rs +++ b/vm/src/builtins/enumerate.rs @@ -111,10 +111,10 @@ impl PyReverseSequenceIterator { #[pymethod] fn __length_hint__(&self, vm: &VirtualMachine) -> PyResult<usize> { let internal = self.internal.lock(); - if let IterStatus::Active(obj) = &internal.status { - if internal.position <= obj.length(vm)? { - return Ok(internal.position + 1); - } + if let IterStatus::Active(obj) = &internal.status + && internal.position <= obj.length(vm)? + { + return Ok(internal.position + 1); } Ok(0) } diff --git a/vm/src/builtins/frame.rs b/vm/src/builtins/frame.rs index 65ac3e798d..17dc88ac04 100644 --- a/vm/src/builtins/frame.rs +++ b/vm/src/builtins/frame.rs @@ -60,7 +60,7 @@ impl Frame { #[pygetset] pub fn f_lineno(&self) -> usize { - self.current_location().row.get() + self.current_location().line.get() } #[pygetset] diff --git a/vm/src/builtins/function.rs b/vm/src/builtins/function.rs index c73db94edf..02e983f2ff 100644 --- a/vm/src/builtins/function.rs +++ b/vm/src/builtins/function.rs @@ -280,11 +280,11 @@ impl PyFunction { .take(code.kwonlyarg_count as usize) .filter(|(slot, _)| slot.is_none()) { - if let Some(defaults) = &get_defaults!().1 { - if let Some(default) = defaults.get_item_opt(&**kwarg, vm)? { - *slot = Some(default); - continue; - } + if let Some(defaults) = &get_defaults!().1 + && let Some(default) = defaults.get_item_opt(&**kwarg, vm)? + { + *slot = Some(default); + continue; } // No default value and not specified. diff --git a/vm/src/builtins/genericalias.rs b/vm/src/builtins/genericalias.rs index 9043871266..bcaeb1bfb2 100644 --- a/vm/src/builtins/genericalias.rs +++ b/vm/src/builtins/genericalias.rs @@ -269,22 +269,22 @@ pub(crate) fn make_parameters(args: &Py<PyTuple>, vm: &VirtualMachine) -> PyTupl parameters[iparam] = arg.clone(); iparam += 1; } - } else if let Ok(subparams) = arg.get_attr(identifier!(vm, __parameters__), vm) { - if let Ok(sub_params) = subparams.try_to_ref::<PyTuple>(vm) { - let len2 = sub_params.len(); - // Resize if needed - if iparam + len2 > parameters.len() { - parameters.resize(iparam + len2, vm.ctx.none()); - } - for sub_param in sub_params { - // Use tuple_add equivalent logic - if tuple_index(¶meters[..iparam], sub_param).is_none() { - if iparam >= parameters.len() { - parameters.resize(iparam + 1, vm.ctx.none()); - } - parameters[iparam] = sub_param.clone(); - iparam += 1; + } else if let Ok(subparams) = arg.get_attr(identifier!(vm, __parameters__), vm) + && let Ok(sub_params) = subparams.try_to_ref::<PyTuple>(vm) + { + let len2 = sub_params.len(); + // Resize if needed + if iparam + len2 > parameters.len() { + parameters.resize(iparam + len2, vm.ctx.none()); + } + for sub_param in sub_params { + // Use tuple_add equivalent logic + if tuple_index(¶meters[..iparam], sub_param).is_none() { + if iparam >= parameters.len() { + parameters.resize(iparam + 1, vm.ctx.none()); } + parameters[iparam] = sub_param.clone(); + iparam += 1; } } } @@ -376,23 +376,22 @@ fn unpack_args(item: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyTupleRef> { } // Try to get __typing_unpacked_tuple_args__ - if let Ok(sub_args) = item.get_attr(identifier!(vm, __typing_unpacked_tuple_args__), vm) { - if !sub_args.is(&vm.ctx.none) { - if let Ok(tuple) = sub_args.try_to_ref::<PyTuple>(vm) { - // Check for ellipsis at the end - let has_ellipsis_at_end = tuple - .as_slice() - .last() - .is_some_and(|item| item.is(&vm.ctx.ellipsis)); - - if !has_ellipsis_at_end { - // Safe to unpack - add all elements's PyList_SetSlice - for arg in tuple { - new_args.push(arg.clone()); - } - continue; - } + if let Ok(sub_args) = item.get_attr(identifier!(vm, __typing_unpacked_tuple_args__), vm) + && !sub_args.is(&vm.ctx.none) + && let Ok(tuple) = sub_args.try_to_ref::<PyTuple>(vm) + { + // Check for ellipsis at the end + let has_ellipsis_at_end = tuple + .as_slice() + .last() + .is_some_and(|item| item.is(&vm.ctx.ellipsis)); + + if !has_ellipsis_at_end { + // Safe to unpack - add all elements's PyList_SetSlice + for arg in tuple { + new_args.push(arg.clone()); } + continue; } } @@ -421,17 +420,17 @@ pub fn subs_parameters( // Step 2: Call __typing_prepare_subst__ on each parameter for param in parameters.iter() { - if let Ok(prepare) = param.get_attr(identifier!(vm, __typing_prepare_subst__), vm) { - if !prepare.is(&vm.ctx.none) { - // Call prepare(self, item) - item = if item.try_to_ref::<PyTuple>(vm).is_ok() { - prepare.call((alias.clone(), item.clone()), vm)? - } else { - // Create a tuple with the single item's "O(O)" format - let tuple_args = PyTuple::new_ref(vec![item.clone()], &vm.ctx); - prepare.call((alias.clone(), tuple_args.to_pyobject(vm)), vm)? - }; - } + if let Ok(prepare) = param.get_attr(identifier!(vm, __typing_prepare_subst__), vm) + && !prepare.is(&vm.ctx.none) + { + // Call prepare(self, item) + item = if item.try_to_ref::<PyTuple>(vm).is_ok() { + prepare.call((alias.clone(), item.clone()), vm)? + } else { + // Create a tuple with the single item's "O(O)" format + let tuple_args = PyTuple::new_ref(vec![item.clone()], &vm.ctx); + prepare.call((alias.clone(), tuple_args.to_pyobject(vm)), vm)? + }; } } @@ -526,12 +525,11 @@ impl Callable for PyGenericAlias { type Args = FuncArgs; fn call(zelf: &Py<Self>, args: FuncArgs, vm: &VirtualMachine) -> PyResult { PyType::call(&zelf.origin, args, vm).map(|obj| { - if let Err(exc) = obj.set_attr(identifier!(vm, __orig_class__), zelf.to_owned(), vm) { - if !exc.fast_isinstance(vm.ctx.exceptions.attribute_error) - && !exc.fast_isinstance(vm.ctx.exceptions.type_error) - { - return Err(exc); - } + if let Err(exc) = obj.set_attr(identifier!(vm, __orig_class__), zelf.to_owned(), vm) + && !exc.fast_isinstance(vm.ctx.exceptions.attribute_error) + && !exc.fast_isinstance(vm.ctx.exceptions.type_error) + { + return Err(exc); } Ok(obj) })? diff --git a/vm/src/builtins/int.rs b/vm/src/builtins/int.rs index 70fd9bad9b..36cfd42532 100644 --- a/vm/src/builtins/int.rs +++ b/vm/src/builtins/int.rs @@ -501,34 +501,33 @@ impl PyInt { let ndigits = ndigits.as_bigint(); // round(12345, -2) == 12300 // If precision >= 0, then any integer is already rounded correctly - if let Some(ndigits) = ndigits.neg().to_u32() { - if ndigits > 0 { - // Work with positive integers and negate at the end if necessary - let sign = if zelf.value.is_negative() { - BigInt::from(-1) - } else { - BigInt::from(1) - }; - let value = zelf.value.abs(); - - // Divide and multiply by the power of 10 to get the approximate answer - let pow10 = BigInt::from(10).pow(ndigits); - let quotient = &value / &pow10; - let rounded = "ient * &pow10; - - // Malachite division uses floor rounding, Python uses half-even - let remainder = &value - &rounded; - let half_pow10 = &pow10 / BigInt::from(2); - let correction = if remainder > half_pow10 - || (remainder == half_pow10 && quotient.is_odd()) - { + if let Some(ndigits) = ndigits.neg().to_u32() + && ndigits > 0 + { + // Work with positive integers and negate at the end if necessary + let sign = if zelf.value.is_negative() { + BigInt::from(-1) + } else { + BigInt::from(1) + }; + let value = zelf.value.abs(); + + // Divide and multiply by the power of 10 to get the approximate answer + let pow10 = BigInt::from(10).pow(ndigits); + let quotient = &value / &pow10; + let rounded = "ient * &pow10; + + // Malachite division uses floor rounding, Python uses half-even + let remainder = &value - &rounded; + let half_pow10 = &pow10 / BigInt::from(2); + let correction = + if remainder > half_pow10 || (remainder == half_pow10 && quotient.is_odd()) { pow10 } else { BigInt::from(0) }; - let rounded = (rounded + correction) * sign; - return Ok(vm.ctx.new_int(rounded)); - } + let rounded = (rounded + correction) * sign; + return Ok(vm.ctx.new_int(rounded)); } } Ok(zelf) diff --git a/vm/src/builtins/iter.rs b/vm/src/builtins/iter.rs index f12da710ee..56dfc14d16 100644 --- a/vm/src/builtins/iter.rs +++ b/vm/src/builtins/iter.rs @@ -151,10 +151,10 @@ impl<T> PositionIterInternal<T> { where F: FnOnce(&T) -> usize, { - if let IterStatus::Active(obj) = &self.status { - if self.position <= f(obj) { - return self.position + 1; - } + if let IterStatus::Active(obj) = &self.status + && self.position <= f(obj) + { + return self.position + 1; } 0 } diff --git a/vm/src/builtins/mappingproxy.rs b/vm/src/builtins/mappingproxy.rs index d3acc91e9b..04eb6ef671 100644 --- a/vm/src/builtins/mappingproxy.rs +++ b/vm/src/builtins/mappingproxy.rs @@ -62,17 +62,17 @@ impl Constructor for PyMappingProxy { type Args = PyObjectRef; fn py_new(cls: PyTypeRef, mapping: Self::Args, vm: &VirtualMachine) -> PyResult { - if let Some(methods) = PyMapping::find_methods(&mapping) { - if !mapping.downcastable::<PyList>() && !mapping.downcastable::<PyTuple>() { - return Self { - mapping: MappingProxyInner::Mapping(ArgMapping::with_methods( - mapping, - unsafe { methods.borrow_static() }, - )), - } - .into_ref_with_type(vm, cls) - .map(Into::into); + if let Some(methods) = PyMapping::find_methods(&mapping) + && !mapping.downcastable::<PyList>() + && !mapping.downcastable::<PyTuple>() + { + return Self { + mapping: MappingProxyInner::Mapping(ArgMapping::with_methods(mapping, unsafe { + methods.borrow_static() + })), } + .into_ref_with_type(vm, cls) + .map(Into::into); } Err(vm.new_type_error(format!( "mappingproxy() argument must be a mapping, not {}", diff --git a/vm/src/builtins/memory.rs b/vm/src/builtins/memory.rs index 7ec03bf971..87e31256c3 100644 --- a/vm/src/builtins/memory.rs +++ b/vm/src/builtins/memory.rs @@ -330,10 +330,10 @@ impl PyMemoryView { return Ok(false); } - if let Some(other) = other.downcast_ref::<Self>() { - if other.released.load() { - return Ok(false); - } + if let Some(other) = other.downcast_ref::<Self>() + && other.released.load() + { + return Ok(false); } let other = match PyBuffer::try_from_borrowed_object(vm, other) { @@ -668,10 +668,10 @@ impl PyMemoryView { if needle.is(&vm.ctx.ellipsis) { return Ok(zelf.into()); } - if let Some(tuple) = needle.downcast_ref::<PyTuple>() { - if tuple.is_empty() { - return zelf.unpack_single(0, vm); - } + if let Some(tuple) = needle.downcast_ref::<PyTuple>() + && tuple.is_empty() + { + return zelf.unpack_single(0, vm); } return Err(vm.new_type_error("invalid indexing of 0-dim memory")); } @@ -867,10 +867,10 @@ impl Py<PyMemoryView> { // TODO: merge branches when we got conditional if let if needle.is(&vm.ctx.ellipsis) { return self.pack_single(0, value, vm); - } else if let Some(tuple) = needle.downcast_ref::<PyTuple>() { - if tuple.is_empty() { - return self.pack_single(0, value, vm); - } + } else if let Some(tuple) = needle.downcast_ref::<PyTuple>() + && tuple.is_empty() + { + return self.pack_single(0, value, vm); } return Err(vm.new_type_error("invalid indexing of 0-dim memory")); } diff --git a/vm/src/builtins/object.rs b/vm/src/builtins/object.rs index ee32e35475..810e3c2540 100644 --- a/vm/src/builtins/object.rs +++ b/vm/src/builtins/object.rs @@ -53,14 +53,12 @@ impl Constructor for PyBaseObject { let tp_init = cls.get_attr(identifier!(vm, __init__)); let object_init = vm.ctx.types.object_type.get_attr(identifier!(vm, __init__)); - if let (Some(tp_init), Some(object_init)) = (tp_init, object_init) { - if tp_init.is(&object_init) { - // Both __new__ and __init__ are object's versions, - // so the type accepts no arguments - return Err( - vm.new_type_error(format!("{}() takes no arguments", cls.name())) - ); - } + if let (Some(tp_init), Some(object_init)) = (tp_init, object_init) + && tp_init.is(&object_init) + { + // Both __new__ and __init__ are object's versions, + // so the type accepts no arguments + return Err(vm.new_type_error(format!("{}() takes no arguments", cls.name()))); } // If tp_init != object_init, then the type has custom __init__ // which might accept arguments, so we allow it @@ -75,32 +73,31 @@ impl Constructor for PyBaseObject { }; // Ensure that all abstract methods are implemented before instantiating instance. - if let Some(abs_methods) = cls.get_attr(identifier!(vm, __abstractmethods__)) { - if let Some(unimplemented_abstract_method_count) = abs_methods.length_opt(vm) { - let methods: Vec<PyStrRef> = abs_methods.try_to_value(vm)?; - let methods: String = - Itertools::intersperse(methods.iter().map(|name| name.as_str()), "', '") - .collect(); - - let unimplemented_abstract_method_count = unimplemented_abstract_method_count?; - let name = cls.name().to_string(); - - match unimplemented_abstract_method_count { - 0 => {} - 1 => { - return Err(vm.new_type_error(format!( - "class {name} without an implementation for abstract method '{methods}'" - ))); - } - 2.. => { - return Err(vm.new_type_error(format!( - "class {name} without an implementation for abstract methods '{methods}'" - ))); - } - // TODO: remove `allow` when redox build doesn't complain about it - #[allow(unreachable_patterns)] - _ => unreachable!(), + if let Some(abs_methods) = cls.get_attr(identifier!(vm, __abstractmethods__)) + && let Some(unimplemented_abstract_method_count) = abs_methods.length_opt(vm) + { + let methods: Vec<PyStrRef> = abs_methods.try_to_value(vm)?; + let methods: String = + Itertools::intersperse(methods.iter().map(|name| name.as_str()), "', '").collect(); + + let unimplemented_abstract_method_count = unimplemented_abstract_method_count?; + let name = cls.name().to_string(); + + match unimplemented_abstract_method_count { + 0 => {} + 1 => { + return Err(vm.new_type_error(format!( + "class {name} without an implementation for abstract method '{methods}'" + ))); } + 2.. => { + return Err(vm.new_type_error(format!( + "class {name} without an implementation for abstract methods '{methods}'" + ))); + } + // TODO: remove `allow` when redox build doesn't complain about it + #[allow(unreachable_patterns)] + _ => unreachable!(), } } diff --git a/vm/src/builtins/property.rs b/vm/src/builtins/property.rs index 925ec35f49..1568cb08d8 100644 --- a/vm/src/builtins/property.rs +++ b/vm/src/builtins/property.rs @@ -74,10 +74,10 @@ impl PyProperty { } // Otherwise try to get __name__ from getter - if let Some(getter) = self.getter.read().as_ref() { - if let Ok(name) = getter.get_attr("__name__", vm) { - return Some(name); - } + if let Some(getter) = self.getter.read().as_ref() + && let Ok(name) = getter.get_attr("__name__", vm) + { + return Some(name); } None @@ -249,24 +249,24 @@ impl PyProperty { }; // Check getter - if let Some(getter) = self.getter.read().as_ref() { - if is_abstract(getter)? { - return Ok(vm.ctx.new_bool(true).into()); - } + if let Some(getter) = self.getter.read().as_ref() + && is_abstract(getter)? + { + return Ok(vm.ctx.new_bool(true).into()); } // Check setter - if let Some(setter) = self.setter.read().as_ref() { - if is_abstract(setter)? { - return Ok(vm.ctx.new_bool(true).into()); - } + if let Some(setter) = self.setter.read().as_ref() + && is_abstract(setter)? + { + return Ok(vm.ctx.new_bool(true).into()); } // Check deleter - if let Some(deleter) = self.deleter.read().as_ref() { - if is_abstract(deleter)? { - return Ok(vm.ctx.new_bool(true).into()); - } + if let Some(deleter) = self.deleter.read().as_ref() + && is_abstract(deleter)? + { + return Ok(vm.ctx.new_bool(true).into()); } Ok(vm.ctx.new_bool(false).into()) diff --git a/vm/src/builtins/str.rs b/vm/src/builtins/str.rs index ddf04e6deb..3ee5e3369d 100644 --- a/vm/src/builtins/str.rs +++ b/vm/src/builtins/str.rs @@ -324,12 +324,12 @@ impl IterNext for PyStrIterator { internal.1 = offset + ch.len_wtf8(); return Ok(PyIterReturn::Return(ch.to_pyobject(vm))); } - } else if let Some(value) = value.get(internal.1..) { - if let Some(ch) = value.code_points().next() { - internal.0.position += 1; - internal.1 += ch.len_wtf8(); - return Ok(PyIterReturn::Return(ch.to_pyobject(vm))); - } + } else if let Some(value) = value.get(internal.1..) + && let Some(ch) = value.code_points().next() + { + internal.0.position += 1; + internal.1 += ch.len_wtf8(); + return Ok(PyIterReturn::Return(ch.to_pyobject(vm))); } internal.0.status = Exhausted; } diff --git a/vm/src/builtins/super.rs b/vm/src/builtins/super.rs index 81c357e232..ce467ecefa 100644 --- a/vm/src/builtins/super.rs +++ b/vm/src/builtins/super.rs @@ -239,19 +239,20 @@ impl Representable for PySuper { } fn super_check(ty: PyTypeRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyTypeRef> { - if let Ok(cls) = obj.clone().downcast::<PyType>() { - if cls.fast_issubclass(&ty) { - return Ok(cls); - } + if let Ok(cls) = obj.clone().downcast::<PyType>() + && cls.fast_issubclass(&ty) + { + return Ok(cls); } if obj.fast_isinstance(&ty) { return Ok(obj.class().to_owned()); } let class_attr = obj.get_attr("__class__", vm)?; - if let Ok(cls) = class_attr.downcast::<PyType>() { - if !cls.is(&ty) && cls.fast_issubclass(&ty) { - return Ok(cls); - } + if let Ok(cls) = class_attr.downcast::<PyType>() + && !cls.is(&ty) + && cls.fast_issubclass(&ty) + { + return Ok(cls); } Err(vm.new_type_error("super(type, obj): obj must be an instance or subtype of type")) } diff --git a/vm/src/builtins/type.rs b/vm/src/builtins/type.rs index cce5808947..849c03bc78 100644 --- a/vm/src/builtins/type.rs +++ b/vm/src/builtins/type.rs @@ -273,25 +273,24 @@ impl PyType { // First check in our own attributes let abc_tpflags_name = ctx.intern_str("__abc_tpflags__"); - if let Some(abc_tpflags_obj) = attrs.get(abc_tpflags_name) { - if let Some(int_obj) = abc_tpflags_obj.downcast_ref::<crate::builtins::int::PyInt>() { - let flags_val = int_obj.as_bigint().to_i64().unwrap_or(0); - let abc_flags = PyTypeFlags::from_bits_truncate(flags_val as u64); - slots.flags |= abc_flags & COLLECTION_FLAGS; - return; - } + if let Some(abc_tpflags_obj) = attrs.get(abc_tpflags_name) + && let Some(int_obj) = abc_tpflags_obj.downcast_ref::<crate::builtins::int::PyInt>() + { + let flags_val = int_obj.as_bigint().to_i64().unwrap_or(0); + let abc_flags = PyTypeFlags::from_bits_truncate(flags_val as u64); + slots.flags |= abc_flags & COLLECTION_FLAGS; + return; } // Then check in base classes for base in bases { - if let Some(abc_tpflags_obj) = base.find_name_in_mro(abc_tpflags_name) { - if let Some(int_obj) = abc_tpflags_obj.downcast_ref::<crate::builtins::int::PyInt>() - { - let flags_val = int_obj.as_bigint().to_i64().unwrap_or(0); - let abc_flags = PyTypeFlags::from_bits_truncate(flags_val as u64); - slots.flags |= abc_flags & COLLECTION_FLAGS; - return; - } + if let Some(abc_tpflags_obj) = base.find_name_in_mro(abc_tpflags_name) + && let Some(int_obj) = abc_tpflags_obj.downcast_ref::<crate::builtins::int::PyInt>() + { + let flags_val = int_obj.as_bigint().to_i64().unwrap_or(0); + let abc_flags = PyTypeFlags::from_bits_truncate(flags_val as u64); + slots.flags |= abc_flags & COLLECTION_FLAGS; + return; } } } @@ -322,13 +321,13 @@ impl PyType { slots.basicsize = base.slots.basicsize; } - if let Some(qualname) = attrs.get(identifier!(ctx, __qualname__)) { - if !qualname.fast_isinstance(ctx.types.str_type) { - return Err(format!( - "type __qualname__ must be a str, not {}", - qualname.class().name() - )); - } + if let Some(qualname) = attrs.get(identifier!(ctx, __qualname__)) + && !qualname.fast_isinstance(ctx.types.str_type) + { + return Err(format!( + "type __qualname__ must be a str, not {}", + qualname.class().name() + )); } let new_type = PyRef::new_ref( @@ -945,10 +944,10 @@ impl PyType { fn __type_params__(&self, vm: &VirtualMachine) -> PyTupleRef { let attrs = self.attributes.read(); let key = identifier!(vm, __type_params__); - if let Some(params) = attrs.get(&key) { - if let Ok(tuple) = params.clone().downcast::<PyTuple>() { - return tuple; - } + if let Some(params) = attrs.get(&key) + && let Ok(tuple) = params.clone().downcast::<PyTuple>() + { + return tuple; } // Return empty tuple if not found or not a tuple vm.ctx.empty_tuple.clone() @@ -1064,16 +1063,16 @@ impl Constructor for PyType { }); let mut attributes = dict.to_attributes(vm); - if let Some(f) = attributes.get_mut(identifier!(vm, __init_subclass__)) { - if f.class().is(vm.ctx.types.function_type) { - *f = PyClassMethod::from(f.clone()).into_pyobject(vm); - } + if let Some(f) = attributes.get_mut(identifier!(vm, __init_subclass__)) + && f.class().is(vm.ctx.types.function_type) + { + *f = PyClassMethod::from(f.clone()).into_pyobject(vm); } - if let Some(f) = attributes.get_mut(identifier!(vm, __class_getitem__)) { - if f.class().is(vm.ctx.types.function_type) { - *f = PyClassMethod::from(f.clone()).into_pyobject(vm); - } + if let Some(f) = attributes.get_mut(identifier!(vm, __class_getitem__)) + && f.class().is(vm.ctx.types.function_type) + { + *f = PyClassMethod::from(f.clone()).into_pyobject(vm); } if let Some(current_frame) = vm.current_frame() { @@ -1370,12 +1369,12 @@ impl Py<PyType> { fn __doc__(&self, vm: &VirtualMachine) -> PyResult { // Similar to CPython's type_get_doc // For non-heap types (static types), check if there's an internal doc - if !self.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) { - if let Some(internal_doc) = self.slots.doc { - // Process internal doc, removing signature if present - let doc_str = get_doc_from_internal_doc(&self.name(), internal_doc); - return Ok(vm.ctx.new_str(doc_str).into()); - } + if !self.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) + && let Some(internal_doc) = self.slots.doc + { + // Process internal doc, removing signature if present + let doc_str = get_doc_from_internal_doc(&self.name(), internal_doc); + return Ok(vm.ctx.new_str(doc_str).into()); } // Check if there's a __doc__ in the type's dict diff --git a/vm/src/byte.rs b/vm/src/byte.rs index db3f93ba32..d9e927cbfa 100644 --- a/vm/src/byte.rs +++ b/vm/src/byte.rs @@ -8,10 +8,10 @@ pub fn bytes_from_object(vm: &VirtualMachine, obj: &PyObject) -> PyResult<Vec<u8 return Ok(elements); } - if !obj.fast_isinstance(vm.ctx.types.str_type) { - if let Ok(elements) = vm.map_iterable_object(obj, |x| value_from_object(vm, &x)) { - return elements; - } + if !obj.fast_isinstance(vm.ctx.types.str_type) + && let Ok(elements) = vm.map_iterable_object(obj, |x| value_from_object(vm, &x)) + { + return elements; } Err(vm.new_type_error("can assign only bytes, buffers, or iterables of ints in range(0, 256)")) diff --git a/vm/src/cformat.rs b/vm/src/cformat.rs index 6957c6dbb7..507079e7de 100644 --- a/vm/src/cformat.rs +++ b/vm/src/cformat.rs @@ -195,10 +195,10 @@ fn spec_format_string( .ok_or_else(|| vm.new_overflow_error("%c arg not in range(0x110000)"))?; return Ok(spec.format_char(ch)); } - if let Some(s) = obj.downcast_ref::<PyStr>() { - if let Ok(ch) = s.as_wtf8().code_points().exactly_one() { - return Ok(spec.format_char(ch)); - } + if let Some(s) = obj.downcast_ref::<PyStr>() + && let Ok(ch) = s.as_wtf8().code_points().exactly_one() + { + return Ok(spec.format_char(ch)); } Err(vm.new_type_error("%c requires int or char")) } diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index 187853ec51..20bf467c45 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -1535,10 +1535,10 @@ pub(super) mod types { if !vm.is_none(&filename) { let mut args_reduced: Vec<PyObjectRef> = vec![errno, msg, filename]; - if let Ok(filename2) = obj.get_attr("filename2", vm) { - if !vm.is_none(&filename2) { - args_reduced.push(filename2); - } + if let Ok(filename2) = obj.get_attr("filename2", vm) + && !vm.is_none(&filename2) + { + args_reduced.push(filename2); } result.push(args_reduced.into_pytuple(vm).into()); } else { @@ -1658,28 +1658,27 @@ pub(super) mod types { zelf.set_attr("print_file_and_line", vm.ctx.none(), vm)?; - if len == 2 { - if let Ok(location_tuple) = new_args.args[1] + if len == 2 + && let Ok(location_tuple) = new_args.args[1] .clone() .downcast::<crate::builtins::PyTuple>() + { + let location_tup_len = location_tuple.len(); + for (i, &attr) in [ + "filename", + "lineno", + "offset", + "text", + "end_lineno", + "end_offset", + ] + .iter() + .enumerate() { - let location_tup_len = location_tuple.len(); - for (i, &attr) in [ - "filename", - "lineno", - "offset", - "text", - "end_lineno", - "end_offset", - ] - .iter() - .enumerate() - { - if location_tup_len > i { - zelf.set_attr(attr, location_tuple[i].to_owned(), vm)?; - } else { - break; - } + if location_tup_len > i { + zelf.set_attr(attr, location_tuple[i].to_owned(), vm)?; + } else { + break; } } } diff --git a/vm/src/frame.rs b/vm/src/frame.rs index ba9abffc6e..89d19f15c8 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -171,7 +171,7 @@ impl Frame { } pub fn current_location(&self) -> SourceLocation { - self.code.locations[self.lasti() as usize - 1].clone() + self.code.locations[self.lasti() as usize - 1] } pub fn lasti(&self) -> u32 { @@ -388,11 +388,15 @@ impl ExecutingFrame<'_> { // 2. Add new entry with current execution position (filename, lineno, code_object) to traceback. // 3. Unwind block stack till appropriate handler is found. - let loc = frame.code.locations[idx].clone(); + let loc = frame.code.locations[idx]; let next = exception.__traceback__(); - let new_traceback = - PyTraceback::new(next, frame.object.to_owned(), frame.lasti(), loc.row); - vm_trace!("Adding to traceback: {:?} {:?}", new_traceback, loc.row); + let new_traceback = PyTraceback::new( + next, + frame.object.to_owned(), + frame.lasti(), + loc.line, + ); + vm_trace!("Adding to traceback: {:?} {:?}", new_traceback, loc.line); exception.set_traceback_typed(Some(new_traceback.into_ref(&vm.ctx))); vm.contextualize_exception(&exception); @@ -2449,6 +2453,9 @@ impl ExecutingFrame<'_> { .map_err(|_| vm.new_type_error("Type params must be a tuple."))? }; + let name = name.downcast::<crate::builtins::PyStr>().map_err(|_| { + vm.new_type_error("TypeAliasType name must be a string".to_owned()) + })?; let type_alias = typing::TypeAliasType::new(name, type_params, value); Ok(type_alias.into_ref(&vm.ctx).into()) } diff --git a/vm/src/function/argument.rs b/vm/src/function/argument.rs index 62d172c37d..d657ff6be8 100644 --- a/vm/src/function/argument.rs +++ b/vm/src/function/argument.rs @@ -55,6 +55,10 @@ into_func_args_from_tuple!((v1, T1), (v2, T2), (v3, T3)); into_func_args_from_tuple!((v1, T1), (v2, T2), (v3, T3), (v4, T4)); into_func_args_from_tuple!((v1, T1), (v2, T2), (v3, T3), (v4, T4), (v5, T5)); into_func_args_from_tuple!((v1, T1), (v2, T2), (v3, T3), (v4, T4), (v5, T5), (v6, T6)); +// We currently allows only 6 unnamed positional arguments. +// Please use `#[derive(FromArgs)]` and a struct for more complex argument parsing. +// The number of limitation came from: +// https://rust-lang.github.io/rust-clippy/master/index.html#too_many_arguments /// The `FuncArgs` struct is one of the most used structs then creating /// a rust function that can be called from python. It holds both positional diff --git a/vm/src/object/core.rs b/vm/src/object/core.rs index 57576ce703..fb2c5b6303 100644 --- a/vm/src/object/core.rs +++ b/vm/src/object/core.rs @@ -192,12 +192,10 @@ impl WeakRefList { })) }); let mut inner = unsafe { inner_ptr.as_ref().lock() }; - if is_generic { - if let Some(generic_weakref) = inner.generic_weakref { - let generic_weakref = unsafe { generic_weakref.as_ref() }; - if generic_weakref.0.ref_count.get() != 0 { - return generic_weakref.to_owned(); - } + if is_generic && let Some(generic_weakref) = inner.generic_weakref { + let generic_weakref = unsafe { generic_weakref.as_ref() }; + if generic_weakref.0.ref_count.get() != 0 { + return generic_weakref.to_owned(); } } let obj = PyWeak { diff --git a/vm/src/protocol/mapping.rs b/vm/src/protocol/mapping.rs index f5da3f7de6..a942303dbb 100644 --- a/vm/src/protocol/mapping.rs +++ b/vm/src/protocol/mapping.rs @@ -78,13 +78,13 @@ impl AsRef<PyObject> for PyMapping<'_> { impl<'a> PyMapping<'a> { pub fn try_protocol(obj: &'a PyObject, vm: &VirtualMachine) -> PyResult<Self> { - if let Some(methods) = Self::find_methods(obj) { - if methods.as_ref().check() { - return Ok(Self { - obj, - methods: unsafe { methods.borrow_static() }, - }); - } + if let Some(methods) = Self::find_methods(obj) + && methods.as_ref().check() + { + return Ok(Self { + obj, + methods: unsafe { methods.borrow_static() }, + }); } Err(vm.new_type_error(format!("{} is not a mapping object", obj.class()))) diff --git a/vm/src/protocol/object.rs b/vm/src/protocol/object.rs index d398da0237..0e8d7cf5fa 100644 --- a/vm/src/protocol/object.rs +++ b/vm/src/protocol/object.rs @@ -220,14 +220,13 @@ impl PyObject { Some(descr) => { let descr_cls = descr.class(); let descr_get = descr_cls.mro_find_map(|cls| cls.slots.descr_get.load()); - if let Some(descr_get) = descr_get { - if descr_cls + if let Some(descr_get) = descr_get + && descr_cls .mro_find_map(|cls| cls.slots.descr_set.load()) .is_some() - { - let cls = obj_cls.to_owned().into(); - return descr_get(descr, Some(self.to_owned()), Some(cls), vm).map(Some); - } + { + let cls = obj_cls.to_owned().into(); + return descr_get(descr, Some(self.to_owned()), Some(cls), vm).map(Some); } Some((descr, descr_get)) } @@ -556,12 +555,10 @@ impl PyObject { // Check __class__ attribute, only masking AttributeError if let Some(i_cls) = vm.get_attribute_opt(self.to_owned(), identifier!(vm, __class__))? + && let Ok(i_cls_type) = PyTypeRef::try_from_object(vm, i_cls) + && !i_cls_type.is(self.class()) { - if let Ok(i_cls_type) = PyTypeRef::try_from_object(vm, i_cls) { - if !i_cls_type.is(self.class()) { - retval = i_cls_type.is_subtype(cls); - } - } + retval = i_cls_type.is_subtype(cls); } } Ok(retval) @@ -642,20 +639,14 @@ impl PyObject { } pub fn hash(&self, vm: &VirtualMachine) -> PyResult<PyHash> { - let hash = self.get_class_attr(identifier!(vm, __hash__)).unwrap(); - if vm.is_none(&hash) { - return Err(vm.new_exception_msg( - vm.ctx.exceptions.type_error.to_owned(), - format!("unhashable type: '{}'", self.class().name()), - )); + if let Some(hash) = self.class().mro_find_map(|cls| cls.slots.hash.load()) { + return hash(self, vm); } - let hash = self - .class() - .mro_find_map(|cls| cls.slots.hash.load()) - .unwrap(); - - hash(self, vm) + Err(vm.new_exception_msg( + vm.ctx.exceptions.type_error.to_owned(), + format!("unhashable type: '{}'", self.class().name()), + )) } // type protocol diff --git a/vm/src/readline.rs b/vm/src/readline.rs index c9fdc10ef8..839ac98003 100644 --- a/vm/src/readline.rs +++ b/vm/src/readline.rs @@ -13,6 +13,8 @@ pub enum ReadlineResult { Eof, Interrupt, Io(std::io::Error), + #[cfg(unix)] + OsError(nix::Error), Other(OtherError), } @@ -95,10 +97,10 @@ mod rustyline_readline { } pub fn save_history(&mut self, path: &Path) -> OtherResult<()> { - if !path.exists() { - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent)?; - } + if !path.exists() + && let Some(parent) = path.parent() + { + std::fs::create_dir_all(parent)?; } self.repl.save_history(path)?; Ok(()) @@ -118,6 +120,8 @@ mod rustyline_readline { Err(ReadlineError::Eof) => ReadlineResult::Eof, Err(ReadlineError::Io(e)) => ReadlineResult::Io(e), Err(ReadlineError::Signal(_)) => continue, + #[cfg(unix)] + Err(ReadlineError::Errno(num)) => ReadlineResult::OsError(num), Err(e) => ReadlineResult::Other(e.into()), }; } diff --git a/vm/src/signal.rs b/vm/src/signal.rs index 4157a2c67e..1074c8e8f1 100644 --- a/vm/src/signal.rs +++ b/vm/src/signal.rs @@ -35,12 +35,11 @@ fn trigger_signals(vm: &VirtualMachine) -> PyResult<()> { let signal_handlers = vm.signal_handlers.as_ref().unwrap().borrow(); for (signum, trigger) in TRIGGERS.iter().enumerate().skip(1) { let triggered = trigger.swap(false, Ordering::Relaxed); - if triggered { - if let Some(handler) = &signal_handlers[signum] { - if let Some(callable) = handler.to_callable() { - callable.invoke((signum, vm.ctx.none()), vm)?; - } - } + if triggered + && let Some(handler) = &signal_handlers[signum] + && let Some(callable) = handler.to_callable() + { + callable.invoke((signum, vm.ctx.none()), vm)?; } } if let Some(signal_rx) = &vm.signal_rx { diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 1533f903c5..00aad0213f 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -23,7 +23,7 @@ use node::Node; use ruff_python_ast as ruff; use ruff_text_size::{Ranged, TextRange, TextSize}; use rustpython_compiler_core::{ - LineIndex, OneIndexed, SourceFile, SourceFileBuilder, SourceLocation, + LineIndex, OneIndexed, PositionEncoding, SourceFile, SourceFileBuilder, SourceLocation, }; #[cfg(feature = "parser")] @@ -92,8 +92,8 @@ pub struct PySourceLocation { impl PySourceLocation { const fn to_source_location(&self) -> SourceLocation { SourceLocation { - row: self.row.get_one_indexed(), - column: self.column.get_one_indexed(), + line: self.row.get_one_indexed(), + character_offset: self.column.get_one_indexed(), } } } @@ -194,14 +194,14 @@ fn source_range_to_text_range(source_file: &SourceFile, location: PySourceRange) } let start = index.offset( - location.start.row.get_one_indexed(), - location.start.column.get_one_indexed(), + location.start.to_source_location(), source, + PositionEncoding::Utf8, ); let end = index.offset( - location.end.row.get_one_indexed(), - location.end.column.get_one_indexed(), + location.end.to_source_location(), source, + PositionEncoding::Utf8, ); TextRange::new(start, end) @@ -273,9 +273,11 @@ pub(crate) fn compile( let ast: Mod = Node::ast_from_object(vm, &source_file, object)?; let ast = match ast { Mod::Module(m) => ruff::Mod::Module(m), - Mod::Interactive(ModInteractive { range, body }) => { - ruff::Mod::Module(ruff::ModModule { range, body }) - } + Mod::Interactive(ModInteractive { range, body }) => ruff::Mod::Module(ruff::ModModule { + node_index: Default::default(), + range, + body, + }), Mod::Expression(e) => ruff::Mod::Expression(e), Mod::FunctionType(_) => todo!(), }; diff --git a/vm/src/stdlib/ast/argument.rs b/vm/src/stdlib/ast/argument.rs index a1207f2c05..a13200e650 100644 --- a/vm/src/stdlib/ast/argument.rs +++ b/vm/src/stdlib/ast/argument.rs @@ -57,6 +57,7 @@ pub(super) fn merge_function_call_arguments( let range = pos_args.range.cover(key_args.range); ruff::Arguments { + node_index: Default::default(), range, args: pos_args.args, keywords: key_args.keywords, @@ -67,6 +68,7 @@ pub(super) fn split_function_call_arguments( args: ruff::Arguments, ) -> (PositionalArguments, KeywordArguments) { let ruff::Arguments { + node_index: _, range: _, args, keywords, @@ -105,6 +107,7 @@ pub(super) fn split_class_def_args( Some(args) => *args, }; let ruff::Arguments { + node_index: _, range: _, args, keywords, @@ -155,6 +158,7 @@ pub(super) fn merge_class_def_args( }; Some(Box::new(ruff::Arguments { + node_index: Default::default(), range: Default::default(), // TODO args, keywords, diff --git a/vm/src/stdlib/ast/constant.rs b/vm/src/stdlib/ast/constant.rs index b6c3e9596b..83b2a7f701 100644 --- a/vm/src/stdlib/ast/constant.rs +++ b/vm/src/stdlib/ast/constant.rs @@ -247,14 +247,21 @@ impl Node for ConstantLiteral { fn constant_to_ruff_expr(value: Constant) -> ruff::Expr { let Constant { value, range } = value; match value { - ConstantLiteral::None => ruff::Expr::NoneLiteral(ruff::ExprNoneLiteral { range }), - ConstantLiteral::Bool(value) => { - ruff::Expr::BooleanLiteral(ruff::ExprBooleanLiteral { range, value }) - } + ConstantLiteral::None => ruff::Expr::NoneLiteral(ruff::ExprNoneLiteral { + node_index: Default::default(), + range, + }), + ConstantLiteral::Bool(value) => ruff::Expr::BooleanLiteral(ruff::ExprBooleanLiteral { + node_index: Default::default(), + range, + value, + }), ConstantLiteral::Str { value, prefix } => { ruff::Expr::StringLiteral(ruff::ExprStringLiteral { + node_index: Default::default(), range, value: ruff::StringLiteralValue::single(ruff::StringLiteral { + node_index: Default::default(), range, value, flags: ruff::StringLiteralFlags::empty().with_prefix(prefix), @@ -263,8 +270,10 @@ fn constant_to_ruff_expr(value: Constant) -> ruff::Expr { } ConstantLiteral::Bytes(value) => { ruff::Expr::BytesLiteral(ruff::ExprBytesLiteral { + node_index: Default::default(), range, value: ruff::BytesLiteralValue::single(ruff::BytesLiteral { + node_index: Default::default(), range, value, flags: ruff::BytesLiteralFlags::empty(), // TODO @@ -272,10 +281,12 @@ fn constant_to_ruff_expr(value: Constant) -> ruff::Expr { }) } ConstantLiteral::Int(value) => ruff::Expr::NumberLiteral(ruff::ExprNumberLiteral { + node_index: Default::default(), range, value: ruff::Number::Int(value), }), ConstantLiteral::Tuple(value) => ruff::Expr::Tuple(ruff::ExprTuple { + node_index: Default::default(), range, elts: value .into_iter() @@ -291,14 +302,17 @@ fn constant_to_ruff_expr(value: Constant) -> ruff::Expr { parenthesized: true, }), ConstantLiteral::FrozenSet(value) => ruff::Expr::Call(ruff::ExprCall { + node_index: Default::default(), range, // idk lol func: Box::new(ruff::Expr::Name(ruff::ExprName { + node_index: Default::default(), range: TextRange::default(), id: ruff::name::Name::new_static("frozenset"), ctx: ruff::ExprContext::Load, })), arguments: ruff::Arguments { + node_index: Default::default(), range, args: value .into_iter() @@ -313,18 +327,21 @@ fn constant_to_ruff_expr(value: Constant) -> ruff::Expr { }, }), ConstantLiteral::Float(value) => ruff::Expr::NumberLiteral(ruff::ExprNumberLiteral { + node_index: Default::default(), range, value: ruff::Number::Float(value), }), ConstantLiteral::Complex { real, imag } => { ruff::Expr::NumberLiteral(ruff::ExprNumberLiteral { + node_index: Default::default(), range, value: ruff::Number::Complex { real, imag }, }) } - ConstantLiteral::Ellipsis => { - ruff::Expr::EllipsisLiteral(ruff::ExprEllipsisLiteral { range }) - } + ConstantLiteral::Ellipsis => ruff::Expr::EllipsisLiteral(ruff::ExprEllipsisLiteral { + node_index: Default::default(), + range, + }), } } @@ -333,7 +350,11 @@ pub(super) fn number_literal_to_object( source_file: &SourceFile, constant: ruff::ExprNumberLiteral, ) -> PyObjectRef { - let ruff::ExprNumberLiteral { range, value } = constant; + let ruff::ExprNumberLiteral { + node_index: _, + range, + value, + } = constant; let c = match value { ruff::Number::Int(n) => Constant::new_int(n, range), ruff::Number::Float(n) => Constant::new_float(n, range), @@ -347,7 +368,11 @@ pub(super) fn string_literal_to_object( source_file: &SourceFile, constant: ruff::ExprStringLiteral, ) -> PyObjectRef { - let ruff::ExprStringLiteral { range, value } = constant; + let ruff::ExprStringLiteral { + node_index: _, + range, + value, + } = constant; let prefix = value .iter() .next() @@ -361,7 +386,11 @@ pub(super) fn bytes_literal_to_object( source_file: &SourceFile, constant: ruff::ExprBytesLiteral, ) -> PyObjectRef { - let ruff::ExprBytesLiteral { range, value } = constant; + let ruff::ExprBytesLiteral { + node_index: _, + range, + value, + } = constant; let bytes = value.as_slice().iter().flat_map(|b| b.value.iter()); let c = Constant::new_bytes(bytes.copied().collect(), range); c.ast_to_object(vm, source_file) @@ -372,7 +401,11 @@ pub(super) fn boolean_literal_to_object( source_file: &SourceFile, constant: ruff::ExprBooleanLiteral, ) -> PyObjectRef { - let ruff::ExprBooleanLiteral { range, value } = constant; + let ruff::ExprBooleanLiteral { + node_index: _, + range, + value, + } = constant; let c = Constant::new_bool(value, range); c.ast_to_object(vm, source_file) } @@ -382,7 +415,10 @@ pub(super) fn none_literal_to_object( source_file: &SourceFile, constant: ruff::ExprNoneLiteral, ) -> PyObjectRef { - let ruff::ExprNoneLiteral { range } = constant; + let ruff::ExprNoneLiteral { + node_index: _, + range, + } = constant; let c = Constant::new_none(range); c.ast_to_object(vm, source_file) } @@ -392,7 +428,10 @@ pub(super) fn ellipsis_literal_to_object( source_file: &SourceFile, constant: ruff::ExprEllipsisLiteral, ) -> PyObjectRef { - let ruff::ExprEllipsisLiteral { range } = constant; + let ruff::ExprEllipsisLiteral { + node_index: _, + range, + } = constant; let c = Constant::new_ellipsis(range); c.ast_to_object(vm, source_file) } diff --git a/vm/src/stdlib/ast/elif_else_clause.rs b/vm/src/stdlib/ast/elif_else_clause.rs index 2b427b1ec1..581fc499b8 100644 --- a/vm/src/stdlib/ast/elif_else_clause.rs +++ b/vm/src/stdlib/ast/elif_else_clause.rs @@ -7,7 +7,12 @@ pub(super) fn ast_to_object( vm: &VirtualMachine, source_file: &SourceFile, ) -> PyObjectRef { - let ruff::ElifElseClause { range, test, body } = clause; + let ruff::ElifElseClause { + node_index: _, + range, + test, + body, + } = clause; let Some(test) = test else { assert!(rest.len() == 0); return body.ast_to_object(vm, source_file); @@ -55,6 +60,7 @@ pub(super) fn ast_from_object( let elif_else_clauses = if let [ruff::Stmt::If(_)] = &*orelse { let Some(ruff::Stmt::If(ruff::StmtIf { + node_index: _, range, test, body, @@ -66,6 +72,7 @@ pub(super) fn ast_from_object( elif_else_clauses.insert( 0, ruff::ElifElseClause { + node_index: Default::default(), range, test: Some(*test), body, @@ -74,6 +81,7 @@ pub(super) fn ast_from_object( elif_else_clauses } else { vec![ruff::ElifElseClause { + node_index: Default::default(), range, test: None, body: orelse, @@ -81,6 +89,7 @@ pub(super) fn ast_from_object( }; Ok(ruff::StmtIf { + node_index: Default::default(), test, body, elif_else_clauses, diff --git a/vm/src/stdlib/ast/exception.rs b/vm/src/stdlib/ast/exception.rs index df6b391aa1..b5b3ca2709 100644 --- a/vm/src/stdlib/ast/exception.rs +++ b/vm/src/stdlib/ast/exception.rs @@ -35,6 +35,7 @@ impl Node for ruff::ExceptHandler { impl Node for ruff::ExceptHandlerExceptHandler { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, type_, name, body, @@ -63,6 +64,7 @@ impl Node for ruff::ExceptHandlerExceptHandler { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), type_: get_node_field_opt(_vm, &_object, "type")? .map(|obj| Node::ast_from_object(_vm, source_file, obj)) .transpose()?, diff --git a/vm/src/stdlib/ast/expression.rs b/vm/src/stdlib/ast/expression.rs index 31b65bd13a..83d7737438 100644 --- a/vm/src/stdlib/ast/expression.rs +++ b/vm/src/stdlib/ast/expression.rs @@ -36,6 +36,7 @@ impl Node for ruff::Expr { Self::NumberLiteral(cons) => constant::number_literal_to_object(vm, source_file, cons), Self::StringLiteral(cons) => constant::string_literal_to_object(vm, source_file, cons), Self::FString(cons) => string::fstring_to_object(vm, source_file, cons), + Self::TString(_) => unimplemented!(), Self::BytesLiteral(cons) => constant::bytes_literal_to_object(vm, source_file, cons), Self::BooleanLiteral(cons) => { constant::boolean_literal_to_object(vm, source_file, cons) @@ -145,7 +146,12 @@ impl Node for ruff::Expr { // constructor impl Node for ruff::ExprBoolOp { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { - let Self { op, values, range } = self; + let Self { + node_index: _, + op, + values, + range, + } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprBoolOp::static_type().to_owned()) .unwrap(); @@ -164,6 +170,7 @@ impl Node for ruff::ExprBoolOp { object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), op: Node::ast_from_object( vm, source_file, @@ -183,6 +190,7 @@ impl Node for ruff::ExprBoolOp { impl Node for ruff::ExprNamed { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, target, value, range, @@ -205,6 +213,7 @@ impl Node for ruff::ExprNamed { object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), target: Node::ast_from_object( vm, source_file, @@ -224,6 +233,7 @@ impl Node for ruff::ExprNamed { impl Node for ruff::ExprBinOp { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, left, op, right, @@ -249,6 +259,7 @@ impl Node for ruff::ExprBinOp { object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), left: Node::ast_from_object( vm, source_file, @@ -272,7 +283,12 @@ impl Node for ruff::ExprBinOp { // constructor impl Node for ruff::ExprUnaryOp { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { - let Self { op, operand, range } = self; + let Self { + node_index: _, + op, + operand, + range, + } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprUnaryOp::static_type().to_owned()) .unwrap(); @@ -290,6 +306,7 @@ impl Node for ruff::ExprUnaryOp { object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), op: Node::ast_from_object( vm, source_file, @@ -309,6 +326,7 @@ impl Node for ruff::ExprUnaryOp { impl Node for ruff::ExprLambda { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, parameters, body, range: _range, @@ -331,6 +349,7 @@ impl Node for ruff::ExprLambda { object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), parameters: Node::ast_from_object( vm, source_file, @@ -350,6 +369,7 @@ impl Node for ruff::ExprLambda { impl Node for ruff::ExprIf { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, test, body, orelse, @@ -375,6 +395,7 @@ impl Node for ruff::ExprIf { object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), test: Node::ast_from_object( vm, source_file, @@ -398,7 +419,11 @@ impl Node for ruff::ExprIf { // constructor impl Node for ruff::ExprDict { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { - let Self { items, range } = self; + let Self { + node_index: _, + items, + range, + } = self; let (keys, values) = items .into_iter() @@ -440,6 +465,7 @@ impl Node for ruff::ExprDict { .map(|(key, value)| ruff::DictItem { key, value }) .collect(); Ok(Self { + node_index: Default::default(), items, range: range_from_object(vm, source_file, object, "Dict")?, }) @@ -449,7 +475,11 @@ impl Node for ruff::ExprDict { // constructor impl Node for ruff::ExprSet { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { - let Self { elts, range } = self; + let Self { + node_index: _, + elts, + range, + } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprSet::static_type().to_owned()) .unwrap(); @@ -465,6 +495,7 @@ impl Node for ruff::ExprSet { object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), elts: Node::ast_from_object( vm, source_file, @@ -479,6 +510,7 @@ impl Node for ruff::ExprSet { impl Node for ruff::ExprListComp { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, elt, generators, range, @@ -501,6 +533,7 @@ impl Node for ruff::ExprListComp { object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), elt: Node::ast_from_object( vm, source_file, @@ -520,6 +553,7 @@ impl Node for ruff::ExprListComp { impl Node for ruff::ExprSetComp { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, elt, generators, range, @@ -542,6 +576,7 @@ impl Node for ruff::ExprSetComp { object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), elt: Node::ast_from_object( vm, source_file, @@ -561,6 +596,7 @@ impl Node for ruff::ExprSetComp { impl Node for ruff::ExprDictComp { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, key, value, generators, @@ -586,6 +622,7 @@ impl Node for ruff::ExprDictComp { object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), key: Node::ast_from_object( vm, source_file, @@ -610,6 +647,7 @@ impl Node for ruff::ExprDictComp { impl Node for ruff::ExprGenerator { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, elt, generators, range, @@ -633,6 +671,7 @@ impl Node for ruff::ExprGenerator { object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), elt: Node::ast_from_object( vm, source_file, @@ -653,7 +692,11 @@ impl Node for ruff::ExprGenerator { // constructor impl Node for ruff::ExprAwait { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { - let Self { value, range } = self; + let Self { + node_index: _, + value, + range, + } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprAwait::static_type().to_owned()) .unwrap(); @@ -669,6 +712,7 @@ impl Node for ruff::ExprAwait { object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), value: Node::ast_from_object( vm, source_file, @@ -682,7 +726,11 @@ impl Node for ruff::ExprAwait { // constructor impl Node for ruff::ExprYield { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { - let Self { value, range } = self; + let Self { + node_index: _, + value, + range, + } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprYield::static_type().to_owned()) .unwrap(); @@ -699,6 +747,7 @@ impl Node for ruff::ExprYield { object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), value: get_node_field_opt(vm, &object, "value")? .map(|obj| Node::ast_from_object(vm, source_file, obj)) .transpose()?, @@ -710,7 +759,11 @@ impl Node for ruff::ExprYield { // constructor impl Node for ruff::ExprYieldFrom { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { - let Self { value, range } = self; + let Self { + node_index: _, + value, + range, + } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprYieldFrom::static_type().to_owned()) .unwrap(); @@ -727,6 +780,7 @@ impl Node for ruff::ExprYieldFrom { object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), value: Node::ast_from_object( vm, source_file, @@ -741,6 +795,7 @@ impl Node for ruff::ExprYieldFrom { impl Node for ruff::ExprCompare { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, left, ops, comparators, @@ -770,6 +825,7 @@ impl Node for ruff::ExprCompare { object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), left: Node::ast_from_object( vm, source_file, @@ -800,6 +856,7 @@ impl Node for ruff::ExprCompare { impl Node for ruff::ExprCall { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, func, arguments, range, @@ -833,6 +890,7 @@ impl Node for ruff::ExprCall { object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), func: Node::ast_from_object( vm, source_file, @@ -859,6 +917,7 @@ impl Node for ruff::ExprCall { impl Node for ruff::ExprAttribute { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, value, attr, ctx, @@ -884,6 +943,7 @@ impl Node for ruff::ExprAttribute { object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), value: Node::ast_from_object( vm, source_file, @@ -908,6 +968,7 @@ impl Node for ruff::ExprAttribute { impl Node for ruff::ExprSubscript { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, value, slice, ctx, @@ -932,6 +993,7 @@ impl Node for ruff::ExprSubscript { object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), value: Node::ast_from_object( vm, source_file, @@ -955,7 +1017,12 @@ impl Node for ruff::ExprSubscript { // constructor impl Node for ruff::ExprStarred { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { - let Self { value, ctx, range } = self; + let Self { + node_index: _, + value, + ctx, + range, + } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprStarred::static_type().to_owned()) .unwrap(); @@ -973,6 +1040,7 @@ impl Node for ruff::ExprStarred { object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), value: Node::ast_from_object( vm, source_file, @@ -991,7 +1059,12 @@ impl Node for ruff::ExprStarred { // constructor impl Node for ruff::ExprName { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { - let Self { id, ctx, range } = self; + let Self { + node_index: _, + id, + ctx, + range, + } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprName::static_type().to_owned()) .unwrap(); @@ -1009,6 +1082,7 @@ impl Node for ruff::ExprName { object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), id: Node::ast_from_object(vm, source_file, get_node_field(vm, &object, "id", "Name")?)?, ctx: Node::ast_from_object( vm, @@ -1023,7 +1097,12 @@ impl Node for ruff::ExprName { // constructor impl Node for ruff::ExprList { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { - let Self { elts, ctx, range } = self; + let Self { + node_index: _, + elts, + ctx, + range, + } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprList::static_type().to_owned()) .unwrap(); @@ -1042,6 +1121,7 @@ impl Node for ruff::ExprList { object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), elts: Node::ast_from_object( vm, source_file, @@ -1061,6 +1141,7 @@ impl Node for ruff::ExprList { impl Node for ruff::ExprTuple { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, elts, ctx, range: _range, @@ -1084,6 +1165,7 @@ impl Node for ruff::ExprTuple { object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), elts: Node::ast_from_object( vm, source_file, @@ -1104,6 +1186,7 @@ impl Node for ruff::ExprTuple { impl Node for ruff::ExprSlice { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, lower, upper, step, @@ -1129,6 +1212,7 @@ impl Node for ruff::ExprSlice { object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), lower: get_node_field_opt(vm, &object, "lower")? .map(|obj| Node::ast_from_object(vm, source_file, obj)) .transpose()?, @@ -1185,6 +1269,7 @@ impl Node for ruff::ExprContext { impl Node for ruff::Comprehension { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, target, iter, ifs, @@ -1212,6 +1297,7 @@ impl Node for ruff::Comprehension { object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), target: Node::ast_from_object( vm, source_file, diff --git a/vm/src/stdlib/ast/module.rs b/vm/src/stdlib/ast/module.rs index 33ee4c567b..6fae8f10a3 100644 --- a/vm/src/stdlib/ast/module.rs +++ b/vm/src/stdlib/ast/module.rs @@ -66,6 +66,7 @@ impl Node for Mod { impl Node for ruff::ModModule { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, body, // type_ignores, range, @@ -95,6 +96,7 @@ impl Node for ruff::ModModule { object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), body: Node::ast_from_object( vm, source_file, @@ -147,7 +149,11 @@ impl Node for ModInteractive { // constructor impl Node for ruff::ModExpression { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { - let Self { body, range } = self; + let Self { + node_index: _, + body, + range, + } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeModExpression::static_type().to_owned()) .unwrap(); @@ -164,6 +170,7 @@ impl Node for ruff::ModExpression { object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), body: Node::ast_from_object( vm, source_file, diff --git a/vm/src/stdlib/ast/other.rs b/vm/src/stdlib/ast/other.rs index eddbef9748..780d7cc712 100644 --- a/vm/src/stdlib/ast/other.rs +++ b/vm/src/stdlib/ast/other.rs @@ -55,7 +55,11 @@ impl Node for ruff::Decorator { ) -> PyResult<Self> { let expression = ruff::Expr::ast_from_object(vm, source_file, object)?; let range = expression.range(); - Ok(Self { expression, range }) + Ok(Self { + node_index: Default::default(), + expression, + range, + }) } } @@ -63,6 +67,7 @@ impl Node for ruff::Decorator { impl Node for ruff::Alias { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, name, asname, range: _range, @@ -85,6 +90,7 @@ impl Node for ruff::Alias { object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), name: Node::ast_from_object( vm, source_file, @@ -102,6 +108,7 @@ impl Node for ruff::Alias { impl Node for ruff::WithItem { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, context_expr, optional_vars, range: _range, @@ -131,6 +138,7 @@ impl Node for ruff::WithItem { object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), context_expr: Node::ast_from_object( vm, source_file, diff --git a/vm/src/stdlib/ast/parameter.rs b/vm/src/stdlib/ast/parameter.rs index 347b3b4ec0..87fa736687 100644 --- a/vm/src/stdlib/ast/parameter.rs +++ b/vm/src/stdlib/ast/parameter.rs @@ -5,6 +5,7 @@ use rustpython_compiler_core::SourceFile; impl Node for ruff::Parameters { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, posonlyargs, args, vararg, @@ -80,6 +81,7 @@ impl Node for ruff::Parameters { let (posonlyargs, args) = merge_positional_parameter_defaults(posonlyargs, args, defaults); Ok(Self { + node_index: Default::default(), posonlyargs, args, vararg: get_node_field_opt(vm, &object, "vararg")? @@ -102,6 +104,7 @@ impl Node for ruff::Parameters { impl Node for ruff::Parameter { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, name, annotation, // type_comment, @@ -135,6 +138,7 @@ impl Node for ruff::Parameter { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), name: Node::ast_from_object( _vm, source_file, @@ -155,6 +159,7 @@ impl Node for ruff::Parameter { impl Node for ruff::Keyword { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, arg, value, range: _range, @@ -176,6 +181,7 @@ impl Node for ruff::Keyword { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), arg: get_node_field_opt(_vm, &_object, "arg")? .map(|obj| Node::ast_from_object(_vm, source_file, obj)) .transpose()?, @@ -328,6 +334,7 @@ fn merge_positional_parameter_defaults( let mut posonlyargs: Vec<_> = <Box<[_]> as IntoIterator>::into_iter(posonlyargs) .map(|parameter| ruff::ParameterWithDefault { + node_index: Default::default(), range: Default::default(), parameter, default: None, @@ -335,6 +342,7 @@ fn merge_positional_parameter_defaults( .collect(); let mut args: Vec<_> = <Box<[_]> as IntoIterator>::into_iter(args) .map(|parameter| ruff::ParameterWithDefault { + node_index: Default::default(), range: Default::default(), parameter, default: None, @@ -397,6 +405,7 @@ fn merge_keyword_parameter_defaults( ) -> Vec<ruff::ParameterWithDefault> { std::iter::zip(kw_only_args.keywords, defaults.defaults) .map(|(parameter, default)| ruff::ParameterWithDefault { + node_index: Default::default(), parameter, default, range: Default::default(), diff --git a/vm/src/stdlib/ast/pattern.rs b/vm/src/stdlib/ast/pattern.rs index 0c484d6787..9567ed38d4 100644 --- a/vm/src/stdlib/ast/pattern.rs +++ b/vm/src/stdlib/ast/pattern.rs @@ -5,6 +5,7 @@ use rustpython_compiler_core::SourceFile; impl Node for ruff::MatchCase { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, pattern, guard, body, @@ -29,6 +30,7 @@ impl Node for ruff::MatchCase { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), pattern: Node::ast_from_object( _vm, source_file, @@ -127,6 +129,7 @@ impl Node for ruff::Pattern { impl Node for ruff::PatternMatchValue { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, value, range: _range, } = self; @@ -146,6 +149,7 @@ impl Node for ruff::PatternMatchValue { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), value: Node::ast_from_object( _vm, source_file, @@ -160,6 +164,7 @@ impl Node for ruff::PatternMatchValue { impl Node for ruff::PatternMatchSingleton { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, value, range: _range, } = self; @@ -182,6 +187,7 @@ impl Node for ruff::PatternMatchSingleton { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), value: Node::ast_from_object( _vm, source_file, @@ -210,6 +216,7 @@ impl Node for ruff::Singleton { impl Node for ruff::PatternMatchSequence { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, patterns, range: _range, } = self; @@ -232,6 +239,7 @@ impl Node for ruff::PatternMatchSequence { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), patterns: Node::ast_from_object( _vm, source_file, @@ -246,6 +254,7 @@ impl Node for ruff::PatternMatchSequence { impl Node for ruff::PatternMatchMapping { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, keys, patterns, rest, @@ -274,6 +283,7 @@ impl Node for ruff::PatternMatchMapping { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), keys: Node::ast_from_object( _vm, source_file, @@ -296,6 +306,7 @@ impl Node for ruff::PatternMatchMapping { impl Node for ruff::PatternMatchClass { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, cls, arguments, range: _range, @@ -344,6 +355,7 @@ impl Node for ruff::PatternMatchClass { let (patterns, keywords) = merge_pattern_match_class(patterns, kwd_attrs, kwd_patterns); Ok(Self { + node_index: Default::default(), cls: Node::ast_from_object( vm, source_file, @@ -351,6 +363,7 @@ impl Node for ruff::PatternMatchClass { )?, range: range_from_object(vm, source_file, object, "MatchClass")?, arguments: ruff::PatternArguments { + node_index: Default::default(), range: Default::default(), patterns, keywords, @@ -416,6 +429,7 @@ impl Node for PatternMatchClassKeywordPatterns { impl Node for ruff::PatternMatchStar { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, name, range: _range, } = self; @@ -435,6 +449,7 @@ impl Node for ruff::PatternMatchStar { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), name: get_node_field_opt(_vm, &_object, "name")? .map(|obj| Node::ast_from_object(_vm, source_file, obj)) .transpose()?, @@ -447,6 +462,7 @@ impl Node for ruff::PatternMatchStar { impl Node for ruff::PatternMatchAs { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, pattern, name, range: _range, @@ -469,6 +485,7 @@ impl Node for ruff::PatternMatchAs { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), pattern: get_node_field_opt(_vm, &_object, "pattern")? .map(|obj| Node::ast_from_object(_vm, source_file, obj)) .transpose()?, @@ -484,6 +501,7 @@ impl Node for ruff::PatternMatchAs { impl Node for ruff::PatternMatchOr { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, patterns, range: _range, } = self; @@ -502,6 +520,7 @@ impl Node for ruff::PatternMatchOr { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), patterns: Node::ast_from_object( _vm, source_file, diff --git a/vm/src/stdlib/ast/python.rs b/vm/src/stdlib/ast/python.rs index d0b19d49da..40ab3ce72d 100644 --- a/vm/src/stdlib/ast/python.rs +++ b/vm/src/stdlib/ast/python.rs @@ -33,14 +33,14 @@ pub(crate) mod _ast { zelf.set_attr(name, arg, vm)?; } for (key, value) in args.kwargs { - if let Some(pos) = fields.iter().position(|f| f.as_str() == key) { - if pos < n_args { - return Err(vm.new_type_error(format!( - "{} got multiple values for argument '{}'", - zelf.class().name(), - key - ))); - } + if let Some(pos) = fields.iter().position(|f| f.as_str() == key) + && pos < n_args + { + return Err(vm.new_type_error(format!( + "{} got multiple values for argument '{}'", + zelf.class().name(), + key + ))); } zelf.set_attr(vm.ctx.intern_str(key), value, vm)?; } diff --git a/vm/src/stdlib/ast/statement.rs b/vm/src/stdlib/ast/statement.rs index fab7ea7696..5925ca1fc2 100644 --- a/vm/src/stdlib/ast/statement.rs +++ b/vm/src/stdlib/ast/statement.rs @@ -172,6 +172,7 @@ impl Node for ruff::Stmt { impl Node for ruff::StmtFunctionDef { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, name, parameters, body, @@ -225,6 +226,7 @@ impl Node for ruff::StmtFunctionDef { let _cls = _object.class(); let is_async = _cls.is(pyast::NodeStmtAsyncFunctionDef::static_type()); Ok(Self { + node_index: Default::default(), name: Node::ast_from_object( _vm, source_file, @@ -267,6 +269,7 @@ impl Node for ruff::StmtFunctionDef { impl Node for ruff::StmtClassDef { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, name, arguments, body, @@ -318,6 +321,7 @@ impl Node for ruff::StmtClassDef { get_node_field(_vm, &_object, "keywords", "ClassDef")?, )?; Ok(Self { + node_index: Default::default(), name: Node::ast_from_object( _vm, source_file, @@ -347,6 +351,7 @@ impl Node for ruff::StmtClassDef { impl Node for ruff::StmtReturn { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, value, range: _range, } = self; @@ -365,6 +370,7 @@ impl Node for ruff::StmtReturn { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), value: get_node_field_opt(_vm, &_object, "value")? .map(|obj| Node::ast_from_object(_vm, source_file, obj)) .transpose()?, @@ -376,6 +382,7 @@ impl Node for ruff::StmtReturn { impl Node for ruff::StmtDelete { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, targets, range: _range, } = self; @@ -394,6 +401,7 @@ impl Node for ruff::StmtDelete { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), targets: Node::ast_from_object( _vm, source_file, @@ -408,6 +416,7 @@ impl Node for ruff::StmtDelete { impl Node for ruff::StmtAssign { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, targets, value, // type_comment, @@ -432,6 +441,7 @@ impl Node for ruff::StmtAssign { object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), targets: Node::ast_from_object( vm, source_file, @@ -454,6 +464,7 @@ impl Node for ruff::StmtAssign { impl Node for ruff::StmtTypeAlias { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, name, type_params, value, @@ -483,6 +494,7 @@ impl Node for ruff::StmtTypeAlias { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), name: Node::ast_from_object( _vm, source_file, @@ -507,6 +519,7 @@ impl Node for ruff::StmtTypeAlias { impl Node for ruff::StmtAugAssign { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, target, op, value, @@ -531,6 +544,7 @@ impl Node for ruff::StmtAugAssign { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), target: Node::ast_from_object( _vm, source_file, @@ -555,6 +569,7 @@ impl Node for ruff::StmtAugAssign { impl Node for ruff::StmtAnnAssign { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, target, annotation, value, @@ -586,6 +601,7 @@ impl Node for ruff::StmtAnnAssign { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), target: Node::ast_from_object( _vm, source_file, @@ -613,6 +629,7 @@ impl Node for ruff::StmtAnnAssign { impl Node for ruff::StmtFor { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, is_async, target, iter, @@ -656,6 +673,7 @@ impl Node for ruff::StmtFor { ); let is_async = _cls.is(pyast::NodeStmtAsyncFor::static_type()); Ok(Self { + node_index: Default::default(), target: Node::ast_from_object( _vm, source_file, @@ -689,6 +707,7 @@ impl Node for ruff::StmtFor { impl Node for ruff::StmtWhile { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, test, body, orelse, @@ -714,6 +733,7 @@ impl Node for ruff::StmtWhile { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), test: Node::ast_from_object( _vm, source_file, @@ -737,6 +757,7 @@ impl Node for ruff::StmtWhile { impl Node for ruff::StmtIf { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, test, body, range, @@ -744,6 +765,7 @@ impl Node for ruff::StmtIf { } = self; elif_else_clause::ast_to_object( ruff::ElifElseClause { + node_index: Default::default(), range, test: Some(*test), body, @@ -765,6 +787,7 @@ impl Node for ruff::StmtIf { impl Node for ruff::StmtWith { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, is_async, items, body, @@ -801,6 +824,7 @@ impl Node for ruff::StmtWith { ); let is_async = _cls.is(pyast::NodeStmtAsyncWith::static_type()); Ok(Self { + node_index: Default::default(), items: Node::ast_from_object( _vm, source_file, @@ -823,6 +847,7 @@ impl Node for ruff::StmtWith { impl Node for ruff::StmtMatch { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, subject, cases, range: _range, @@ -844,6 +869,7 @@ impl Node for ruff::StmtMatch { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), subject: Node::ast_from_object( _vm, source_file, @@ -862,6 +888,7 @@ impl Node for ruff::StmtMatch { impl Node for ruff::StmtRaise { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, exc, cause, range: _range, @@ -883,6 +910,7 @@ impl Node for ruff::StmtRaise { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), exc: get_node_field_opt(_vm, &_object, "exc")? .map(|obj| Node::ast_from_object(_vm, source_file, obj)) .transpose()?, @@ -897,6 +925,7 @@ impl Node for ruff::StmtRaise { impl Node for ruff::StmtTry { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, body, handlers, orelse, @@ -940,6 +969,7 @@ impl Node for ruff::StmtTry { ); Ok(Self { + node_index: Default::default(), body: Node::ast_from_object( _vm, source_file, @@ -969,6 +999,7 @@ impl Node for ruff::StmtTry { impl Node for ruff::StmtAssert { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, test, msg, range: _range, @@ -990,6 +1021,7 @@ impl Node for ruff::StmtAssert { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), test: Node::ast_from_object( _vm, source_file, @@ -1006,6 +1038,7 @@ impl Node for ruff::StmtAssert { impl Node for ruff::StmtImport { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, names, range: _range, } = self; @@ -1024,6 +1057,7 @@ impl Node for ruff::StmtImport { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), names: Node::ast_from_object( _vm, source_file, @@ -1037,6 +1071,7 @@ impl Node for ruff::StmtImport { impl Node for ruff::StmtImportFrom { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, module, names, level, @@ -1061,6 +1096,7 @@ impl Node for ruff::StmtImportFrom { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), module: get_node_field_opt(vm, &_object, "module")? .map(|obj| Node::ast_from_object(vm, source_file, obj)) .transpose()?, @@ -1081,6 +1117,7 @@ impl Node for ruff::StmtImportFrom { impl Node for ruff::StmtGlobal { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, names, range: _range, } = self; @@ -1099,6 +1136,7 @@ impl Node for ruff::StmtGlobal { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), names: Node::ast_from_object( _vm, source_file, @@ -1112,6 +1150,7 @@ impl Node for ruff::StmtGlobal { impl Node for ruff::StmtNonlocal { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, names, range: _range, } = self; @@ -1130,6 +1169,7 @@ impl Node for ruff::StmtNonlocal { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), names: Node::ast_from_object( _vm, source_file, @@ -1143,6 +1183,7 @@ impl Node for ruff::StmtNonlocal { impl Node for ruff::StmtExpr { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, value, range: _range, } = self; @@ -1161,6 +1202,7 @@ impl Node for ruff::StmtExpr { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), value: Node::ast_from_object( _vm, source_file, @@ -1173,7 +1215,10 @@ impl Node for ruff::StmtExpr { // constructor impl Node for ruff::StmtPass { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { - let Self { range: _range } = self; + let Self { + node_index: _, + range: _range, + } = self; let node = NodeAst .into_ref_with_type(_vm, pyast::NodeStmtPass::static_type().to_owned()) .unwrap(); @@ -1187,6 +1232,7 @@ impl Node for ruff::StmtPass { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), range: range_from_object(_vm, source_file, _object, "Pass")?, }) } @@ -1194,7 +1240,10 @@ impl Node for ruff::StmtPass { // constructor impl Node for ruff::StmtBreak { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { - let Self { range: _range } = self; + let Self { + node_index: _, + range: _range, + } = self; let node = NodeAst .into_ref_with_type(_vm, pyast::NodeStmtBreak::static_type().to_owned()) .unwrap(); @@ -1209,6 +1258,7 @@ impl Node for ruff::StmtBreak { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), range: range_from_object(_vm, source_file, _object, "Break")?, }) } @@ -1217,7 +1267,10 @@ impl Node for ruff::StmtBreak { // constructor impl Node for ruff::StmtContinue { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { - let Self { range: _range } = self; + let Self { + node_index: _, + range: _range, + } = self; let node = NodeAst .into_ref_with_type(_vm, pyast::NodeStmtContinue::static_type().to_owned()) .unwrap(); @@ -1231,6 +1284,7 @@ impl Node for ruff::StmtContinue { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), range: range_from_object(_vm, source_file, _object, "Continue")?, }) } diff --git a/vm/src/stdlib/ast/string.rs b/vm/src/stdlib/ast/string.rs index 5d8654270d..f3df8d9926 100644 --- a/vm/src/stdlib/ast/string.rs +++ b/vm/src/stdlib/ast/string.rs @@ -5,24 +5,26 @@ fn ruff_fstring_value_into_iter( mut fstring_value: ruff::FStringValue, ) -> impl Iterator<Item = ruff::FStringPart> + 'static { let default = ruff::FStringPart::FString(ruff::FString { + node_index: Default::default(), range: Default::default(), elements: Default::default(), flags: ruff::FStringFlags::empty(), }); (0..fstring_value.as_slice().len()).map(move |i| { - let fstring_value = &mut fstring_value; - let tmp = fstring_value.into_iter().nth(i).unwrap(); + let tmp = fstring_value.iter_mut().nth(i).unwrap(); std::mem::replace(tmp, default.clone()) }) } fn ruff_fstring_element_into_iter( - mut fstring_element: ruff::FStringElements, -) -> impl Iterator<Item = ruff::FStringElement> + 'static { - let default = ruff::FStringElement::Literal(ruff::FStringLiteralElement { - range: Default::default(), - value: Default::default(), - }); + mut fstring_element: ruff::InterpolatedStringElements, +) -> impl Iterator<Item = ruff::InterpolatedStringElement> + 'static { + let default = + ruff::InterpolatedStringElement::Literal(ruff::InterpolatedStringLiteralElement { + node_index: Default::default(), + range: Default::default(), + value: Default::default(), + }); (0..fstring_element.into_iter().len()).map(move |i| { let fstring_element = &mut fstring_element; let tmp = fstring_element.into_iter().nth(i).unwrap(); @@ -36,6 +38,7 @@ fn fstring_part_to_joined_str_part(fstring_part: ruff::FStringPart) -> Vec<Joine range, value, flags, + node_index: _, }) => { vec![JoinedStrPart::Constant(Constant::new_str( value, @@ -47,27 +50,33 @@ fn fstring_part_to_joined_str_part(fstring_part: ruff::FStringPart) -> Vec<Joine range: _, elements, flags: _, // TODO + node_index: _, }) => ruff_fstring_element_into_iter(elements) .map(ruff_fstring_element_to_joined_str_part) .collect(), } } -fn ruff_fstring_element_to_joined_str_part(element: ruff::FStringElement) -> JoinedStrPart { +fn ruff_fstring_element_to_joined_str_part( + element: ruff::InterpolatedStringElement, +) -> JoinedStrPart { match element { - ruff::FStringElement::Literal(ruff::FStringLiteralElement { range, value }) => { - JoinedStrPart::Constant(Constant::new_str( - value, - ruff::str_prefix::StringLiteralPrefix::Empty, - range, - )) - } - ruff::FStringElement::Expression(ruff::FStringExpressionElement { + ruff::InterpolatedStringElement::Literal(ruff::InterpolatedStringLiteralElement { + range, + value, + node_index: _, + }) => JoinedStrPart::Constant(Constant::new_str( + value, + ruff::str_prefix::StringLiteralPrefix::Empty, + range, + )), + ruff::InterpolatedStringElement::Interpolation(ruff::InterpolatedElement { range, expression, debug_text: _, // TODO: What is this? conversion, format_spec, + node_index: _, }) => JoinedStrPart::FormattedValue(FormattedValue { value: expression, conversion, @@ -78,12 +87,16 @@ fn ruff_fstring_element_to_joined_str_part(element: ruff::FStringElement) -> Joi } fn ruff_format_spec_to_joined_str( - format_spec: Option<Box<ruff::FStringFormatSpec>>, + format_spec: Option<Box<ruff::InterpolatedStringFormatSpec>>, ) -> Option<Box<JoinedStr>> { match format_spec { None => None, Some(format_spec) => { - let ruff::FStringFormatSpec { range, elements } = *format_spec; + let ruff::InterpolatedStringFormatSpec { + range, + elements, + node_index: _, + } = *format_spec; let values: Vec<_> = ruff_fstring_element_into_iter(elements) .map(ruff_fstring_element_to_joined_str_part) .collect(); @@ -93,45 +106,37 @@ fn ruff_format_spec_to_joined_str( } } -fn ruff_fstring_element_to_ruff_fstring_part(element: ruff::FStringElement) -> ruff::FStringPart { +fn ruff_fstring_element_to_ruff_fstring_part( + element: ruff::InterpolatedStringElement, +) -> ruff::FStringPart { match element { - ruff::FStringElement::Literal(value) => { - let ruff::FStringLiteralElement { range, value } = value; - ruff::FStringPart::Literal(ruff::StringLiteral { + ruff::InterpolatedStringElement::Literal(value) => { + let ruff::InterpolatedStringLiteralElement { + node_index, range, value, - flags: ruff::StringLiteralFlags::empty(), - }) - } - ruff::FStringElement::Expression(value) => { - let ruff::FStringExpressionElement { - range, - expression, - debug_text, - conversion, - format_spec, } = value; - ruff::FStringPart::FString(ruff::FString { + ruff::FStringPart::Literal(ruff::StringLiteral { + node_index, range, - elements: vec![ruff::FStringElement::Expression( - ruff::FStringExpressionElement { - range, - expression, - debug_text, - conversion, - format_spec, - }, - )] - .into(), - flags: ruff::FStringFlags::empty(), + value, + flags: ruff::StringLiteralFlags::empty(), }) } + ruff::InterpolatedStringElement::Interpolation(ruff::InterpolatedElement { + range, .. + }) => ruff::FStringPart::FString(ruff::FString { + node_index: Default::default(), + range, + elements: vec![element].into(), + flags: ruff::FStringFlags::empty(), + }), } } fn joined_str_to_ruff_format_spec( joined_str: Option<Box<JoinedStr>>, -) -> Option<Box<ruff::FStringFormatSpec>> { +) -> Option<Box<ruff::InterpolatedStringFormatSpec>> { match joined_str { None => None, Some(joined_str) => { @@ -139,7 +144,8 @@ fn joined_str_to_ruff_format_spec( let elements: Vec<_> = Box::into_iter(values) .map(joined_str_part_to_ruff_fstring_element) .collect(); - let format_spec = ruff::FStringFormatSpec { + let format_spec = ruff::InterpolatedStringFormatSpec { + node_index: Default::default(), range, elements: elements.into(), }; @@ -158,10 +164,12 @@ impl JoinedStr { pub(super) fn into_expr(self) -> ruff::Expr { let Self { range, values } = self; ruff::Expr::FString(ruff::ExprFString { + node_index: Default::default(), range: Default::default(), value: match values.len() { // ruff represents an empty fstring like this: 0 => ruff::FStringValue::single(ruff::FString { + node_index: Default::default(), range, elements: vec![].into(), flags: ruff::FStringFlags::empty(), @@ -170,6 +178,7 @@ impl JoinedStr { Box::<[_]>::into_iter(values) .map(joined_str_part_to_ruff_fstring_element) .map(|element| ruff::FString { + node_index: Default::default(), range, elements: vec![element].into(), flags: ruff::FStringFlags::empty(), @@ -188,10 +197,11 @@ impl JoinedStr { } } -fn joined_str_part_to_ruff_fstring_element(part: JoinedStrPart) -> ruff::FStringElement { +fn joined_str_part_to_ruff_fstring_element(part: JoinedStrPart) -> ruff::InterpolatedStringElement { match part { JoinedStrPart::FormattedValue(value) => { - ruff::FStringElement::Expression(ruff::FStringExpressionElement { + ruff::InterpolatedStringElement::Interpolation(ruff::InterpolatedElement { + node_index: Default::default(), range: value.range, expression: value.value.clone(), debug_text: None, // TODO: What is this? @@ -200,7 +210,8 @@ fn joined_str_part_to_ruff_fstring_element(part: JoinedStrPart) -> ruff::FString }) } JoinedStrPart::Constant(value) => { - ruff::FStringElement::Literal(ruff::FStringLiteralElement { + ruff::InterpolatedStringElement::Literal(ruff::InterpolatedStringLiteralElement { + node_index: Default::default(), range: value.range, value: match value.value { ConstantLiteral::Str { value, .. } => value, @@ -344,7 +355,11 @@ pub(super) fn fstring_to_object( source_file: &SourceFile, expression: ruff::ExprFString, ) -> PyObjectRef { - let ruff::ExprFString { range, value } = expression; + let ruff::ExprFString { + range, + value, + node_index: _, + } = expression; let values: Vec<_> = ruff_fstring_value_into_iter(value) .flat_map(fstring_part_to_joined_str_part) .collect(); diff --git a/vm/src/stdlib/ast/type_parameters.rs b/vm/src/stdlib/ast/type_parameters.rs index 505cd04d28..017470f7e6 100644 --- a/vm/src/stdlib/ast/type_parameters.rs +++ b/vm/src/stdlib/ast/type_parameters.rs @@ -15,7 +15,11 @@ impl Node for ruff::TypeParams { let range = Option::zip(type_params.first(), type_params.last()) .map(|(first, last)| first.range().cover(last.range())) .unwrap_or_default(); - Ok(Self { type_params, range }) + Ok(Self { + node_index: Default::default(), + type_params, + range, + }) } fn is_none(&self) -> bool { @@ -70,6 +74,7 @@ impl Node for ruff::TypeParam { impl Node for ruff::TypeParamTypeVar { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, name, bound, range: _range, @@ -93,6 +98,7 @@ impl Node for ruff::TypeParamTypeVar { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), name: Node::ast_from_object( _vm, source_file, @@ -115,6 +121,7 @@ impl Node for ruff::TypeParamTypeVar { impl Node for ruff::TypeParamParamSpec { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, name, range: _range, default, @@ -141,6 +148,7 @@ impl Node for ruff::TypeParamParamSpec { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), name: Node::ast_from_object( _vm, source_file, @@ -160,6 +168,7 @@ impl Node for ruff::TypeParamParamSpec { impl Node for ruff::TypeParamTypeVarTuple { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { + node_index: _, name, range: _range, default, @@ -189,6 +198,7 @@ impl Node for ruff::TypeParamTypeVarTuple { _object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { + node_index: Default::default(), name: Node::ast_from_object( _vm, source_file, diff --git a/vm/src/stdlib/builtins.rs b/vm/src/stdlib/builtins.rs index 52eb698fbb..8cf99770e7 100644 --- a/vm/src/stdlib/builtins.rs +++ b/vm/src/stdlib/builtins.rs @@ -489,6 +489,8 @@ mod builtins { Err(vm.new_exception_empty(vm.ctx.exceptions.keyboard_interrupt.to_owned())) } ReadlineResult::Io(e) => Err(vm.new_os_error(e.to_string())), + #[cfg(unix)] + ReadlineResult::OsError(num) => Err(vm.new_os_error(num.to_string())), ReadlineResult::Other(e) => Err(vm.new_runtime_error(e.to_string())), } } else { @@ -969,17 +971,13 @@ mod builtins { if let Ok(type_params) = function .as_object() .get_attr(identifier!(vm, __type_params__), vm) + && let Some(type_params_tuple) = type_params.downcast_ref::<PyTuple>() + && !type_params_tuple.is_empty() { - if let Some(type_params_tuple) = type_params.downcast_ref::<PyTuple>() { - if !type_params_tuple.is_empty() { - // Set .type_params in namespace so the compiler-generated code can use it - namespace.as_object().set_item( - vm.ctx.intern_str(".type_params"), - type_params, - vm, - )?; - } - } + // Set .type_params in namespace so the compiler-generated code can use it + namespace + .as_object() + .set_item(vm.ctx.intern_str(".type_params"), type_params, vm)?; } let classcell = function.invoke_with_locals(().into(), Some(namespace.clone()), vm)?; @@ -1006,14 +1004,12 @@ mod builtins { if let Ok(type_params) = function .as_object() .get_attr(identifier!(vm, __type_params__), vm) + && let Some(type_params_tuple) = type_params.downcast_ref::<PyTuple>() + && !type_params_tuple.is_empty() { - if let Some(type_params_tuple) = type_params.downcast_ref::<PyTuple>() { - if !type_params_tuple.is_empty() { - class.set_attr(identifier!(vm, __type_params__), type_params.clone(), vm)?; - // Also set __parameters__ for compatibility with typing module - class.set_attr(identifier!(vm, __parameters__), type_params, vm)?; - } - } + class.set_attr(identifier!(vm, __type_params__), type_params.clone(), vm)?; + // Also set __parameters__ for compatibility with typing module + class.set_attr(identifier!(vm, __parameters__), type_params, vm)?; } if let Some(ref classcell) = classcell { diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index fe6e20f08a..e06560780c 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -3394,11 +3394,9 @@ mod _io { output.to_mut().insert(0, '\r'.into()); self.pendingcr = false; } - if !final_ { - if let Some(s) = output.strip_suffix("\r".as_ref()) { - output = Cow::Owned(s.to_owned()); - self.pendingcr = true; - } + if !final_ && let Some(s) = output.strip_suffix("\r".as_ref()) { + output = Cow::Owned(s.to_owned()); + self.pendingcr = true; } if output.is_empty() { @@ -3972,7 +3970,7 @@ mod _io { // check file descriptor validity #[cfg(unix)] if let Ok(crate::ospath::OsPathOrFd::Fd(fd)) = file.clone().try_into_value(vm) { - nix::fcntl::fcntl(fd.as_raw(), nix::fcntl::F_GETFD) + nix::fcntl::fcntl(fd, nix::fcntl::F_GETFD) .map_err(|_| crate::stdlib::os::errno_err(vm))?; } diff --git a/vm/src/stdlib/itertools.rs b/vm/src/stdlib/itertools.rs index addfddd176..ba629a8877 100644 --- a/vm/src/stdlib/itertools.rs +++ b/vm/src/stdlib/itertools.rs @@ -19,7 +19,7 @@ mod decl { identifier, protocol::{PyIter, PyIterReturn, PyNumber}, raise_if_stop, - stdlib::sys, + stdlib::{sys, warnings}, types::{Constructor, IterNext, Iterable, Representable, SelfIter}, }; use crossbeam_utils::atomic::AtomicCell; @@ -29,6 +29,15 @@ mod decl { use num_traits::{Signed, ToPrimitive}; use std::fmt; + fn pickle_deprecation(vm: &VirtualMachine) -> PyResult<()> { + warnings::warn( + vm.ctx.exceptions.deprecation_warning, + "Itertool pickle/copy/deepcopy support will be removed in a Python 3.14.".to_owned(), + 1, + vm, + ) + } + #[pyattr] #[pyclass(name = "chain")] #[derive(Debug, PyPayload)] @@ -74,6 +83,7 @@ mod decl { #[pymethod] fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + pickle_deprecation(vm)?; let source = zelf.source.read().clone(); let active = zelf.active.read().clone(); let cls = zelf.class().to_owned(); @@ -204,7 +214,8 @@ mod decl { #[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] impl PyItertoolsCompress { #[pymethod] - fn __reduce__(zelf: PyRef<Self>) -> (PyTypeRef, (PyIter, PyIter)) { + fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> (PyTypeRef, (PyIter, PyIter)) { + let _ = pickle_deprecation(vm); ( zelf.class().to_owned(), (zelf.data.clone(), zelf.selectors.clone()), @@ -274,7 +285,8 @@ mod decl { // if (lz->cnt == PY_SSIZE_T_MAX) // return Py_BuildValue("0(00)", Py_TYPE(lz), lz->long_cnt, lz->long_step); #[pymethod] - fn __reduce__(zelf: PyRef<Self>) -> (PyTypeRef, (PyObjectRef,)) { + fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> (PyTypeRef, (PyObjectRef,)) { + let _ = pickle_deprecation(vm); (zelf.class().to_owned(), (zelf.cur.read().clone(),)) } } @@ -406,6 +418,7 @@ mod decl { #[pymethod] fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + pickle_deprecation(vm)?; let cls = zelf.class().to_owned(); Ok(match zelf.times { Some(ref times) => vm.new_tuple((cls, (zelf.object.clone(), *times.read()))), @@ -474,7 +487,11 @@ mod decl { #[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] impl PyItertoolsStarmap { #[pymethod] - fn __reduce__(zelf: PyRef<Self>) -> (PyTypeRef, (PyObjectRef, PyIter)) { + fn __reduce__( + zelf: PyRef<Self>, + vm: &VirtualMachine, + ) -> (PyTypeRef, (PyObjectRef, PyIter)) { + let _ = pickle_deprecation(vm); ( zelf.class().to_owned(), (zelf.function.clone(), zelf.iterable.clone()), @@ -539,7 +556,11 @@ mod decl { #[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] impl PyItertoolsTakewhile { #[pymethod] - fn __reduce__(zelf: PyRef<Self>) -> (PyTypeRef, (PyObjectRef, PyIter), u32) { + fn __reduce__( + zelf: PyRef<Self>, + vm: &VirtualMachine, + ) -> (PyTypeRef, (PyObjectRef, PyIter), u32) { + let _ = pickle_deprecation(vm); ( zelf.class().to_owned(), (zelf.predicate.clone(), zelf.iterable.clone()), @@ -623,7 +644,11 @@ mod decl { #[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] impl PyItertoolsDropwhile { #[pymethod] - fn __reduce__(zelf: PyRef<Self>) -> (PyTypeRef, (PyObjectRef, PyIter), u32) { + fn __reduce__( + zelf: PyRef<Self>, + vm: &VirtualMachine, + ) -> (PyTypeRef, (PyObjectRef, PyIter), u32) { + let _ = pickle_deprecation(vm); ( zelf.class().to_owned(), (zelf.predicate.clone().into(), zelf.iterable.clone()), @@ -937,6 +962,7 @@ mod decl { #[pymethod] fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + pickle_deprecation(vm)?; let cls = zelf.class().to_owned(); let itr = zelf.iterable.clone(); let cur = zelf.cur.take(); @@ -976,10 +1002,10 @@ mod decl { zelf.cur.fetch_add(1); } - if let Some(stop) = zelf.stop { - if zelf.cur.load() >= stop { - return Ok(PyIterReturn::StopIteration(None)); - } + if let Some(stop) = zelf.stop + && zelf.cur.load() >= stop + { + return Ok(PyIterReturn::StopIteration(None)); } let obj = raise_if_stop!(zelf.iterable.next(vm)?); @@ -1032,7 +1058,11 @@ mod decl { #[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] impl PyItertoolsFilterFalse { #[pymethod] - fn __reduce__(zelf: PyRef<Self>) -> (PyTypeRef, (PyObjectRef, PyIter)) { + fn __reduce__( + zelf: PyRef<Self>, + vm: &VirtualMachine, + ) -> (PyTypeRef, (PyObjectRef, PyIter)) { + let _ = pickle_deprecation(vm); ( zelf.class().to_owned(), (zelf.predicate.clone(), zelf.iterable.clone()), @@ -1110,6 +1140,7 @@ mod decl { #[pymethod] fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyTupleRef { + let _ = pickle_deprecation(vm); let class = zelf.class().to_owned(); let bin_op = zelf.bin_op.clone(); let it = zelf.iterable.clone(); @@ -1374,6 +1405,7 @@ mod decl { #[pymethod] fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyTupleRef { + let _ = pickle_deprecation(vm); let class = zelf.class().to_owned(); if zelf.stop.load() { @@ -1483,6 +1515,7 @@ mod decl { impl PyItertoolsCombinations { #[pymethod] fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyTupleRef { + let _ = pickle_deprecation(vm); let r = zelf.r.load(); let class = zelf.class().to_owned(); @@ -1724,6 +1757,7 @@ mod decl { impl PyItertoolsPermutations { #[pymethod] fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyRef<PyTuple> { + let _ = pickle_deprecation(vm); vm.new_tuple(( zelf.class().to_owned(), vm.new_tuple((zelf.pool.clone(), vm.ctx.new_int(zelf.r.load()))), @@ -1837,6 +1871,7 @@ mod decl { impl PyItertoolsZipLongest { #[pymethod] fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + pickle_deprecation(vm)?; let args: Vec<PyObjectRef> = zelf .iterators .iter() diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index 2c4442900b..9fae516fe0 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -18,6 +18,7 @@ mod stat; mod string; #[cfg(feature = "compiler")] mod symtable; +mod sysconfig; mod sysconfigdata; #[cfg(feature = "threading")] pub mod thread; @@ -97,6 +98,7 @@ pub fn get_module_inits() -> StdlibMap { "_signal" => signal::make_module, "_sre" => sre::make_module, "_stat" => stat::make_module, + "_sysconfig" => sysconfig::make_module, "_string" => string::make_module, "time" => time::make_module, "_typing" => typing::make_module, diff --git a/vm/src/stdlib/nt.rs b/vm/src/stdlib/nt.rs index b7eeac80bf..cd2ff476ff 100644 --- a/vm/src/stdlib/nt.rs +++ b/vm/src/stdlib/nt.rs @@ -363,20 +363,20 @@ pub(crate) mod module { return Ok((total, free)); } let err = io::Error::last_os_error(); - if err.raw_os_error() == Some(Foundation::ERROR_DIRECTORY as i32) { - if let Some(parent) = path.as_ref().parent() { - let parent = widestring::WideCString::from_os_str(parent).unwrap(); - - let ret = unsafe { - GetDiskFreeSpaceExW(parent.as_ptr(), &mut _free_to_me, &mut total, &mut free) - }; - - return if ret == 0 { - Err(errno_err(vm)) - } else { - Ok((total, free)) - }; - } + if err.raw_os_error() == Some(Foundation::ERROR_DIRECTORY as i32) + && let Some(parent) = path.as_ref().parent() + { + let parent = widestring::WideCString::from_os_str(parent).unwrap(); + + let ret = unsafe { + GetDiskFreeSpaceExW(parent.as_ptr(), &mut _free_to_me, &mut total, &mut free) + }; + + return if ret == 0 { + Err(errno_err(vm)) + } else { + Ok((total, free)) + }; } Err(err.to_pyexception(vm)) } diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 35c164d6d2..1c65d4d12c 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -378,8 +378,7 @@ pub(super) mod _os { #[cfg(all(unix, not(target_os = "redox")))] { use rustpython_common::os::ffi::OsStrExt; - let new_fd = - nix::unistd::dup(fno.as_raw()).map_err(|e| e.into_pyexception(vm))?; + let new_fd = nix::unistd::dup(fno).map_err(|e| e.into_pyexception(vm))?; let mut dir = nix::dir::Dir::from_fd(new_fd).map_err(|e| e.into_pyexception(vm))?; dir.iter() diff --git a/vm/src/stdlib/posix.rs b/vm/src/stdlib/posix.rs index 84a52e3d24..e8a01d0e68 100644 --- a/vm/src/stdlib/posix.rs +++ b/vm/src/stdlib/posix.rs @@ -1,9 +1,9 @@ // spell-checker:disable use crate::{PyRef, VirtualMachine, builtins::PyModule}; -use std::os::unix::io::RawFd; +use std::os::fd::BorrowedFd; -pub fn raw_set_inheritable(fd: RawFd, inheritable: bool) -> nix::Result<()> { +pub fn set_inheritable(fd: BorrowedFd<'_>, inheritable: bool) -> nix::Result<()> { use nix::fcntl; let flags = fcntl::FdFlag::from_bits_truncate(fcntl::fcntl(fd, fcntl::FcntlArg::F_GETFD)?); let mut new_flags = flags; @@ -43,7 +43,7 @@ pub mod module { env, ffi::{CStr, CString}, fs, io, - os::fd::{AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd, RawFd}, + os::fd::{AsFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd}, }; use strum_macros::{EnumIter, EnumString}; @@ -460,7 +460,7 @@ pub mod module { let dst = args.dst.into_cstring(vm)?; #[cfg(not(target_os = "redox"))] { - nix::unistd::symlinkat(&*src, args.dir_fd.raw_opt(), &*dst) + nix::unistd::symlinkat(&*src, args.dir_fd.get(), &*dst) .map_err(|err| err.into_pyexception(vm)) } #[cfg(target_os = "redox")] @@ -473,7 +473,7 @@ pub mod module { #[cfg(not(target_os = "redox"))] #[pyfunction] - fn fchdir(fd: RawFd, vm: &VirtualMachine) -> PyResult<()> { + fn fchdir(fd: BorrowedFd<'_>, vm: &VirtualMachine) -> PyResult<()> { nix::unistd::fchdir(fd).map_err(|err| err.into_pyexception(vm)) } @@ -522,12 +522,11 @@ pub mod module { nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW }; - let dir_fd = dir_fd.raw_opt(); match path { OsPathOrFd::Path(ref p) => { - nix::unistd::fchownat(dir_fd, p.path.as_os_str(), uid, gid, flag) + nix::unistd::fchownat(dir_fd.get(), p.path.as_os_str(), uid, gid, flag) } - OsPathOrFd::Fd(fd) => nix::unistd::fchown(fd.as_raw(), uid, gid), + OsPathOrFd::Fd(fd) => nix::unistd::fchown(fd, uid, gid), } .map_err(|err| { // Use `From<nix::Error> for io::Error` when it is available @@ -945,7 +944,7 @@ pub mod module { } #[pyfunction] - fn get_inheritable(fd: RawFd, vm: &VirtualMachine) -> PyResult<bool> { + fn get_inheritable(fd: BorrowedFd<'_>, vm: &VirtualMachine) -> PyResult<bool> { let flags = fcntl::fcntl(fd, fcntl::FcntlArg::F_GETFD); match flags { Ok(ret) => Ok((ret & libc::FD_CLOEXEC) == 0), @@ -954,12 +953,12 @@ pub mod module { } #[pyfunction] - fn set_inheritable(fd: i32, inheritable: bool, vm: &VirtualMachine) -> PyResult<()> { - super::raw_set_inheritable(fd, inheritable).map_err(|err| err.into_pyexception(vm)) + fn set_inheritable(fd: BorrowedFd<'_>, inheritable: bool, vm: &VirtualMachine) -> PyResult<()> { + super::set_inheritable(fd, inheritable).map_err(|err| err.into_pyexception(vm)) } #[pyfunction] - fn get_blocking(fd: RawFd, vm: &VirtualMachine) -> PyResult<bool> { + fn get_blocking(fd: BorrowedFd<'_>, vm: &VirtualMachine) -> PyResult<bool> { let flags = fcntl::fcntl(fd, fcntl::FcntlArg::F_GETFL); match flags { Ok(ret) => Ok((ret & libc::O_NONBLOCK) == 0), @@ -968,7 +967,7 @@ pub mod module { } #[pyfunction] - fn set_blocking(fd: RawFd, blocking: bool, vm: &VirtualMachine) -> PyResult<()> { + fn set_blocking(fd: BorrowedFd<'_>, blocking: bool, vm: &VirtualMachine) -> PyResult<()> { let _set_flag = || { use nix::fcntl::{FcntlArg, OFlag, fcntl}; @@ -987,8 +986,8 @@ pub mod module { fn pipe(vm: &VirtualMachine) -> PyResult<(OwnedFd, OwnedFd)> { use nix::unistd::pipe; let (rfd, wfd) = pipe().map_err(|err| err.into_pyexception(vm))?; - set_inheritable(rfd.as_raw_fd(), false, vm)?; - set_inheritable(wfd.as_raw_fd(), false, vm)?; + set_inheritable(rfd.as_fd(), false, vm)?; + set_inheritable(wfd.as_fd(), false, vm)?; Ok((rfd, wfd)) } @@ -1030,7 +1029,7 @@ pub mod module { #[cfg(not(target_os = "redox"))] fn _fchmod(fd: BorrowedFd<'_>, mode: u32, vm: &VirtualMachine) -> PyResult<()> { nix::sys::stat::fchmod( - fd.as_raw_fd(), + fd, nix::sys::stat::Mode::from_bits(mode as libc::mode_t).unwrap(), ) .map_err(|err| err.into_pyexception(vm)) @@ -1313,8 +1312,7 @@ pub mod module { fn openpty(vm: &VirtualMachine) -> PyResult<(OwnedFd, OwnedFd)> { let r = nix::pty::openpty(None, None).map_err(|err| err.into_pyexception(vm))?; for fd in [&r.master, &r.slave] { - super::raw_set_inheritable(fd.as_raw_fd(), false) - .map_err(|e| e.into_pyexception(vm))?; + super::set_inheritable(fd.as_fd(), false).map_err(|e| e.into_pyexception(vm))?; } Ok((r.master, r.slave)) } @@ -1513,11 +1511,8 @@ pub mod module { .into_cstring(vm) .map_err(|_| vm.new_value_error("path should not have nul bytes"))?; - let mut file_actions = unsafe { - let mut fa = std::mem::MaybeUninit::uninit(); - assert!(libc::posix_spawn_file_actions_init(fa.as_mut_ptr()) == 0); - fa.assume_init() - }; + let mut file_actions = + nix::spawn::PosixSpawnFileActions::init().map_err(|e| e.into_pyexception(vm))?; if let Some(it) = self.file_actions { for action in it.iter(vm)? { let action = action?; @@ -1536,41 +1531,30 @@ pub mod module { "POSIX_SPAWN_OPEN path should not have nul bytes", ) })?; - unsafe { - libc::posix_spawn_file_actions_addopen( - &mut file_actions, - fd, - path.as_ptr(), - oflag, - mode, - ) - } + let oflag = nix::fcntl::OFlag::from_bits_retain(oflag); + let mode = nix::sys::stat::Mode::from_bits_retain(mode); + file_actions.add_open(fd, &*path, oflag, mode) } PosixSpawnFileActionIdentifier::Close => { let (fd,) = args.bind(vm)?; - unsafe { - libc::posix_spawn_file_actions_addclose(&mut file_actions, fd) - } + file_actions.add_close(fd) } PosixSpawnFileActionIdentifier::Dup2 => { let (fd, newfd) = args.bind(vm)?; - unsafe { - libc::posix_spawn_file_actions_adddup2(&mut file_actions, fd, newfd) - } + file_actions.add_dup2(fd, newfd) } }; - if ret != 0 { - let err = std::io::Error::from_raw_os_error(ret); + if let Err(err) = ret { + let err = err.into(); return Err(IOErrorBuilder::with_filename(&err, self.path, vm)); } } } - let mut attrp = unsafe { - let mut sa = std::mem::MaybeUninit::uninit(); - assert!(libc::posix_spawnattr_init(sa.as_mut_ptr()) == 0); - sa.assume_init() - }; + let mut attrp = + nix::spawn::PosixSpawnAttr::init().map_err(|e| e.into_pyexception(vm))?; + let mut flags = nix::spawn::PosixSpawnFlags::empty(); + if let Some(sigs) = self.setsigdef { use nix::sys::signal; let mut set = signal::SigSet::empty(); @@ -1581,37 +1565,39 @@ pub mod module { })?; set.add(sig); } - assert!( - unsafe { libc::posix_spawnattr_setsigdefault(&mut attrp, set.as_ref()) } == 0 - ); + attrp + .set_sigdefault(&set) + .map_err(|e| e.into_pyexception(vm))?; + flags.insert(nix::spawn::PosixSpawnFlags::POSIX_SPAWN_SETSIGDEF); } - // Handle new posix_spawn attributes - let mut flags = 0i32; - if let Some(pgid) = self.setpgroup { - let ret = unsafe { libc::posix_spawnattr_setpgroup(&mut attrp, pgid) }; - if ret != 0 { - return Err(vm.new_os_error(format!("posix_spawnattr_setpgroup failed: {ret}"))); - } - flags |= libc::POSIX_SPAWN_SETPGROUP; + attrp + .set_pgroup(nix::unistd::Pid::from_raw(pgid)) + .map_err(|e| e.into_pyexception(vm))?; + flags.insert(nix::spawn::PosixSpawnFlags::POSIX_SPAWN_SETPGROUP); } if self.resetids { - flags |= libc::POSIX_SPAWN_RESETIDS; + flags.insert(nix::spawn::PosixSpawnFlags::POSIX_SPAWN_RESETIDS); } if self.setsid { // Note: POSIX_SPAWN_SETSID may not be available on all platforms - #[cfg(target_os = "linux")] - { - flags |= 0x0080; // POSIX_SPAWN_SETSID value on Linux - } - #[cfg(not(target_os = "linux"))] - { - return Err(vm.new_not_implemented_error( - "setsid parameter is not supported on this platform", - )); + cfg_if::cfg_if! { + if #[cfg(any( + target_os = "linux", + target_os = "haiku", + target_os = "solaris", + target_os = "illumos", + target_os = "hurd", + ))] { + flags.insert(nix::spawn::PosixSpawnFlags::from_bits_retain(libc::POSIX_SPAWN_SETSID)); + } else { + return Err(vm.new_not_implemented_error( + "setsid parameter is not supported on this platform", + )); + } } } @@ -1625,13 +1611,10 @@ pub mod module { })?; set.add(sig); } - let ret = unsafe { libc::posix_spawnattr_setsigmask(&mut attrp, set.as_ref()) }; - if ret != 0 { - return Err( - vm.new_os_error(format!("posix_spawnattr_setsigmask failed: {ret}")) - ); - } - flags |= libc::POSIX_SPAWN_SETSIGMASK; + attrp + .set_sigmask(&set) + .map_err(|e| e.into_pyexception(vm))?; + flags.insert(nix::spawn::PosixSpawnFlags::POSIX_SPAWN_SETSIGMASK); } if let Some(_scheduler) = self.scheduler { @@ -1642,19 +1625,11 @@ pub mod module { ); } - if flags != 0 { - // Check for potential overflow when casting to c_short - if flags > libc::c_short::MAX as i32 { - return Err(vm.new_value_error("Too many flags set for posix_spawn")); - } - let ret = - unsafe { libc::posix_spawnattr_setflags(&mut attrp, flags as libc::c_short) }; - if ret != 0 { - return Err(vm.new_os_error(format!("posix_spawnattr_setflags failed: {ret}"))); - } + if !flags.is_empty() { + attrp.set_flags(flags).map_err(|e| e.into_pyexception(vm))?; } - let mut args: Vec<CString> = self + let args: Vec<CString> = self .args .iter(vm)? .map(|res| { @@ -1662,47 +1637,15 @@ pub mod module { .map_err(|_| vm.new_value_error("path should not have nul bytes")) }) .collect::<Result<_, _>>()?; - let argv: Vec<*mut libc::c_char> = args - .iter_mut() - .map(|s| s.as_ptr() as _) - .chain(std::iter::once(std::ptr::null_mut())) - .collect(); - let mut env = envp_from_dict(self.env, vm)?; - let envp: Vec<*mut libc::c_char> = env - .iter_mut() - .map(|s| s.as_ptr() as _) - .chain(std::iter::once(std::ptr::null_mut())) - .collect(); - - let mut pid = 0; - let ret = unsafe { - if spawnp { - libc::posix_spawnp( - &mut pid, - path.as_ptr(), - &file_actions, - &attrp, - argv.as_ptr(), - envp.as_ptr(), - ) - } else { - libc::posix_spawn( - &mut pid, - path.as_ptr(), - &file_actions, - &attrp, - argv.as_ptr(), - envp.as_ptr(), - ) - } - }; + let env = envp_from_dict(self.env, vm)?; - if ret == 0 { - Ok(pid) + let ret = if spawnp { + nix::spawn::posix_spawnp(&path, &file_actions, &attrp, &args, &env) } else { - let err = std::io::Error::from_raw_os_error(ret); - Err(IOErrorBuilder::with_filename(&err, self.path, vm)) - } + nix::spawn::posix_spawn(&*path, &file_actions, &attrp, &args, &env) + }; + ret.map(Into::into) + .map_err(|err| IOErrorBuilder::with_filename(&err.into(), self.path, vm)) } } @@ -1823,36 +1766,32 @@ pub mod module { } #[pyfunction] - fn dup(fd: i32, vm: &VirtualMachine) -> PyResult<i32> { + fn dup(fd: BorrowedFd<'_>, vm: &VirtualMachine) -> PyResult<OwnedFd> { let fd = nix::unistd::dup(fd).map_err(|e| e.into_pyexception(vm))?; - super::raw_set_inheritable(fd, false) + super::set_inheritable(fd.as_fd(), false) .map(|()| fd) - .map_err(|e| { - let _ = nix::unistd::close(fd); - e.into_pyexception(vm) - }) + .map_err(|e| e.into_pyexception(vm)) } #[derive(FromArgs)] - struct Dup2Args { + struct Dup2Args<'fd> { #[pyarg(positional)] - fd: i32, + fd: BorrowedFd<'fd>, #[pyarg(positional)] - fd2: i32, + fd2: OwnedFd, #[pyarg(any, default = true)] inheritable: bool, } #[pyfunction] - fn dup2(args: Dup2Args, vm: &VirtualMachine) -> PyResult<i32> { - let fd = nix::unistd::dup2(args.fd, args.fd2).map_err(|e| e.into_pyexception(vm))?; + fn dup2(args: Dup2Args<'_>, vm: &VirtualMachine) -> PyResult<OwnedFd> { + let mut fd2 = std::mem::ManuallyDrop::new(args.fd2); + nix::unistd::dup2(args.fd, &mut fd2).map_err(|e| e.into_pyexception(vm))?; + let fd2 = std::mem::ManuallyDrop::into_inner(fd2); if !args.inheritable { - super::raw_set_inheritable(fd, false).map_err(|e| { - let _ = nix::unistd::close(fd); - e.into_pyexception(vm) - })? + super::set_inheritable(fd2.as_fd(), false).map_err(|e| e.into_pyexception(vm))? } - Ok(fd) + Ok(fd2) } pub(crate) fn support_funcs() -> Vec<SupportFunc> { diff --git a/vm/src/stdlib/signal.rs b/vm/src/stdlib/signal.rs index ff59208ade..bab45b3415 100644 --- a/vm/src/stdlib/signal.rs +++ b/vm/src/stdlib/signal.rs @@ -270,13 +270,16 @@ pub(crate) mod _signal { false }; #[cfg(unix)] - if fd != INVALID_WAKEUP { + if let Ok(fd) = unsafe { crate::common::crt_fd::Borrowed::try_borrow_raw(fd) } { use nix::fcntl; let oflags = fcntl::fcntl(fd, fcntl::F_GETFL).map_err(|e| e.into_pyexception(vm))?; let nonblock = fcntl::OFlag::from_bits_truncate(oflags).contains(fcntl::OFlag::O_NONBLOCK); if !nonblock { - return Err(vm.new_value_error(format!("the fd {fd} must be in non-blocking mode"))); + return Err(vm.new_value_error(format!( + "the fd {} must be in non-blocking mode", + fd.as_raw() + ))); } } diff --git a/vm/src/stdlib/sys.rs b/vm/src/stdlib/sys.rs index 5f30876b30..d650f72443 100644 --- a/vm/src/stdlib/sys.rs +++ b/vm/src/stdlib/sys.rs @@ -183,16 +183,16 @@ mod sys { let ctx = &vm.ctx; #[cfg(not(target_arch = "wasm32"))] { - if let Some(exec_path) = env::args_os().next() { - if let Ok(path) = which::which(exec_path) { - return ctx - .new_str( - path.into_os_string() - .into_string() - .unwrap_or_else(|p| p.to_string_lossy().into_owned()), - ) - .into(); - } + if let Some(exec_path) = env::args_os().next() + && let Ok(path) = which::which(exec_path) + { + return ctx + .new_str( + path.into_os_string() + .into_string() + .unwrap_or_else(|p| p.to_string_lossy().into_owned()), + ) + .into(); } } if let Some(exec_path) = env::args().next() { @@ -203,16 +203,16 @@ mod sys { if path.is_absolute() { return ctx.new_str(exec_path).into(); } - if let Ok(dir) = env::current_dir() { - if let Ok(dir) = dir.into_os_string().into_string() { - return ctx - .new_str(format!( - "{}/{}", - dir, - exec_path.strip_prefix("./").unwrap_or(&exec_path) - )) - .into(); - } + if let Ok(dir) = env::current_dir() + && let Ok(dir) = dir.into_os_string().into_string() + { + return ctx + .new_str(format!( + "{}/{}", + dir, + exec_path.strip_prefix("./").unwrap_or(&exec_path) + )) + .into(); } } ctx.none() @@ -868,22 +868,22 @@ mod sys { #[pyfunction] fn set_asyncgen_hooks(args: SetAsyncgenHooksArgs, vm: &VirtualMachine) -> PyResult<()> { - if let Some(Some(finalizer)) = args.finalizer.as_option() { - if !finalizer.is_callable() { - return Err(vm.new_type_error(format!( - "callable finalizer expected, got {:.50}", - finalizer.class().name() - ))); - } + if let Some(Some(finalizer)) = args.finalizer.as_option() + && !finalizer.is_callable() + { + return Err(vm.new_type_error(format!( + "callable finalizer expected, got {:.50}", + finalizer.class().name() + ))); } - if let Some(Some(firstiter)) = args.firstiter.as_option() { - if !firstiter.is_callable() { - return Err(vm.new_type_error(format!( - "callable firstiter expected, got {:.50}", - firstiter.class().name() - ))); - } + if let Some(Some(firstiter)) = args.firstiter.as_option() + && !firstiter.is_callable() + { + return Err(vm.new_type_error(format!( + "callable firstiter expected, got {:.50}", + firstiter.class().name() + ))); } if let Some(finalizer) = args.finalizer.into_option() { diff --git a/vm/src/stdlib/sysconfig.rs b/vm/src/stdlib/sysconfig.rs new file mode 100644 index 0000000000..2e0a8a51c7 --- /dev/null +++ b/vm/src/stdlib/sysconfig.rs @@ -0,0 +1,14 @@ +pub(crate) use sysconfig::make_module; + +#[pymodule(name = "_sysconfig")] +pub(crate) mod sysconfig { + use crate::{VirtualMachine, builtins::PyDictRef, convert::ToPyObject}; + + #[pyfunction] + fn config_vars(vm: &VirtualMachine) -> PyDictRef { + let vars = vm.ctx.new_dict(); + vars.set_item("Py_GIL_DISABLED", true.to_pyobject(vm), vm) + .unwrap(); + vars + } +} diff --git a/vm/src/stdlib/time.rs b/vm/src/stdlib/time.rs index 6a061b3454..4ad29d0fe1 100644 --- a/vm/src/stdlib/time.rs +++ b/vm/src/stdlib/time.rs @@ -91,12 +91,11 @@ mod decl { #[pyfunction] fn sleep(seconds: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { let dur = seconds.try_into_value::<Duration>(vm).map_err(|e| { - if e.class().is(vm.ctx.exceptions.value_error) { - if let Some(s) = e.args().first().and_then(|arg| arg.str(vm).ok()) { - if s.as_str() == "negative duration" { - return vm.new_value_error("sleep length must be non-negative"); - } - } + if e.class().is(vm.ctx.exceptions.value_error) + && let Some(s) = e.args().first().and_then(|arg| arg.str(vm).ok()) + && s.as_str() == "negative duration" + { + return vm.new_value_error("sleep length must be non-negative"); } e })?; diff --git a/vm/src/stdlib/typing.rs b/vm/src/stdlib/typing.rs index 2c4517fc7b..c014266935 100644 --- a/vm/src/stdlib/typing.rs +++ b/vm/src/stdlib/typing.rs @@ -31,7 +31,7 @@ pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { pub(crate) mod decl { use crate::{ Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, - builtins::{PyTupleRef, PyTypeRef, pystr::AsPyStr}, + builtins::{PyStrRef, PyTupleRef, PyTypeRef, pystr::AsPyStr}, function::{FuncArgs, IntoFuncArgs}, types::{Constructor, Representable}, }; @@ -98,7 +98,7 @@ pub(crate) mod decl { #[derive(Debug, PyPayload)] #[allow(dead_code)] pub(crate) struct TypeAliasType { - name: PyObjectRef, // TODO PyStrRef? + name: PyStrRef, type_params: PyTupleRef, value: PyObjectRef, // compute_value: PyObjectRef, @@ -106,7 +106,7 @@ pub(crate) mod decl { } #[pyclass(with(Constructor, Representable), flags(BASETYPE))] impl TypeAliasType { - pub const fn new(name: PyObjectRef, type_params: PyTupleRef, value: PyObjectRef) -> Self { + pub const fn new(name: PyStrRef, type_params: PyTupleRef, value: PyObjectRef) -> Self { Self { name, type_params, @@ -116,7 +116,7 @@ pub(crate) mod decl { #[pygetset] fn __name__(&self) -> PyObjectRef { - self.name.clone() + self.name.clone().into() } #[pygetset] @@ -154,7 +154,10 @@ pub(crate) mod decl { ))); } - let name = args.args[0].clone(); + let name = args.args[0] + .clone() + .downcast::<crate::builtins::PyStr>() + .map_err(|_| vm.new_type_error("TypeAliasType name must be a string".to_owned()))?; let value = args.args[1].clone(); let type_params = if let Some(tp) = args.kwargs.get("type_params") { @@ -171,9 +174,8 @@ pub(crate) mod decl { } impl Representable for TypeAliasType { - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - let name = zelf.name.str(vm)?; - Ok(name.as_str().to_owned()) + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok(zelf.name.as_str().to_owned()) } } diff --git a/vm/src/stdlib/warnings.rs b/vm/src/stdlib/warnings.rs index a8ffee4579..2d61c3b571 100644 --- a/vm/src/stdlib/warnings.rs +++ b/vm/src/stdlib/warnings.rs @@ -9,10 +9,10 @@ pub fn warn( vm: &VirtualMachine, ) -> PyResult<()> { // TODO: use rust warnings module - if let Ok(module) = vm.import("warnings", 0) { - if let Ok(func) = module.get_attr("warn", vm) { - let _ = func.call((message, category.to_owned(), stack_level), vm); - } + if let Ok(module) = vm.import("warnings", 0) + && let Ok(func) = module.get_attr("warn", vm) + { + let _ = func.call((message, category.to_owned(), stack_level), vm); } Ok(()) } diff --git a/vm/src/vm/context.rs b/vm/src/vm/context.rs index 03ac4a1777..4411aa0ff6 100644 --- a/vm/src/vm/context.rs +++ b/vm/src/vm/context.rs @@ -387,22 +387,22 @@ impl Context { #[inline] pub fn new_int<T: Into<BigInt> + ToPrimitive>(&self, i: T) -> PyIntRef { - if let Some(i) = i.to_i32() { - if Self::INT_CACHE_POOL_RANGE.contains(&i) { - let inner_idx = (i - Self::INT_CACHE_POOL_MIN) as usize; - return self.int_cache_pool[inner_idx].clone(); - } + if let Some(i) = i.to_i32() + && Self::INT_CACHE_POOL_RANGE.contains(&i) + { + let inner_idx = (i - Self::INT_CACHE_POOL_MIN) as usize; + return self.int_cache_pool[inner_idx].clone(); } PyInt::from(i).into_ref(self) } #[inline] pub fn new_bigint(&self, i: &BigInt) -> PyIntRef { - if let Some(i) = i.to_i32() { - if Self::INT_CACHE_POOL_RANGE.contains(&i) { - let inner_idx = (i - Self::INT_CACHE_POOL_MIN) as usize; - return self.int_cache_pool[inner_idx].clone(); - } + if let Some(i) = i.to_i32() + && Self::INT_CACHE_POOL_RANGE.contains(&i) + { + let inner_idx = (i - Self::INT_CACHE_POOL_MIN) as usize; + return self.int_cache_pool[inner_idx].clone(); } PyInt::from(i.clone()).into_ref(self) } diff --git a/vm/src/vm/method.rs b/vm/src/vm/method.rs index e10eb2db83..5df01c556e 100644 --- a/vm/src/vm/method.rs +++ b/vm/src/vm/method.rs @@ -42,14 +42,13 @@ impl PyMethod { None } else { let descr_get = descr_cls.mro_find_map(|cls| cls.slots.descr_get.load()); - if let Some(descr_get) = descr_get { - if descr_cls + if let Some(descr_get) = descr_get + && descr_cls .mro_find_map(|cls| cls.slots.descr_set.load()) .is_some() - { - let cls = cls.to_owned().into(); - return descr_get(descr, Some(obj), Some(cls), vm).map(Self::Attribute); - } + { + let cls = cls.to_owned().into(); + return descr_get(descr, Some(obj), Some(cls), vm).map(Self::Attribute); } descr_get }; @@ -58,10 +57,10 @@ impl PyMethod { None => None, }; - if let Some(dict) = obj.dict() { - if let Some(attr) = dict.get_item_opt(name, vm)? { - return Ok(Self::Attribute(attr)); - } + if let Some(dict) = obj.dict() + && let Some(attr) = dict.get_item_opt(name, vm)? + { + return Ok(Self::Attribute(attr)); } if let Some((attr, descr_get)) = cls_attr { diff --git a/vm/src/vm/mod.rs b/vm/src/vm/mod.rs index e88153ecee..a973140a99 100644 --- a/vm/src/vm/mod.rs +++ b/vm/src/vm/mod.rs @@ -363,13 +363,14 @@ impl VirtualMachine { let res = essential_init(); let importlib = self.expect_pyresult(res, "essential initialization failed"); - if self.state.settings.allow_external_library && cfg!(feature = "rustpython-compiler") { - if let Err(e) = import::init_importlib_package(self, importlib) { - eprintln!( - "importlib initialization failed. This is critical for many complicated packages." - ); - self.print_exception(e); - } + if self.state.settings.allow_external_library + && cfg!(feature = "rustpython-compiler") + && let Err(e) = import::init_importlib_package(self, importlib) + { + eprintln!( + "importlib initialization failed. This is critical for many complicated packages." + ); + self.print_exception(e); } let expect_stdlib = @@ -714,10 +715,10 @@ impl VirtualMachine { }; // TODO: fix extend to do this check (?), see test_extend in Lib/test/list_tests.py, // https://github.com/python/cpython/blob/v3.9.0/Objects/listobject.c#L922-L928 - if let Some(cap) = cap { - if cap >= isize::MAX as usize { - return Ok(Vec::new()); - } + if let Some(cap) = cap + && cap >= isize::MAX as usize + { + return Ok(Vec::new()); } let mut results = PyIterIter::new(self, iter.as_ref(), cap) @@ -829,18 +830,18 @@ impl VirtualMachine { } pub(crate) fn contextualize_exception(&self, exception: &PyBaseExceptionRef) { - if let Some(context_exc) = self.topmost_exception() { - if !context_exc.is(exception) { - let mut o = context_exc.clone(); - while let Some(context) = o.__context__() { - if context.is(exception) { - o.set___context__(None); - break; - } - o = context; + if let Some(context_exc) = self.topmost_exception() + && !context_exc.is(exception) + { + let mut o = context_exc.clone(); + while let Some(context) = o.__context__() { + if context.is(exception) { + o.set___context__(None); + break; } - exception.set___context__(Some(context_exc)) + o = context; } + exception.set___context__(Some(context_exc)) } } diff --git a/vm/src/vm/vm_new.rs b/vm/src/vm/vm_new.rs index 671f5df34d..d0b78cfe5b 100644 --- a/vm/src/vm/vm_new.rs +++ b/vm/src/vm/vm_new.rs @@ -321,7 +321,7 @@ impl VirtualMachine { error: ruff_python_parser::ParseErrorType::Lexical( ruff_python_parser::LexicalErrorType::FStringError( - ruff_python_parser::FStringErrorType::UnterminatedTripleQuotedString, + ruff_python_parser::InterpolatedStringErrorType::UnterminatedTripleQuotedString, ), ), .. @@ -347,10 +347,11 @@ impl VirtualMachine { if let Some(source) = source { let loc = raw_location.start().to_usize(); let mut iter = source.chars(); - if let Some(quote) = iter.nth(loc) { - if iter.next() == Some(quote) && iter.next() == Some(quote) { - is_incomplete = true; - } + if let Some(quote) = iter.nth(loc) + && iter.next() == Some(quote) + && iter.next() == Some(quote) + { + is_incomplete = true; } } @@ -411,7 +412,7 @@ impl VirtualMachine { fn get_statement(source: &str, loc: Option<SourceLocation>) -> Option<String> { let line = source .split('\n') - .nth(loc?.row.to_zero_indexed())? + .nth(loc?.line.to_zero_indexed())? .to_owned(); Some(line + "\n") } diff --git a/vm/src/vm/vm_ops.rs b/vm/src/vm/vm_ops.rs index d4bf4563c8..840b79c258 100644 --- a/vm/src/vm/vm_ops.rs +++ b/vm/src/vm/vm_ops.rs @@ -171,14 +171,14 @@ impl VirtualMachine { } if let Some(slot_a) = slot_a { - if let Some(slot_bb) = slot_b { - if class_b.fast_issubclass(class_a) { - let ret = slot_bb(a, b, self)?; - if !ret.is(&self.ctx.not_implemented) { - return Ok(ret); - } - slot_b = None; + if let Some(slot_bb) = slot_b + && class_b.fast_issubclass(class_a) + { + let ret = slot_bb(a, b, self)?; + if !ret.is(&self.ctx.not_implemented) { + return Ok(ret); } + slot_b = None; } let ret = slot_a(a, b, self)?; if !ret.is(&self.ctx.not_implemented) { @@ -277,14 +277,14 @@ impl VirtualMachine { } if let Some(slot_a) = slot_a { - if let Some(slot_bb) = slot_b { - if class_b.fast_issubclass(class_a) { - let ret = slot_bb(a, b, c, self)?; - if !ret.is(&self.ctx.not_implemented) { - return Ok(ret); - } - slot_b = None; + if let Some(slot_bb) = slot_b + && class_b.fast_issubclass(class_a) + { + let ret = slot_bb(a, b, c, self)?; + if !ret.is(&self.ctx.not_implemented) { + return Ok(ret); } + slot_b = None; } let ret = slot_a(a, b, c, self)?; if !ret.is(&self.ctx.not_implemented) { @@ -299,14 +299,13 @@ impl VirtualMachine { } } - if let Some(slot_c) = class_c.slots.as_number.left_ternary_op(op_slot) { - if slot_a.is_some_and(|slot_a| !std::ptr::fn_addr_eq(slot_a, slot_c)) - && slot_b.is_some_and(|slot_b| !std::ptr::fn_addr_eq(slot_b, slot_c)) - { - let ret = slot_c(a, b, c, self)?; - if !ret.is(&self.ctx.not_implemented) { - return Ok(ret); - } + if let Some(slot_c) = class_c.slots.as_number.left_ternary_op(op_slot) + && slot_a.is_some_and(|slot_a| !std::ptr::fn_addr_eq(slot_a, slot_c)) + && slot_b.is_some_and(|slot_b| !std::ptr::fn_addr_eq(slot_b, slot_c)) + { + let ret = slot_c(a, b, c, self)?; + if !ret.is(&self.ctx.not_implemented) { + return Ok(ret); } } diff --git a/vm/src/windows.rs b/vm/src/windows.rs index a14216768e..907f61c7b8 100644 --- a/vm/src/windows.rs +++ b/vm/src/windows.rs @@ -110,16 +110,16 @@ fn win32_xstat_impl(path: &OsStr, traverse: bool) -> std::io::Result<StatStruct> } } Err(e) => { - if let Some(errno) = e.raw_os_error() { - if matches!( + if let Some(errno) = e.raw_os_error() + && matches!( errno as u32, Foundation::ERROR_FILE_NOT_FOUND | Foundation::ERROR_PATH_NOT_FOUND | Foundation::ERROR_NOT_READY | Foundation::ERROR_BAD_NET_NAME - ) { - return Err(e); - } + ) + { + return Err(e); } } } diff --git a/wasm/demo/package-lock.json b/wasm/demo/package-lock.json index ce1a1219c0..e3cf79298a 100644 --- a/wasm/demo/package-lock.json +++ b/wasm/demo/package-lock.json @@ -21,7 +21,7 @@ "css-loader": "^7.1.2", "html-webpack-plugin": "^5.6.3", "mini-css-extract-plugin": "^2.9.2", - "serve": "^14.2.4", + "serve": "^14.2.5", "webpack": "^5.97.1", "webpack-cli": "^6.0.1", "webpack-dev-server": "^5.2.1" @@ -1542,24 +1542,65 @@ } }, "node_modules/compression": { - "version": "1.7.4", - "resolved": "https://registry.npmjs.org/compression/-/compression-1.7.4.tgz", - "integrity": "sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ==", + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/compression/-/compression-1.8.1.tgz", + "integrity": "sha512-9mAqGPHLakhCLeNyxPkK4xVo746zQ/czLH1Ky+vkitMnWfWZps8r0qXuwhwizagCRttsL4lfG4pIOvaWLpAP0w==", "dev": true, "license": "MIT", "dependencies": { - "accepts": "~1.3.5", - "bytes": "3.0.0", - "compressible": "~2.0.16", + "bytes": "3.1.2", + "compressible": "~2.0.18", "debug": "2.6.9", - "on-headers": "~1.0.2", - "safe-buffer": "5.1.2", + "negotiator": "~0.6.4", + "on-headers": "~1.1.0", + "safe-buffer": "5.2.1", "vary": "~1.1.2" }, "engines": { "node": ">= 0.8.0" } }, + "node_modules/compression/node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/compression/node_modules/negotiator": { + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.4.tgz", + "integrity": "sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/compression/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -3540,9 +3581,9 @@ } }, "node_modules/on-headers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz", - "integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.1.0.tgz", + "integrity": "sha512-737ZY3yNnXy37FHkQxPzt4UZ2UWPWiCZWLvFZ4fu5cueciegX0zGPnrlY6bwRg4FdQOe9YU8MkmJwGhoMybl8A==", "dev": true, "license": "MIT", "engines": { @@ -4310,9 +4351,9 @@ } }, "node_modules/serve": { - "version": "14.2.4", - "resolved": "https://registry.npmjs.org/serve/-/serve-14.2.4.tgz", - "integrity": "sha512-qy1S34PJ/fcY8gjVGszDB3EXiPSk5FKhUa7tQe0UPRddxRidc2V6cNHPNewbE1D7MAkgLuWEt3Vw56vYy73tzQ==", + "version": "14.2.5", + "resolved": "https://registry.npmjs.org/serve/-/serve-14.2.5.tgz", + "integrity": "sha512-Qn/qMkzCcMFVPb60E/hQy+iRLpiU8PamOfOSYoAHmmF+fFFmpPpqa6Oci2iWYpTdOUM3VF+TINud7CfbQnsZbA==", "dev": true, "license": "MIT", "dependencies": { @@ -4323,7 +4364,7 @@ "chalk": "5.0.1", "chalk-template": "0.4.0", "clipboardy": "3.0.0", - "compression": "1.7.4", + "compression": "1.8.1", "is-port-reachable": "4.0.0", "serve-handler": "6.1.6", "update-check": "1.5.4" diff --git a/wasm/demo/package.json b/wasm/demo/package.json index 22c82e1557..2c08e5c416 100644 --- a/wasm/demo/package.json +++ b/wasm/demo/package.json @@ -16,7 +16,7 @@ "css-loader": "^7.1.2", "html-webpack-plugin": "^5.6.3", "mini-css-extract-plugin": "^2.9.2", - "serve": "^14.2.4", + "serve": "^14.2.5", "webpack": "^5.97.1", "webpack-cli": "^6.0.1", "webpack-dev-server": "^5.2.1" diff --git a/wasm/lib/.cargo/config.toml b/wasm/lib/.cargo/config.toml index ce1e7c694a..f4e8c002fc 100644 --- a/wasm/lib/.cargo/config.toml +++ b/wasm/lib/.cargo/config.toml @@ -1,5 +1,2 @@ [build] target = "wasm32-unknown-unknown" - -[target.wasm32-unknown-unknown] -rustflags = ["--cfg=getrandom_backend=\"wasm_js\""] diff --git a/wasm/lib/Cargo.toml b/wasm/lib/Cargo.toml index 768a1e671f..a2bb1a9f94 100644 --- a/wasm/lib/Cargo.toml +++ b/wasm/lib/Cargo.toml @@ -30,7 +30,7 @@ wasm-bindgen = { workspace = true } console_error_panic_hook = "0.1" js-sys = "0.3" -serde-wasm-bindgen = "0.3.1" +serde-wasm-bindgen = "0.6.5" wasm-bindgen-futures = "0.4" web-sys = { version = "0.3", features = [ "console", diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs index 94476ff226..d1821f2e73 100644 --- a/wasm/lib/src/convert.rs +++ b/wasm/lib/src/convert.rs @@ -80,63 +80,63 @@ pub fn js_err_to_py_err(vm: &VirtualMachine, js_err: &JsValue) -> PyBaseExceptio } pub fn py_to_js(vm: &VirtualMachine, py_obj: PyObjectRef) -> JsValue { - if let Some(ref wasm_id) = vm.wasm_id { - if py_obj.fast_isinstance(vm.ctx.types.function_type) { - let wasm_vm = WASMVirtualMachine { - id: wasm_id.clone(), - }; - let weak_py_obj = wasm_vm.push_held_rc(py_obj).unwrap().unwrap(); + if let Some(ref wasm_id) = vm.wasm_id + && py_obj.fast_isinstance(vm.ctx.types.function_type) + { + let wasm_vm = WASMVirtualMachine { + id: wasm_id.clone(), + }; + let weak_py_obj = wasm_vm.push_held_rc(py_obj).unwrap().unwrap(); - let closure = move |args: Option<Box<[JsValue]>>, - kwargs: Option<Object>| - -> Result<JsValue, JsValue> { - let py_obj = match wasm_vm.assert_valid() { - Ok(_) => weak_py_obj - .upgrade() - .expect("weak_py_obj to be valid if VM is valid"), - Err(err) => { - return Err(err); - } + let closure = move |args: Option<Box<[JsValue]>>, + kwargs: Option<Object>| + -> Result<JsValue, JsValue> { + let py_obj = match wasm_vm.assert_valid() { + Ok(_) => weak_py_obj + .upgrade() + .expect("weak_py_obj to be valid if VM is valid"), + Err(err) => { + return Err(err); + } + }; + stored_vm_from_wasm(&wasm_vm).interp.enter(move |vm| { + let args = match args { + Some(args) => Vec::from(args) + .into_iter() + .map(|arg| js_to_py(vm, arg)) + .collect::<Vec<_>>(), + None => Vec::new(), }; - stored_vm_from_wasm(&wasm_vm).interp.enter(move |vm| { - let args = match args { - Some(args) => Vec::from(args) - .into_iter() - .map(|arg| js_to_py(vm, arg)) - .collect::<Vec<_>>(), - None => Vec::new(), - }; - let mut py_func_args = FuncArgs::from(args); - if let Some(ref kwargs) = kwargs { - for pair in object_entries(kwargs) { - let (key, val) = pair?; - py_func_args - .kwargs - .insert(js_sys::JsString::from(key).into(), js_to_py(vm, val)); - } + let mut py_func_args = FuncArgs::from(args); + if let Some(ref kwargs) = kwargs { + for pair in object_entries(kwargs) { + let (key, val) = pair?; + py_func_args + .kwargs + .insert(js_sys::JsString::from(key).into(), js_to_py(vm, val)); } - let result = py_obj.call(py_func_args, vm); - pyresult_to_js_result(vm, result) - }) - }; - let closure = Closure::wrap(Box::new(closure) - as Box< - dyn FnMut(Option<Box<[JsValue]>>, Option<Object>) -> Result<JsValue, JsValue>, - >); - let func = closure.as_ref().clone(); + } + let result = py_obj.call(py_func_args, vm); + pyresult_to_js_result(vm, result) + }) + }; + let closure = Closure::wrap(Box::new(closure) + as Box< + dyn FnMut(Option<Box<[JsValue]>>, Option<Object>) -> Result<JsValue, JsValue>, + >); + let func = closure.as_ref().clone(); - // stores pretty much nothing, it's fine to leak this because if it gets dropped - // the error message is worse - closure.forget(); + // stores pretty much nothing, it's fine to leak this because if it gets dropped + // the error message is worse + closure.forget(); - return func; - } + return func; } // the browser module might not be injected - if vm.try_class("_js", "Promise").is_ok() { - if let Some(py_prom) = py_obj.downcast_ref::<js_module::PyPromise>() { - return py_prom.as_js(vm).into(); - } + if vm.try_class("_js", "Promise").is_ok() + && let Some(py_prom) = py_obj.downcast_ref::<js_module::PyPromise>() + { + return py_prom.as_js(vm).into(); } if let Ok(bytes) = ArgBytesLike::try_from_borrowed_object(vm, &py_obj) { @@ -251,12 +251,12 @@ pub fn syntax_err(err: CompileError) -> SyntaxError { let _ = Reflect::set( &js_err, &"row".into(), - &(err.location().unwrap().row.get()).into(), + &(err.location().unwrap().line.get()).into(), ); let _ = Reflect::set( &js_err, &"col".into(), - &(err.location().unwrap().column.get()).into(), + &(err.location().unwrap().character_offset.get()).into(), ); // | ParseErrorType::UnrecognizedToken(Token::Dedent, _) let can_continue = matches!( diff --git a/wasm/tests/conftest.py b/wasm/tests/conftest.py index 84a2530575..e01f5eddc5 100644 --- a/wasm/tests/conftest.py +++ b/wasm/tests/conftest.py @@ -41,7 +41,7 @@ def pytest_sessionfinish(session): # From https://gist.github.com/butla/2d9a4c0f35ea47b7452156c96a4e7b12 -def wait_for_port(port, host="0.0.0.0", timeout=5.0): +def wait_for_port(port, host="localhost", timeout=5.0): """Wait until a port starts accepting TCP connections. Args: port (int): Port number. @@ -94,7 +94,7 @@ def wdriver(request): options.add_argument("-headless") driver = Driver(options=options) try: - driver.get(f"http://0.0.0.0:{PORT}") + driver.get(f"http://localhost:{PORT}") WebDriverWait(driver, 5).until( EC.presence_of_element_located((By.ID, "rp_loaded")) ) diff --git a/wasm/tests/requirements.txt b/wasm/tests/requirements.txt index 52dc203912..a474c5f74b 100644 --- a/wasm/tests/requirements.txt +++ b/wasm/tests/requirements.txt @@ -1,3 +1,3 @@ pytest -selenium +selenium==4.36.0 certifi From e92c3d4eb04855d0226b3713bd59db9e819bf053 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Wed, 22 Oct 2025 11:55:32 +0300 Subject: [PATCH 6/8] Reapply ruff code --- Cargo.lock | 25 +++++++++++++++++++++++++ Cargo.toml | 1 + compiler/codegen/src/compile.rs | 14 +++++--------- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 53b4f96922..cc0f080fd1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2282,6 +2282,29 @@ dependencies = [ "thiserror 2.0.17", ] +[[package]] +name = "ruff_python_codegen" +version = "0.0.0" +source = "git+https://github.com/astral-sh/ruff.git?tag=0.14.1#2bffef59665ce7d2630dfd72ee99846663660db8" +dependencies = [ + "ruff_python_ast", + "ruff_python_literal", + "ruff_python_parser", + "ruff_source_file", + "ruff_text_size", +] + +[[package]] +name = "ruff_python_literal" +version = "0.0.0" +source = "git+https://github.com/astral-sh/ruff.git?tag=0.14.1#2bffef59665ce7d2630dfd72ee99846663660db8" +dependencies = [ + "bitflags 2.9.4", + "itertools 0.14.0", + "ruff_python_ast", + "unic-ucd-category", +] + [[package]] name = "ruff_python_parser" version = "0.0.0" @@ -2387,7 +2410,9 @@ dependencies = [ "num-complex", "num-traits", "ruff_python_ast", + "ruff_python_codegen", "ruff_python_parser", + "ruff_source_file", "ruff_text_size", "rustpython-compiler-core", "rustpython-literal", diff --git a/Cargo.toml b/Cargo.toml index 3cdc471dc3..77bba61a3b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -159,6 +159,7 @@ rustpython-wtf8 = { path = "wtf8", version = "0.4.0" } rustpython-doc = { git = "https://github.com/RustPython/__doc__", tag = "0.3.0", version = "0.3.0" } ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" } +ruff_python_codegen = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" } ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" } ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" } ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" } diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index d35f3c6e3b..132998fe83 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -148,16 +148,12 @@ enum ComprehensionType { } fn unparse_expr(expr: &Expr) -> String { - // Hack, because we can't do `ruff_python_codegen::Indentation::default()` - // https://github.com/astral-sh/ruff/pull/20216 - let indentation = { - let contents = r"x = 1"; - let module = ruff_python_parser::parse_module(contents).unwrap(); - let stylist = ruff_python_codegen::Stylist::from_tokens(module.tokens(), contents); - stylist.indentation().clone() - }; + use ruff_python_ast::str::Quote; + use ruff_python_codegen::{Generator, Indentation}; - ruff_python_codegen::Generator::new(&indentation, LineEnding::default()).expr(expr) + Generator::new(&Indentation::default(), LineEnding::default()) + .with_preferred_quote(Some(Quote::Single)) + .expr(expr) } fn validate_duplicate_params(params: &Parameters) -> Result<(), CodegenErrorType> { From 4b80a5d74a3c6d5859e8a25b6b5465523ed0aa97 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Wed, 22 Oct 2025 12:21:23 +0300 Subject: [PATCH 7/8] remove git symbols --- Cargo.lock | 4 ---- Cargo.toml | 8 -------- compiler/codegen/src/compile.rs | 4 ---- 3 files changed, 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fe74c9e0bc..cc0f080fd1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2280,7 +2280,6 @@ dependencies = [ "ruff_text_size", "rustc-hash", "thiserror 2.0.17", -<<<<<<< HEAD ] [[package]] @@ -2304,9 +2303,6 @@ dependencies = [ "itertools 0.14.0", "ruff_python_ast", "unic-ucd-category", -||||||| 056795eed -======= ->>>>>>> upstream/main ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 5b1c625cb4..77bba61a3b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -159,15 +159,7 @@ rustpython-wtf8 = { path = "wtf8", version = "0.4.0" } rustpython-doc = { git = "https://github.com/RustPython/__doc__", tag = "0.3.0", version = "0.3.0" } ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" } -<<<<<<< HEAD ruff_python_codegen = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" } -||||||| 056795eed -ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } -ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } -ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } -ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } -======= ->>>>>>> upstream/main ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" } ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" } ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" } diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index 39023b4acc..132998fe83 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -147,7 +147,6 @@ enum ComprehensionType { Dict, } -<<<<<<< HEAD fn unparse_expr(expr: &Expr) -> String { use ruff_python_ast::str::Quote; use ruff_python_codegen::{Generator, Indentation}; @@ -157,9 +156,6 @@ fn unparse_expr(expr: &Expr) -> String { .expr(expr) } -||||||| 056795eed -======= ->>>>>>> upstream/main fn validate_duplicate_params(params: &Parameters) -> Result<(), CodegenErrorType> { let mut seen_params = HashSet::new(); for param in params { From d963b31d34c540d8c9ec84117489b75fc9d587b5 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Wed, 22 Oct 2025 12:50:17 +0300 Subject: [PATCH 8/8] Unmark passing test --- Lib/test/test_typing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 13d0d72e5b..318c088fbc 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -4952,7 +4952,6 @@ def barfoo(x: AT): ... def barfoo2(x: CT): ... self.assertIs(get_type_hints(barfoo2, globals(), locals())['x'], CT) - @unittest.expectedFailure # TODO: RUSTPYTHON; 'List[list["C2"]]' != "List[list['C2']]" def test_generic_pep585_forward_ref(self): # See https://bugs.python.org/issue41370