From 8175bb76cdce08374d9fa03a7c6eca631869aa5e Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 3 Mar 2026 21:10:12 +0900 Subject: [PATCH 1/4] Add vectorcall (PEP 590) dispatch for function calls Add VectorCallFunc slot to PyTypeSlots and vectorcall dispatch path in the interpreter loop for Call and CallKw instructions. Implement vectorcall for PyFunction (with fast path for simple positional-only calls that fills fastlocals directly), PyBoundMethod (avoids prepend_arg O(n) shift), and PyNativeFunction. Add FuncArgs::from_vectorcall helper for fallback conversion. Vectorcall slot is inherited with call slot and cleared when __call__ is overridden in Python subclasses. --- crates/vm/src/builtins/builtin_func.rs | 46 +++++++++ crates/vm/src/builtins/function.rs | 104 +++++++++++++++++++ crates/vm/src/frame.rs | 137 ++++++++++++++++++++++++- crates/vm/src/function/argument.rs | 31 ++++++ crates/vm/src/protocol/callable.rs | 73 ++++++++++++- crates/vm/src/types/slot.rs | 21 +++- crates/vm/src/types/slot_defs.rs | 10 +- 7 files changed, 412 insertions(+), 10 deletions(-) diff --git a/crates/vm/src/builtins/builtin_func.rs b/crates/vm/src/builtins/builtin_func.rs index f0f6bfee729..45b7feb7906 100644 --- a/crates/vm/src/builtins/builtin_func.rs +++ b/crates/vm/src/builtins/builtin_func.rs @@ -224,8 +224,54 @@ impl fmt::Debug for PyNativeMethod { } } +/// Vectorcall for builtin functions (PEP 590). +/// Avoids `prepend_arg` O(n) shift by building args with self at front. +fn vectorcall_native_function( + zelf_obj: &PyObject, + args: Vec, + nargs: usize, + kwnames: Option<&[PyObjectRef]>, + vm: &VirtualMachine, +) -> PyResult { + let zelf: &Py = zelf_obj.downcast_ref().unwrap(); + + // Build FuncArgs with self already at position 0 (no insert(0) needed) + let needs_self = zelf + .zelf + .as_ref() + .is_some_and(|_| !zelf.value.flags.contains(PyMethodFlags::STATIC)); + + let func_args = if needs_self { + let self_obj = zelf.zelf.as_ref().unwrap().clone(); + let total_pos = nargs + 1; + let mut pos_args = Vec::with_capacity(total_pos); + pos_args.push(self_obj); + pos_args.extend(args.into_iter().take(nargs)); + + if let Some(kwnames_slice) = kwnames { + let kwargs = FuncArgs::from_vectorcall(&pos_args, total_pos, Some(kwnames_slice)).kwargs; + FuncArgs { + args: pos_args, + kwargs, + } + } else { + FuncArgs::from(pos_args) + } + } else { + FuncArgs::from_vectorcall(&args, nargs, kwnames) + }; + + (zelf.value.func)(vm, func_args) +} + pub fn init(context: &'static Context) { PyNativeFunction::extend_class(context, context.types.builtin_function_or_method_type); + context + .types + .builtin_function_or_method_type + .slots + .vectorcall + .store(Some(vectorcall_native_function)); } /// Wrapper that provides access to the common PyNativeFunction data diff --git a/crates/vm/src/builtins/function.rs b/crates/vm/src/builtins/function.rs index 02198785815..f46635f3378 100644 --- a/crates/vm/src/builtins/function.rs +++ b/crates/vm/src/builtins/function.rs @@ -1253,8 +1253,112 @@ impl PyCell { } } +/// Vectorcall implementation for PyFunction (PEP 590). +/// Takes owned args to avoid cloning when filling fastlocals. +fn vectorcall_function( + zelf_obj: &PyObject, + mut args: Vec, + nargs: usize, + kwnames: Option<&[PyObjectRef]>, + vm: &VirtualMachine, +) -> PyResult { + let zelf: &Py = zelf_obj.downcast_ref().unwrap(); + let code: &Py = &zelf.code; + + let has_kwargs = kwnames.is_some_and(|kw| !kw.is_empty()); + let is_simple = !has_kwargs + && !code.flags.contains(bytecode::CodeFlags::VARARGS) + && !code.flags.contains(bytecode::CodeFlags::VARKEYWORDS) + && code.kwonlyarg_count == 0 + && !code + .flags + .intersects(bytecode::CodeFlags::GENERATOR | bytecode::CodeFlags::COROUTINE); + + if is_simple && nargs == code.arg_count as usize { + // FAST PATH: simple positional-only call, exact arg count. + // Move owned args directly into fastlocals — no clone needed. + let locals = if code.flags.contains(bytecode::CodeFlags::NEWLOCALS) { + ArgMapping::from_dict_exact(vm.ctx.new_dict()) + } else { + ArgMapping::from_dict_exact(zelf.globals.clone()) + }; + + let frame = Frame::new( + code.to_owned(), + Scope::new(Some(locals), zelf.globals.clone()), + zelf.builtins.clone(), + zelf.closure.as_ref().map_or(&[], |c| c.as_slice()), + Some(zelf.to_owned().into()), + vm, + ) + .into_ref(&vm.ctx); + + { + let fastlocals = unsafe { frame.fastlocals.borrow_mut() }; + for (slot, arg) in fastlocals.iter_mut().zip(args.drain(..nargs)) { + *slot = Some(arg); + } + } + + if let Some(cell2arg) = code.cell2arg.as_deref() { + let fastlocals = unsafe { frame.fastlocals.borrow_mut() }; + for (cell_idx, arg_idx) in cell2arg.iter().enumerate().filter(|(_, i)| **i != -1) { + let x = fastlocals[*arg_idx as usize].take(); + frame.set_cell_contents(cell_idx, x); + } + } + + return vm.run_frame(frame); + } + + // SLOW PATH: construct FuncArgs from owned Vec and delegate to invoke() + let func_args = if has_kwargs { + FuncArgs::from_vectorcall(&args, nargs, kwnames) + } else { + args.truncate(nargs); + FuncArgs::from(args) + }; + zelf.invoke(func_args, vm) +} + +/// Vectorcall implementation for PyBoundMethod (PEP 590). +fn vectorcall_bound_method( + zelf_obj: &PyObject, + args: Vec, + nargs: usize, + kwnames: Option<&[PyObjectRef]>, + vm: &VirtualMachine, +) -> PyResult { + let zelf: &Py = zelf_obj.downcast_ref().unwrap(); + + // Build args with self prepended: [self.object, arg1, ..., argN, kw_val1, ..., kw_valK] + let kw_count = kwnames.map_or(0, |kw| kw.len()); + let total = nargs + 1 + kw_count; + let mut full_args = Vec::with_capacity(total); + full_args.push(zelf.object.clone()); + full_args.extend(args.into_iter().take(nargs + kw_count)); + + // Delegate to inner function's vectorcall if available + let new_nargs = nargs + 1; + zelf.function.vectorcall(full_args, new_nargs, kwnames, vm) +} + pub fn init(context: &'static Context) { PyFunction::extend_class(context, context.types.function_type); + context + .types + .function_type + .slots + .vectorcall + .store(Some(vectorcall_function)); + PyBoundMethod::extend_class(context, context.types.bound_method_type); + context + .types + .bound_method_type + .slots + .vectorcall + .store(Some(vectorcall_bound_method)); + PyCell::extend_class(context, context.types.cell_type); } diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index ad077132ef1..14e756ac897 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -1409,8 +1409,7 @@ impl ExecutingFrame<'_> { } else { self.specialize_call(vm, nargs_val, instr_idx, cache_base); } - let args = self.collect_positional_args(nargs_val); - self.execute_call(args, vm) + self.execute_call_vectorcall(nargs_val, vm) } Instruction::CallKw { nargs } => { let nargs = nargs.get(arg); @@ -1427,8 +1426,7 @@ impl ExecutingFrame<'_> { self.specialize_call_kw(vm, nargs, instr_idx, cache_base); } // Stack: [callable, self_or_null, arg1, ..., argN, kwarg_names] - let args = self.collect_keyword_args(nargs); - self.execute_call(args, vm) + self.execute_call_kw_vectorcall(nargs, vm) } Instruction::CallFunctionEx => { // Stack: [callable, self_or_null, args_tuple, kwargs_or_null] @@ -5665,6 +5663,137 @@ impl ExecutingFrame<'_> { Ok(()) } + /// Vectorcall dispatch for Instruction::Call (positional args only). + /// Uses vectorcall slot if available, otherwise falls back to FuncArgs. + #[inline] + fn execute_call_vectorcall(&mut self, nargs: u32, vm: &VirtualMachine) -> FrameResult { + let nargs_usize = nargs as usize; + let stack_len = self.state.stack.len(); + let callable_idx = stack_len - nargs_usize - 2; + let self_or_null_idx = stack_len - nargs_usize - 1; + let args_start = stack_len - nargs_usize; + + // Check if callable has vectorcall slot + let has_vectorcall = self.state.stack[callable_idx] + .as_ref() + .is_some_and(|sr| sr.as_object().class().slots.vectorcall.load().is_some()); + + if !has_vectorcall { + // Fallback to existing FuncArgs path + let args = self.collect_positional_args(nargs); + return self.execute_call(args, vm); + } + + // Build args slice: [self_or_null?, arg1, ..., argN] + let self_or_null = self.state.stack[self_or_null_idx] + .take() + .map(|sr| sr.to_pyobj()); + let has_self = self_or_null.is_some(); + + let effective_nargs = if has_self { + nargs_usize + 1 + } else { + nargs_usize + }; + let mut args_vec = Vec::with_capacity(effective_nargs); + if let Some(self_val) = self_or_null { + args_vec.push(self_val); + } + for stack_idx in args_start..stack_len { + let val = self.state.stack[stack_idx].take().unwrap().to_pyobj(); + args_vec.push(val); + } + + let callable_obj = self.state.stack[callable_idx].take().unwrap().to_pyobj(); + self.state.stack.truncate(callable_idx); + + let result = callable_obj.vectorcall(args_vec, effective_nargs, None, vm)?; + self.push_value(result); + Ok(None) + } + + /// Vectorcall dispatch for Instruction::CallKw (positional + keyword args). + #[inline] + fn execute_call_kw_vectorcall(&mut self, nargs: u32, vm: &VirtualMachine) -> FrameResult { + let nargs_usize = nargs as usize; + + // Pop kwarg_names tuple from top of stack + let kwarg_names_obj = self.pop_value(); + let kwarg_names_tuple = kwarg_names_obj + .downcast_ref::() + .expect("kwarg names should be tuple"); + let kw_count = kwarg_names_tuple.len(); + + let stack_len = self.state.stack.len(); + let callable_idx = stack_len - nargs_usize - 2; + let self_or_null_idx = stack_len - nargs_usize - 1; + let args_start = stack_len - nargs_usize; + + // Check if callable has vectorcall slot + let has_vectorcall = self.state.stack[callable_idx] + .as_ref() + .is_some_and(|sr| sr.as_object().class().slots.vectorcall.load().is_some()); + + if !has_vectorcall { + // Fallback: reconstruct kwarg_names iterator and use existing path + let kwarg_names_iter = kwarg_names_tuple.as_slice().iter().map(|pyobj| { + pyobj + .downcast_ref::() + .unwrap() + .as_str() + .to_owned() + }); + let args = self.pop_multiple(nargs_usize); + let func_args = FuncArgs::with_kwargs_names(args, kwarg_names_iter); + // pop self_or_null and callable + let self_or_null = self.pop_value_opt(); + let callable = self.pop_value(); + let final_args = if let Some(self_val) = self_or_null { + let mut args = func_args; + args.prepend_arg(self_val); + args + } else { + func_args + }; + let value = callable.call(final_args, vm)?; + self.push_value(value); + return Ok(None); + } + + // Build args: [self?, pos_arg1, ..., pos_argM, kw_val1, ..., kw_valK] + let self_or_null = self.state.stack[self_or_null_idx] + .take() + .map(|sr| sr.to_pyobj()); + let has_self = self_or_null.is_some(); + + let pos_count = nargs_usize - kw_count; + let effective_nargs = if has_self { + pos_count + 1 + } else { + pos_count + }; + + // Build the full args slice: positional (including self) + kwarg values + let total_args = effective_nargs + kw_count; + let mut args_vec = Vec::with_capacity(total_args); + if let Some(self_val) = self_or_null { + args_vec.push(self_val); + } + for stack_idx in args_start..stack_len { + let val = self.state.stack[stack_idx].take().unwrap().to_pyobj(); + args_vec.push(val); + } + + let callable_obj = self.state.stack[callable_idx].take().unwrap().to_pyobj(); + self.state.stack.truncate(callable_idx); + + let kwnames = kwarg_names_tuple.as_slice(); + let result = + callable_obj.vectorcall(args_vec, effective_nargs, Some(kwnames), vm)?; + self.push_value(result); + Ok(None) + } + #[inline] fn execute_call(&mut self, args: FuncArgs, vm: &VirtualMachine) -> FrameResult { // Stack: [callable, self_or_null, ...] diff --git a/crates/vm/src/function/argument.rs b/crates/vm/src/function/argument.rs index a4877cf4042..c52c2d55d18 100644 --- a/crates/vm/src/function/argument.rs +++ b/crates/vm/src/function/argument.rs @@ -135,6 +135,37 @@ impl FuncArgs { } } + /// Create FuncArgs from a vectorcall-style argument slice (PEP 590). + /// `args[..nargs]` are positional, and if `kwnames` is provided, + /// the last `kwnames.len()` entries in `args[nargs..]` are keyword values. + pub fn from_vectorcall( + args: &[PyObjectRef], + nargs: usize, + kwnames: Option<&[PyObjectRef]>, + ) -> Self { + let pos_args = args[..nargs].to_vec(); + let kwargs = if let Some(names) = kwnames { + names + .iter() + .zip(&args[nargs..nargs + names.len()]) + .map(|(name, val)| { + let key = name + .downcast_ref::() + .expect("kwnames must be strings") + .as_str() + .to_owned(); + (key, val.clone()) + }) + .collect() + } else { + IndexMap::new() + }; + Self { + args: pos_args, + kwargs, + } + } + pub fn is_empty(&self) -> bool { self.args.is_empty() && self.kwargs.is_empty() } diff --git a/crates/vm/src/protocol/callable.rs b/crates/vm/src/protocol/callable.rs index 5ffad41a229..ab3e7d815ab 100644 --- a/crates/vm/src/protocol/callable.rs +++ b/crates/vm/src/protocol/callable.rs @@ -1,7 +1,7 @@ use crate::{ builtins::{PyBoundMethod, PyFunction}, function::{FuncArgs, IntoFuncArgs}, - types::GenericMethod, + types::{GenericMethod, VectorCallFunc}, {PyObject, PyObjectRef, PyResult, VirtualMachine}, }; @@ -33,18 +33,43 @@ impl PyObject { vm_trace!("Invoke: {:?} {:?}", callable, args); callable.invoke(args, vm) } + + /// Vectorcall: call with owned positional args + optional kwnames. + /// Falls back to FuncArgs-based call if no vectorcall slot. + #[inline] + pub fn vectorcall( + &self, + args: Vec, + nargs: usize, + kwnames: Option<&[PyObjectRef]>, + vm: &VirtualMachine, + ) -> PyResult { + let Some(callable) = self.to_callable() else { + return Err( + vm.new_type_error(format!("'{}' object is not callable", self.class().name())) + ); + }; + callable.invoke_vectorcall(args, nargs, kwnames, vm) + } } #[derive(Debug)] pub struct PyCallable<'a> { pub obj: &'a PyObject, pub call: GenericMethod, + pub vectorcall: Option, } impl<'a> PyCallable<'a> { pub fn new(obj: &'a PyObject) -> Option { - let call = obj.class().slots.call.load()?; - Some(PyCallable { obj, call }) + let slots = &obj.class().slots; + let call = slots.call.load()?; + let vectorcall = slots.vectorcall.load(); + Some(PyCallable { + obj, + call, + vectorcall, + }) } pub fn invoke(&self, args: impl IntoFuncArgs, vm: &VirtualMachine) -> PyResult { @@ -71,6 +96,48 @@ impl<'a> PyCallable<'a> { result } } + + /// Vectorcall dispatch: use vectorcall slot if available, else fall back to FuncArgs. + #[inline] + pub fn invoke_vectorcall( + &self, + args: Vec, + nargs: usize, + kwnames: Option<&[PyObjectRef]>, + vm: &VirtualMachine, + ) -> PyResult { + if let Some(vc) = self.vectorcall { + if !vm.use_tracing.get() { + return vc(self.obj, args, nargs, kwnames, vm); + } + let is_python_callable = self.obj.downcast_ref::().is_some() + || self.obj.downcast_ref::().is_some(); + if is_python_callable { + vc(self.obj, args, nargs, kwnames, vm) + } else { + let callable = self.obj.to_owned(); + vm.trace_event(TraceEvent::CCall, Some(callable.clone()))?; + let result = vc(self.obj, args, nargs, kwnames, vm); + if result.is_ok() { + vm.trace_event(TraceEvent::CReturn, Some(callable))?; + } else { + let _ = vm.trace_event(TraceEvent::CException, Some(callable)); + } + result + } + } else { + // Fallback: convert owned Vec to FuncArgs + let func_args = FuncArgs { + args: args[..nargs].to_vec(), + kwargs: if let Some(kwn) = kwnames { + FuncArgs::from_vectorcall(&args, nargs, Some(kwn)).kwargs + } else { + indexmap::IndexMap::new() + }, + }; + self.invoke(func_args, vm) + } + } } /// Trace events for sys.settrace and sys.setprofile. diff --git a/crates/vm/src/types/slot.rs b/crates/vm/src/types/slot.rs index d5ce93bd195..85e0d0a01a7 100644 --- a/crates/vm/src/types/slot.rs +++ b/crates/vm/src/types/slot.rs @@ -141,6 +141,7 @@ pub struct PyTypeSlots { // More standard operations (here for binary compatibility) pub hash: AtomicCell>, pub call: AtomicCell>, + pub vectorcall: AtomicCell>, pub str: AtomicCell>, pub repr: AtomicCell>, pub getattro: AtomicCell>, @@ -266,6 +267,17 @@ impl Default for PyTypeFlags { } pub(crate) type GenericMethod = fn(&PyObject, FuncArgs, &VirtualMachine) -> PyResult; +/// Vectorcall function pointer (PEP 590). +/// args: owned positional args followed by kwarg values. +/// nargs: number of positional args (self prepended by caller if needed). +/// kwnames: keyword argument names (last kwnames.len() entries in args are kwarg values). +pub(crate) type VectorCallFunc = fn( + &PyObject, // callable + Vec, // owned args (positional + kwarg values) + usize, // nargs (positional count) + Option<&[PyObjectRef]>, // kwnames (keyword argument names) + &VirtualMachine, +) -> PyResult; pub(crate) type HashFunc = fn(&PyObject, &VirtualMachine) -> PyResult; // CallFunc = GenericMethod pub(crate) type StringifyFunc = fn(&PyObject, &VirtualMachine) -> PyResult>; @@ -767,7 +779,14 @@ impl PyType { accessor.inherit_from_mro(self); } } - SlotAccessor::TpCall => update_main_slot!(call, call_wrapper, Call), + SlotAccessor::TpCall => { + update_main_slot!(call, call_wrapper, Call); + // When __call__ is overridden in Python, clear vectorcall + // so the slow path through call_wrapper is used. + if ADD { + self.slots.vectorcall.store(None); + } + } SlotAccessor::TpIter => update_main_slot!(iter, iter_wrapper, Iter), SlotAccessor::TpIternext => update_main_slot!(iternext, iternext_wrapper, IterNext), SlotAccessor::TpInit => update_main_slot!(init, init_wrapper, Init), diff --git a/crates/vm/src/types/slot_defs.rs b/crates/vm/src/types/slot_defs.rs index 024776f7893..efbcb9f55f6 100644 --- a/crates/vm/src/types/slot_defs.rs +++ b/crates/vm/src/types/slot_defs.rs @@ -451,7 +451,10 @@ impl SlotAccessor { Self::TpHash => inherit_main!(hash), Self::TpRepr => inherit_main!(repr), Self::TpStr => inherit_main!(str), - Self::TpCall => inherit_main!(call), + Self::TpCall => { + inherit_main!(call); + inherit_main!(vectorcall); + } Self::TpIter => inherit_main!(iter), Self::TpIternext => inherit_main!(iternext), Self::TpInit => inherit_main!(init), @@ -568,7 +571,10 @@ impl SlotAccessor { Self::TpHash => copy_main!(hash), Self::TpRepr => copy_main!(repr), Self::TpStr => copy_main!(str), - Self::TpCall => copy_main!(call), + Self::TpCall => { + copy_main!(call); + copy_main!(vectorcall); + } Self::TpIter => copy_main!(iter), Self::TpIternext => copy_main!(iternext), Self::TpInit => { From 2226ae80c0c828e956aa1d8c69b32e584cf19b55 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 3 Mar 2026 21:34:28 +0900 Subject: [PATCH 2/4] Optimize vectorcall: move args instead of clone, use vectorcall in specialized paths - invoke_exact_args takes Vec by value and uses drain() to move args into fastlocals instead of cloning (eliminates refcount overhead) - CallPyGeneral and CallBoundMethodGeneral now call vectorcall_function directly instead of going through FuncArgs + prepend_arg + invoke - CallKwPy and CallKwBoundMethod use vectorcall_function with kwnames - vectorcall_bound_method uses insert(0) on existing Vec instead of allocating a second Vec --- crates/vm/src/builtins/function.rs | 25 +++---- crates/vm/src/frame.rs | 102 ++++++++++++++++++++--------- 2 files changed, 81 insertions(+), 46 deletions(-) diff --git a/crates/vm/src/builtins/function.rs b/crates/vm/src/builtins/function.rs index f46635f3378..cad82acc0e3 100644 --- a/crates/vm/src/builtins/function.rs +++ b/crates/vm/src/builtins/function.rs @@ -648,7 +648,7 @@ impl Py { /// Skips FuncArgs allocation, prepend_arg, and fill_locals_from_args. /// Only valid when: no VARARGS, no VARKEYWORDS, no kwonlyargs, not generator/coroutine, /// and nargs == co_argcount. - pub fn invoke_exact_args(&self, args: &[PyObjectRef], vm: &VirtualMachine) -> PyResult { + pub fn invoke_exact_args(&self, mut args: Vec, vm: &VirtualMachine) -> PyResult { let code: PyRef = (*self.code).to_owned(); debug_assert_eq!(args.len(), code.arg_count as usize); @@ -671,11 +671,11 @@ impl Py { ) .into_ref(&vm.ctx); - // Copy args directly into fastlocals + // Move args directly into fastlocals (no clone/refcount needed) { let fastlocals = unsafe { frame.fastlocals.borrow_mut() }; - for (i, arg) in args.iter().enumerate() { - fastlocals[i] = Some(arg.clone()); + for (slot, arg) in fastlocals.iter_mut().zip(args.drain(..)) { + *slot = Some(arg); } } @@ -1255,7 +1255,7 @@ impl PyCell { /// Vectorcall implementation for PyFunction (PEP 590). /// Takes owned args to avoid cloning when filling fastlocals. -fn vectorcall_function( +pub(crate) fn vectorcall_function( zelf_obj: &PyObject, mut args: Vec, nargs: usize, @@ -1324,23 +1324,18 @@ fn vectorcall_function( /// Vectorcall implementation for PyBoundMethod (PEP 590). fn vectorcall_bound_method( zelf_obj: &PyObject, - args: Vec, + mut args: Vec, nargs: usize, kwnames: Option<&[PyObjectRef]>, vm: &VirtualMachine, ) -> PyResult { let zelf: &Py = zelf_obj.downcast_ref().unwrap(); - // Build args with self prepended: [self.object, arg1, ..., argN, kw_val1, ..., kw_valK] - let kw_count = kwnames.map_or(0, |kw| kw.len()); - let total = nargs + 1 + kw_count; - let mut full_args = Vec::with_capacity(total); - full_args.push(zelf.object.clone()); - full_args.extend(args.into_iter().take(nargs + kw_count)); - - // Delegate to inner function's vectorcall if available + // Insert self at front of existing Vec (avoids 2nd allocation). + // O(n) memmove is cheaper than a 2nd heap alloc+dealloc for typical arg counts. + args.insert(0, zelf.object.clone()); let new_nargs = nargs + 1; - zelf.function.vectorcall(full_args, new_nargs, kwnames, vm) + zelf.function.vectorcall(args, new_nargs, kwnames, vm) } pub fn init(context: &'static Context) { diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 14e756ac897..bfc747302d6 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -12,7 +12,7 @@ use crate::{ builtin_func::PyNativeFunction, descriptor::{MemberGetter, PyMemberDescriptor, PyMethodDescriptor}, frame::stack_analysis, - function::{PyCell, PyCellRef, PyFunction}, + function::{PyCell, PyCellRef, PyFunction, vectorcall_function}, list::PyListIterator, range::PyRangeIterator, tuple::{PyTuple, PyTupleIterator, PyTupleRef}, @@ -3678,7 +3678,7 @@ impl ExecutingFrame<'_> { let _null = self.pop_value_opt(); // self_or_null (NULL) let callable = self.pop_value(); let func = callable.downcast_ref::().unwrap(); - let result = func.invoke_exact_args(&args, vm)?; + let result = func.invoke_exact_args(args, vm)?; self.push_value(result); Ok(None) } else { @@ -3716,7 +3716,7 @@ impl ExecutingFrame<'_> { let mut all_args = Vec::with_capacity(pos_args.len() + 1); all_args.push(self_val); all_args.extend(pos_args); - let result = func.invoke_exact_args(&all_args, vm)?; + let result = func.invoke_exact_args(all_args, vm)?; self.push_value(result); Ok(None) } else { @@ -3934,18 +3934,21 @@ impl ExecutingFrame<'_> { && func.func_version() == cached_version && cached_version != 0 { - let args = self.collect_positional_args(nargs); + let nargs_usize = nargs as usize; + let pos_args: Vec = + self.pop_multiple(nargs_usize).collect(); let self_or_null = self.pop_value_opt(); let callable = self.pop_value(); - let func = callable.downcast_ref::().unwrap(); - let final_args = if let Some(self_val) = self_or_null { - let mut args = args; - args.prepend_arg(self_val); - args + let (args_vec, effective_nargs) = if let Some(self_val) = self_or_null { + let mut v = Vec::with_capacity(nargs_usize + 1); + v.push(self_val); + v.extend(pos_args); + (v, nargs_usize + 1) } else { - args + (pos_args, nargs_usize) }; - let result = func.invoke(final_args, vm)?; + let result = + vectorcall_function(&callable, args_vec, effective_nargs, None, vm)?; self.push_value(result); Ok(None) } else { @@ -3964,13 +3967,21 @@ impl ExecutingFrame<'_> { && func.func_version() == cached_version && cached_version != 0 { - let args = self.collect_positional_args(nargs); + let nargs_usize = nargs as usize; + let pos_args: Vec = + self.pop_multiple(nargs_usize).collect(); let self_val = self.pop_value(); let callable = self.pop_value(); - let func = callable.downcast_ref::().unwrap(); - let mut final_args = args; - final_args.prepend_arg(self_val); - let result = func.invoke(final_args, vm)?; + let mut args_vec = Vec::with_capacity(nargs_usize + 1); + args_vec.push(self_val); + args_vec.extend(pos_args); + let result = vectorcall_function( + &callable, + args_vec, + nargs_usize + 1, + None, + vm, + )?; self.push_value(result); Ok(None) } else { @@ -4226,24 +4237,38 @@ impl ExecutingFrame<'_> { let cached_version = self.code.instructions.read_cache_u32(cache_base + 1); let nargs: u32 = arg.into(); // Stack: [callable, self_or_null, arg1, ..., argN, kwarg_names] - // callable is at position nargs + 2 from top (nargs args + kwarg_names + self_or_null) let callable = self.nth_value(nargs + 2); if let Some(func) = callable.downcast_ref::() && func.func_version() == cached_version && cached_version != 0 { - let args = self.collect_keyword_args(nargs); + let nargs_usize = nargs as usize; + let kwarg_names_obj = self.pop_value(); + let kwarg_names_tuple = kwarg_names_obj + .downcast_ref::() + .expect("kwarg names should be tuple"); + let kw_count = kwarg_names_tuple.len(); + let all_args: Vec = + self.pop_multiple(nargs_usize).collect(); let self_or_null = self.pop_value_opt(); let callable = self.pop_value(); - let func = callable.downcast_ref::().unwrap(); - let final_args = if let Some(self_val) = self_or_null { - let mut args = args; - args.prepend_arg(self_val); - args + let pos_count = nargs_usize - kw_count; + let (args_vec, effective_nargs) = if let Some(self_val) = self_or_null { + let mut v = Vec::with_capacity(nargs_usize + 1); + v.push(self_val); + v.extend(all_args); + (v, pos_count + 1) } else { - args + (all_args, pos_count) }; - let result = func.invoke(final_args, vm)?; + let kwnames = kwarg_names_tuple.as_slice(); + let result = vectorcall_function( + &callable, + args_vec, + effective_nargs, + Some(kwnames), + vm, + )?; self.push_value(result); return Ok(None); } @@ -4262,13 +4287,28 @@ impl ExecutingFrame<'_> { && func.func_version() == cached_version && cached_version != 0 { - let args = self.collect_keyword_args(nargs); - let self_val = self.pop_value(); // self_or_null is always Some here + let nargs_usize = nargs as usize; + let kwarg_names_obj = self.pop_value(); + let kwarg_names_tuple = kwarg_names_obj + .downcast_ref::() + .expect("kwarg names should be tuple"); + let kw_count = kwarg_names_tuple.len(); + let all_args: Vec = + self.pop_multiple(nargs_usize).collect(); + let self_val = self.pop_value(); let callable = self.pop_value(); - let func = callable.downcast_ref::().unwrap(); - let mut final_args = args; - final_args.prepend_arg(self_val); - let result = func.invoke(final_args, vm)?; + let pos_count = nargs_usize - kw_count; + let mut args_vec = Vec::with_capacity(nargs_usize + 1); + args_vec.push(self_val); + args_vec.extend(all_args); + let kwnames = kwarg_names_tuple.as_slice(); + let result = vectorcall_function( + &callable, + args_vec, + pos_count + 1, + Some(kwnames), + vm, + )?; self.push_value(result); return Ok(None); } From 981dafd5207746f9f84fc2497b564a65f15cf34c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 3 Mar 2026 12:35:14 +0000 Subject: [PATCH 3/4] Auto-format: cargo fmt --all --- crates/vm/src/builtins/builtin_func.rs | 3 +- crates/vm/src/frame.rs | 39 +++++++------------------- crates/vm/src/types/slot.rs | 6 ++-- 3 files changed, 15 insertions(+), 33 deletions(-) diff --git a/crates/vm/src/builtins/builtin_func.rs b/crates/vm/src/builtins/builtin_func.rs index 45b7feb7906..f58f166389b 100644 --- a/crates/vm/src/builtins/builtin_func.rs +++ b/crates/vm/src/builtins/builtin_func.rs @@ -249,7 +249,8 @@ fn vectorcall_native_function( pos_args.extend(args.into_iter().take(nargs)); if let Some(kwnames_slice) = kwnames { - let kwargs = FuncArgs::from_vectorcall(&pos_args, total_pos, Some(kwnames_slice)).kwargs; + let kwargs = + FuncArgs::from_vectorcall(&pos_args, total_pos, Some(kwnames_slice)).kwargs; FuncArgs { args: pos_args, kwargs, diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index bfc747302d6..cc6ee8325b8 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -3935,8 +3935,7 @@ impl ExecutingFrame<'_> { && cached_version != 0 { let nargs_usize = nargs as usize; - let pos_args: Vec = - self.pop_multiple(nargs_usize).collect(); + let pos_args: Vec = self.pop_multiple(nargs_usize).collect(); let self_or_null = self.pop_value_opt(); let callable = self.pop_value(); let (args_vec, effective_nargs) = if let Some(self_val) = self_or_null { @@ -3968,20 +3967,14 @@ impl ExecutingFrame<'_> { && cached_version != 0 { let nargs_usize = nargs as usize; - let pos_args: Vec = - self.pop_multiple(nargs_usize).collect(); + let pos_args: Vec = self.pop_multiple(nargs_usize).collect(); let self_val = self.pop_value(); let callable = self.pop_value(); let mut args_vec = Vec::with_capacity(nargs_usize + 1); args_vec.push(self_val); args_vec.extend(pos_args); - let result = vectorcall_function( - &callable, - args_vec, - nargs_usize + 1, - None, - vm, - )?; + let result = + vectorcall_function(&callable, args_vec, nargs_usize + 1, None, vm)?; self.push_value(result); Ok(None) } else { @@ -4248,8 +4241,7 @@ impl ExecutingFrame<'_> { .downcast_ref::() .expect("kwarg names should be tuple"); let kw_count = kwarg_names_tuple.len(); - let all_args: Vec = - self.pop_multiple(nargs_usize).collect(); + let all_args: Vec = self.pop_multiple(nargs_usize).collect(); let self_or_null = self.pop_value_opt(); let callable = self.pop_value(); let pos_count = nargs_usize - kw_count; @@ -4293,8 +4285,7 @@ impl ExecutingFrame<'_> { .downcast_ref::() .expect("kwarg names should be tuple"); let kw_count = kwarg_names_tuple.len(); - let all_args: Vec = - self.pop_multiple(nargs_usize).collect(); + let all_args: Vec = self.pop_multiple(nargs_usize).collect(); let self_val = self.pop_value(); let callable = self.pop_value(); let pos_count = nargs_usize - kw_count; @@ -4302,13 +4293,8 @@ impl ExecutingFrame<'_> { args_vec.push(self_val); args_vec.extend(all_args); let kwnames = kwarg_names_tuple.as_slice(); - let result = vectorcall_function( - &callable, - args_vec, - pos_count + 1, - Some(kwnames), - vm, - )?; + let result = + vectorcall_function(&callable, args_vec, pos_count + 1, Some(kwnames), vm)?; self.push_value(result); return Ok(None); } @@ -5807,11 +5793,7 @@ impl ExecutingFrame<'_> { let has_self = self_or_null.is_some(); let pos_count = nargs_usize - kw_count; - let effective_nargs = if has_self { - pos_count + 1 - } else { - pos_count - }; + let effective_nargs = if has_self { pos_count + 1 } else { pos_count }; // Build the full args slice: positional (including self) + kwarg values let total_args = effective_nargs + kw_count; @@ -5828,8 +5810,7 @@ impl ExecutingFrame<'_> { self.state.stack.truncate(callable_idx); let kwnames = kwarg_names_tuple.as_slice(); - let result = - callable_obj.vectorcall(args_vec, effective_nargs, Some(kwnames), vm)?; + let result = callable_obj.vectorcall(args_vec, effective_nargs, Some(kwnames), vm)?; self.push_value(result); Ok(None) } diff --git a/crates/vm/src/types/slot.rs b/crates/vm/src/types/slot.rs index 85e0d0a01a7..93447fb07f7 100644 --- a/crates/vm/src/types/slot.rs +++ b/crates/vm/src/types/slot.rs @@ -272,9 +272,9 @@ pub(crate) type GenericMethod = fn(&PyObject, FuncArgs, &VirtualMachine) -> PyRe /// nargs: number of positional args (self prepended by caller if needed). /// kwnames: keyword argument names (last kwnames.len() entries in args are kwarg values). pub(crate) type VectorCallFunc = fn( - &PyObject, // callable - Vec, // owned args (positional + kwarg values) - usize, // nargs (positional count) + &PyObject, // callable + Vec, // owned args (positional + kwarg values) + usize, // nargs (positional count) Option<&[PyObjectRef]>, // kwnames (keyword argument names) &VirtualMachine, ) -> PyResult; From 4d8cf97452fd9f9dad64832a20d56a9d46fc4a10 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 3 Mar 2026 21:49:16 +0900 Subject: [PATCH 4/4] Fix vectorcall_native_function kwarg slice out-of-bounds When needs_self was true and kwargs were present, pos_args only contained positional args (self + original positionals) but from_vectorcall expected kwarg values to follow in the slice. Build the full args array (self + all original args including kwarg values) before passing to from_vectorcall. --- crates/vm/src/builtins/builtin_func.rs | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/crates/vm/src/builtins/builtin_func.rs b/crates/vm/src/builtins/builtin_func.rs index f58f166389b..bc72b1ad533 100644 --- a/crates/vm/src/builtins/builtin_func.rs +++ b/crates/vm/src/builtins/builtin_func.rs @@ -243,21 +243,10 @@ fn vectorcall_native_function( let func_args = if needs_self { let self_obj = zelf.zelf.as_ref().unwrap().clone(); - let total_pos = nargs + 1; - let mut pos_args = Vec::with_capacity(total_pos); - pos_args.push(self_obj); - pos_args.extend(args.into_iter().take(nargs)); - - if let Some(kwnames_slice) = kwnames { - let kwargs = - FuncArgs::from_vectorcall(&pos_args, total_pos, Some(kwnames_slice)).kwargs; - FuncArgs { - args: pos_args, - kwargs, - } - } else { - FuncArgs::from(pos_args) - } + let mut all_args = Vec::with_capacity(args.len() + 1); + all_args.push(self_obj); + all_args.extend(args); + FuncArgs::from_vectorcall(&all_args, nargs + 1, kwnames) } else { FuncArgs::from_vectorcall(&args, nargs, kwnames) };