From 59d61a5f4a6cf9d73eaec7ac3988f8d39e1c7bfc Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sun, 31 Aug 2025 13:25:24 +0300 Subject: [PATCH 1/6] Add base tool --- scripts/libc_posix.py | 119 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 scripts/libc_posix.py diff --git a/scripts/libc_posix.py b/scripts/libc_posix.py new file mode 100644 index 0000000000..735625022f --- /dev/null +++ b/scripts/libc_posix.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +import collections +import re +import urllib.request +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterator + +CONSTS_PAT = re.compile(r"\b_*[A-Z]+(?:_+[A-Z]+)*_*\b") +OS_CONSTS_PAT = re.compile( + r"\bos\.(_*[A-Z]+(?:_+[A-Z]+)*_*)" +) # TODO: Exclude matches if they have `(` after (those are functions) + + +LIBC_VERSION = "0.2.175" + +EXCLUDE = frozenset( + { + # Defined at `vm/src/stdlib/os.rs` + "O_APPEND", + "O_CREAT", + "O_EXCL", + "O_RDONLY", + "O_RDWR", + "O_TRUNC", + "O_WRONLY", + "SEEK_CUR", + "SEEK_END", + "SEEK_SET", + # Functions, not consts + "WCOREDUMP", + "WIFCONTINUED", + "WIFSTOPPED", + "WIFSIGNALED", + "WIFEXITED", + "WEXITSTATUS", + "WSTOPSIG", + "WTERMSIG", + # False positive + # "EOF", + } +) + + +def build_url(fname: str) -> str: + return f"https://raw.githubusercontent.com/rust-lang/libc/refs/tags/{LIBC_VERSION}/libc-test/semver/{fname}.txt" + + +TARGET_OS = { + "android": "android", + "dragonfly": "dragonfly", + "freebsd": "freebsd", + "linux": "linux", + "macos": "apple", + "netbsd": "netbsd", + "openbsd": "openbsd", + "redox": "redox", + # solaris? +} + + +def get_consts(url: str, pattern: re.Pattern = CONSTS_PAT) -> frozenset[str]: + with urllib.request.urlopen(url) as f: + resp = f.read().decode() + + return frozenset(pattern.findall(resp)) - EXCLUDE + + +def format_groups(groups: dict) -> "Iterator[tuple[str, str]]": + # sort by length, then alphabet. so we will have a consistent output + for targets, consts in sorted( + groups.items(), key=lambda t: (len(t[0]), sorted(t[0])) + ): + cond = ", ".join(f'target_os = "{target_os}"' for target_os in sorted(targets)) + if len(targets) > 1: + cond = f"any({cond})" + cfg = f"#[cfg({cond})]" + + imports = ", ".join(sorted(consts)) + use = f"use libc::{{{imports}}};" + yield cfg, use + + +def main(): + wanted_consts = get_consts( + "https://docs.python.org/3.13/library/os.html", # Should we read from https://github.com/python/cpython/blob/bcee1c322115c581da27600f2ae55e5439c027eb/Modules/posixmodule.c#L17023 instead? + pattern=OS_CONSTS_PAT, + ) + available = { + target_os: get_consts(build_url(fname)) + for target_os, fname in TARGET_OS.items() + } + + group_consts = collections.defaultdict(set) + for const in wanted_consts: + target_oses = frozenset( + target_os for target_os, consts in available.items() if const in consts + ) + if not target_oses: + continue + + group_consts[target_oses].add(const) + group_consts = dict(group_consts) + + code = "\n\n".join( + f""" +{cfg} +#[pyattr] +{use} +""".strip() + for cfg, use in format_groups(group_consts) + ) + + print(code) + + +if __name__ == "__main__": + main() From 9f5285cc1ac00e9eab418480ace12ea26646291c Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sun, 31 Aug 2025 20:22:51 +0300 Subject: [PATCH 2/6] Apply tool --- vm/src/stdlib/posix.rs | 209 +++++++++++++++++++++++++++++------------ 1 file changed, 150 insertions(+), 59 deletions(-) diff --git a/vm/src/stdlib/posix.rs b/vm/src/stdlib/posix.rs index 0f8251193d..fa8b999675 100644 --- a/vm/src/stdlib/posix.rs +++ b/vm/src/stdlib/posix.rs @@ -47,39 +47,177 @@ pub mod module { }; use strum_macros::{EnumIter, EnumString}; + #[cfg(target_os = "android")] #[pyattr] - use libc::{PRIO_PGRP, PRIO_PROCESS, PRIO_USER}; + use libc::{ + F_OK, O_CLOEXEC, O_DIRECTORY, O_NOFOLLOW, O_NONBLOCK, PRIO_PGRP, PRIO_PROCESS, PRIO_USER, + R_OK, RTLD_GLOBAL, RTLD_LAZY, RTLD_LOCAL, RTLD_NOW, W_OK, WCONTINUED, WNOHANG, WUNTRACED, + X_OK, + }; + + #[cfg(target_os = "freebsd")] + #[pyattr] + use libc::{MFD_HUGE_MASK, SF_MNOWAIT, SF_NOCACHE, SF_NODISKIO, SF_SYNC}; + + #[cfg(target_os = "linux")] + #[pyattr] + use libc::PIDFD_NONBLOCK; + + #[cfg(target_os = "macos")] + #[pyattr] + use libc::{ + O_EVTONLY, O_NOFOLLOW_ANY, PRIO_DARWIN_BG, PRIO_DARWIN_NONUI, PRIO_DARWIN_PROCESS, + PRIO_DARWIN_THREAD, + }; + + #[cfg(any(target_os = "android", target_os = "linux"))] + #[pyattr] + use libc::{ + CLONE_FILES, CLONE_FS, CLONE_NEWCGROUP, CLONE_NEWIPC, CLONE_NEWNET, CLONE_NEWNS, + CLONE_NEWPID, CLONE_NEWUSER, CLONE_NEWUTS, CLONE_SIGHAND, CLONE_SYSVSEM, CLONE_THREAD, + CLONE_VM, EFD_CLOEXEC, EFD_NONBLOCK, EFD_SEMAPHORE, O_NOATIME, O_TMPFILE, P_PIDFD, + SCHED_BATCH, SCHED_IDLE, SCHED_RESET_ON_FORK, SPLICE_F_MORE, SPLICE_F_MOVE, + SPLICE_F_NONBLOCK, + }; + + #[cfg(any(target_os = "macos", target_os = "redox"))] + #[pyattr] + use libc::O_SYMLINK; + + #[cfg(any(target_os = "android", target_os = "freebsd", target_os = "linux"))] + #[pyattr] + use libc::{ + MFD_ALLOW_SEALING, MFD_CLOEXEC, MFD_HUGETLB, POSIX_FADV_DONTNEED, POSIX_FADV_NOREUSE, + POSIX_FADV_NORMAL, POSIX_FADV_RANDOM, POSIX_FADV_SEQUENTIAL, POSIX_FADV_WILLNEED, + }; + + #[cfg(any(target_os = "android", target_os = "linux", target_os = "netbsd"))] + #[pyattr] + use libc::{TFD_CLOEXEC, TFD_NONBLOCK, TFD_TIMER_ABSTIME, TFD_TIMER_CANCEL_ON_SET}; + + #[cfg(any(target_os = "linux", target_os = "macos", target_os = "netbsd"))] + #[pyattr] + use libc::{XATTR_CREATE, XATTR_REPLACE}; #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "linux", + target_os = "netbsd" + ))] + #[pyattr] + use libc::{GRND_NONBLOCK, GRND_RANDOM}; + + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "linux", + target_os = "redox" + ))] + #[pyattr] + use libc::O_PATH; + + #[cfg(any( + target_os = "android", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd" + ))] + #[pyattr] + use libc::O_RSYNC; + + #[cfg(any( + target_os = "android", target_os = "dragonfly", target_os = "freebsd", target_os = "linux", target_os = "macos" ))] #[pyattr] - use libc::{SEEK_DATA, SEEK_HOLE}; + use libc::{RTLD_NODELETE, SEEK_DATA, SEEK_HOLE}; - #[cfg(not(any(target_os = "redox", target_os = "freebsd")))] + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "linux", + target_os = "netbsd" + ))] #[pyattr] - use libc::O_DSYNC; + use libc::O_DIRECT; + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "linux", + target_os = "macos", + target_os = "netbsd" + ))] #[pyattr] - use libc::{O_CLOEXEC, O_NONBLOCK, WNOHANG}; + use libc::RTLD_NOLOAD; - #[cfg(target_os = "macos")] + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "linux", + target_os = "macos", + target_os = "netbsd", + target_os = "openbsd" + ))] #[pyattr] - use libc::{O_EVTONLY, O_FSYNC, O_NOFOLLOW_ANY, O_SYMLINK}; + use libc::O_DSYNC; - #[cfg(not(target_os = "redox"))] + #[cfg(any( + target_os = "dragonfly", + target_os = "freebsd", + target_os = "linux", + target_os = "macos", + target_os = "netbsd", + target_os = "openbsd" + ))] #[pyattr] - use libc::{O_NDELAY, O_NOCTTY}; + use libc::SCHED_OTHER; + #[cfg(any( + target_os = "dragonfly", + target_os = "freebsd", + target_os = "macos", + target_os = "netbsd", + target_os = "openbsd", + target_os = "redox" + ))] #[pyattr] - use libc::{RTLD_GLOBAL, RTLD_LAZY, RTLD_LOCAL, RTLD_NOW}; + use libc::{O_EXLOCK, O_FSYNC, O_SHLOCK}; - #[cfg(target_os = "linux")] + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "linux", + target_os = "macos", + target_os = "netbsd", + target_os = "openbsd" + ))] #[pyattr] - use libc::{GRND_NONBLOCK, GRND_RANDOM}; + use libc::{ + CLD_CONTINUED, CLD_DUMPED, CLD_EXITED, CLD_KILLED, CLD_STOPPED, CLD_TRAPPED, F_LOCK, + F_TEST, F_TLOCK, F_ULOCK, O_NDELAY, O_NOCTTY, O_SYNC, P_ALL, P_PGID, P_PID, SCHED_FIFO, + SCHED_RR, + }; + + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "linux", + target_os = "macos", + target_os = "netbsd", + target_os = "openbsd", + target_os = "redox" + ))] + #[pyattr] + use libc::{O_ASYNC, WEXITED, WNOWAIT, WSTOPPED}; #[pyattr] const EX_OK: i8 = exitcode::OK as i8; @@ -129,49 +267,6 @@ pub mod module { #[pyattr] const EX_CONFIG: i8 = exitcode::CONFIG as i8; - #[cfg(any( - target_os = "macos", - target_os = "linux", - target_os = "android", - target_os = "freebsd", - target_os = "dragonfly", - target_os = "netbsd", - target_os = "macos" - ))] - #[pyattr] - const SCHED_RR: i32 = libc::SCHED_RR; - - #[cfg(any( - target_os = "macos", - target_os = "linux", - target_os = "android", - target_os = "freebsd", - target_os = "dragonfly", - target_os = "netbsd", - target_os = "macos" - ))] - #[pyattr] - const SCHED_FIFO: i32 = libc::SCHED_FIFO; - - #[cfg(any( - target_os = "macos", - target_os = "linux", - target_os = "freebsd", - target_os = "dragonfly", - target_os = "netbsd", - target_os = "macos" - ))] - #[pyattr] - const SCHED_OTHER: i32 = libc::SCHED_OTHER; - - #[cfg(any(target_os = "linux", target_os = "android"))] - #[pyattr] - const SCHED_IDLE: i32 = libc::SCHED_IDLE; - - #[cfg(any(target_os = "linux", target_os = "android"))] - #[pyattr] - const SCHED_BATCH: i32 = libc::SCHED_BATCH; - #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] #[pyattr] const POSIX_SPAWN_OPEN: i32 = PosixSpawnFileActionIdentifier::Open as i32; @@ -184,10 +279,6 @@ pub mod module { #[pyattr] const POSIX_SPAWN_DUP2: i32 = PosixSpawnFileActionIdentifier::Dup2 as i32; - #[cfg(target_os = "macos")] - #[pyattr] - const _COPYFILE_DATA: u32 = 1 << 3; - impl TryFromObject for BorrowedFd<'_> { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { let fd = i32::try_from_object(vm, obj)?; From ef0d9c286164fd523cadfc52ae9d9f9b22988569 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sun, 31 Aug 2025 20:33:23 +0300 Subject: [PATCH 3/6] Support renames and extras --- scripts/libc_posix.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/libc_posix.py b/scripts/libc_posix.py index 735625022f..c6b293b298 100644 --- a/scripts/libc_posix.py +++ b/scripts/libc_posix.py @@ -42,6 +42,11 @@ } ) +EXTRAS = { + frozenset({"macos"}): {"COPYFILE_DATA"}, +} +RENAMES = {"COPYFILE_DATA": "_COPYFILE_DATA"} + def build_url(fname: str) -> str: return f"https://raw.githubusercontent.com/rust-lang/libc/refs/tags/{LIBC_VERSION}/libc-test/semver/{fname}.txt" @@ -77,7 +82,10 @@ def format_groups(groups: dict) -> "Iterator[tuple[str, str]]": cond = f"any({cond})" cfg = f"#[cfg({cond})]" - imports = ", ".join(sorted(consts)) + imports = ", ".join( + const if const not in RENAMES else f"{const} as {RENAMES[const]}" + for const in sorted(consts) + ) use = f"use libc::{{{imports}}};" yield cfg, use @@ -101,7 +109,7 @@ def main(): continue group_consts[target_oses].add(const) - group_consts = dict(group_consts) + group_consts = {grp: v | EXTRAS.get(grp, set()) for grp, v in group_consts.items()} code = "\n\n".join( f""" From f66367ecd53c258cce5d12deb3beca8be9822c33 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sun, 31 Aug 2025 20:36:19 +0300 Subject: [PATCH 4/6] gen & apply --- vm/src/stdlib/posix.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vm/src/stdlib/posix.rs b/vm/src/stdlib/posix.rs index fa8b999675..bed2532d20 100644 --- a/vm/src/stdlib/posix.rs +++ b/vm/src/stdlib/posix.rs @@ -66,8 +66,8 @@ pub mod module { #[cfg(target_os = "macos")] #[pyattr] use libc::{ - O_EVTONLY, O_NOFOLLOW_ANY, PRIO_DARWIN_BG, PRIO_DARWIN_NONUI, PRIO_DARWIN_PROCESS, - PRIO_DARWIN_THREAD, + COPYFILE_DATA as _COPYFILE_DATA, O_EVTONLY, O_NOFOLLOW_ANY, PRIO_DARWIN_BG, + PRIO_DARWIN_NONUI, PRIO_DARWIN_PROCESS, PRIO_DARWIN_THREAD, }; #[cfg(any(target_os = "android", target_os = "linux"))] From e9594c3e8dcb390bb577c432600760dfb8974fc9 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sun, 31 Aug 2025 21:36:33 +0300 Subject: [PATCH 5/6] Add WNOHANG --- scripts/libc_posix.py | 1 + vm/src/stdlib/posix.rs | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/libc_posix.py b/scripts/libc_posix.py index c6b293b298..4c41fb7a7b 100644 --- a/scripts/libc_posix.py +++ b/scripts/libc_posix.py @@ -28,6 +28,7 @@ "SEEK_CUR", "SEEK_END", "SEEK_SET", + "WNOHANG", # Functions, not consts "WCOREDUMP", "WIFCONTINUED", diff --git a/vm/src/stdlib/posix.rs b/vm/src/stdlib/posix.rs index bed2532d20..0358791264 100644 --- a/vm/src/stdlib/posix.rs +++ b/vm/src/stdlib/posix.rs @@ -51,8 +51,7 @@ pub mod module { #[pyattr] use libc::{ F_OK, O_CLOEXEC, O_DIRECTORY, O_NOFOLLOW, O_NONBLOCK, PRIO_PGRP, PRIO_PROCESS, PRIO_USER, - R_OK, RTLD_GLOBAL, RTLD_LAZY, RTLD_LOCAL, RTLD_NOW, W_OK, WCONTINUED, WNOHANG, WUNTRACED, - X_OK, + R_OK, RTLD_GLOBAL, RTLD_LAZY, RTLD_LOCAL, RTLD_NOW, W_OK, WCONTINUED, WUNTRACED, X_OK, }; #[cfg(target_os = "freebsd")] @@ -219,6 +218,9 @@ pub mod module { #[pyattr] use libc::{O_ASYNC, WEXITED, WNOWAIT, WSTOPPED}; + #[pyattr] + use libc::WNOHANG; + #[pyattr] const EX_OK: i8 = exitcode::OK as i8; From 29c40ae66b2d762fb89f939e35c369dda9518689 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Mon, 1 Sep 2025 01:32:48 +0300 Subject: [PATCH 6/6] Add consts from unix --- scripts/libc_posix.py | 7 +++++-- vm/src/stdlib/posix.rs | 19 ++++++++----------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/scripts/libc_posix.py b/scripts/libc_posix.py index 4c41fb7a7b..9ed6890e70 100644 --- a/scripts/libc_posix.py +++ b/scripts/libc_posix.py @@ -28,7 +28,6 @@ "SEEK_CUR", "SEEK_END", "SEEK_SET", - "WNOHANG", # Functions, not consts "WCOREDUMP", "WIFCONTINUED", @@ -63,6 +62,7 @@ def build_url(fname: str) -> str: "openbsd": "openbsd", "redox": "redox", # solaris? + "unix": "unix", } @@ -78,7 +78,10 @@ def format_groups(groups: dict) -> "Iterator[tuple[str, str]]": for targets, consts in sorted( groups.items(), key=lambda t: (len(t[0]), sorted(t[0])) ): - cond = ", ".join(f'target_os = "{target_os}"' for target_os in sorted(targets)) + cond = ", ".join( + f'target_os = "{target_os}"' if target_os != "unix" else target_os + for target_os in sorted(targets) + ) if len(targets) > 1: cond = f"any({cond})" cfg = f"#[cfg({cond})]" diff --git a/vm/src/stdlib/posix.rs b/vm/src/stdlib/posix.rs index 0358791264..84a52e3d24 100644 --- a/vm/src/stdlib/posix.rs +++ b/vm/src/stdlib/posix.rs @@ -47,13 +47,6 @@ pub mod module { }; use strum_macros::{EnumIter, EnumString}; - #[cfg(target_os = "android")] - #[pyattr] - use libc::{ - F_OK, O_CLOEXEC, O_DIRECTORY, O_NOFOLLOW, O_NONBLOCK, PRIO_PGRP, PRIO_PROCESS, PRIO_USER, - R_OK, RTLD_GLOBAL, RTLD_LAZY, RTLD_LOCAL, RTLD_NOW, W_OK, WCONTINUED, WUNTRACED, X_OK, - }; - #[cfg(target_os = "freebsd")] #[pyattr] use libc::{MFD_HUGE_MASK, SF_MNOWAIT, SF_NOCACHE, SF_NODISKIO, SF_SYNC}; @@ -79,6 +72,14 @@ pub mod module { SPLICE_F_NONBLOCK, }; + #[cfg(any(target_os = "android", unix))] + #[pyattr] + use libc::{ + F_OK, O_CLOEXEC, O_DIRECTORY, O_NOFOLLOW, O_NONBLOCK, PRIO_PGRP, PRIO_PROCESS, PRIO_USER, + R_OK, RTLD_GLOBAL, RTLD_LAZY, RTLD_LOCAL, RTLD_NOW, W_OK, WCONTINUED, WNOHANG, WUNTRACED, + X_OK, + }; + #[cfg(any(target_os = "macos", target_os = "redox"))] #[pyattr] use libc::O_SYMLINK; @@ -217,10 +218,6 @@ pub mod module { ))] #[pyattr] use libc::{O_ASYNC, WEXITED, WNOWAIT, WSTOPPED}; - - #[pyattr] - use libc::WNOHANG; - #[pyattr] const EX_OK: i8 = exitcode::OK as i8;