From 341ca231cce044398dfd903902582f0b8016e58b Mon Sep 17 00:00:00 2001 From: stijn Date: Tue, 24 Oct 2023 17:08:43 +0200 Subject: [PATCH 1/9] tests: Make socket tests compatible with CPython on windows. Note these are mostly the 'bare' socket tests, not the ssl/tls ones for instance: most of these don't run on CPython because of incompatible wrap_socket() arguments. The change mostly consists of checking the WSA error codes next to the errno ones and these are written as numeric values because the names (like WSAEAGAIN) are only available in CPython and not in micropython. Signed-off-by: stijn --- tests/extmod/socket_tcp_basic.py | 2 +- tests/extmod/socket_udp_nonblock.py | 2 +- tests/multi_net/asyncio_tcp_client_rst.py | 2 +- tests/multi_net/asyncio_tcp_client_rst.py.exp | 2 +- tests/multi_net/tcp_accept_recv.py | 2 +- tests/net_hosted/accept_nonblock.py | 2 +- tests/net_hosted/accept_timeout.py | 4 +++- tests/net_hosted/connect_nonblock.py | 2 +- tests/net_hosted/connect_timeout.py | 2 +- tests/net_inet/ssl_errors.py | 2 +- 10 files changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/extmod/socket_tcp_basic.py b/tests/extmod/socket_tcp_basic.py index ebd30f7862cc8..30d7dbd44eb20 100644 --- a/tests/extmod/socket_tcp_basic.py +++ b/tests/extmod/socket_tcp_basic.py @@ -11,4 +11,4 @@ try: s.recv(1) except OSError as er: - print("ENOTCONN:", er.errno == errno.ENOTCONN) + print("ENOTCONN:", er.errno in (errno.ENOTCONN, 10057)) diff --git a/tests/extmod/socket_udp_nonblock.py b/tests/extmod/socket_udp_nonblock.py index 1e74e2917dc30..dae2be2e5251e 100644 --- a/tests/extmod/socket_udp_nonblock.py +++ b/tests/extmod/socket_udp_nonblock.py @@ -18,4 +18,4 @@ try: s.recv(1) except OSError as er: - print("EAGAIN:", er.errno == errno.EAGAIN) + print("EAGAIN:", er.errno in (errno.EAGAIN, 10035)) diff --git a/tests/multi_net/asyncio_tcp_client_rst.py b/tests/multi_net/asyncio_tcp_client_rst.py index af70c2e22eef6..f2d18ed0b0f3a 100644 --- a/tests/multi_net/asyncio_tcp_client_rst.py +++ b/tests/multi_net/asyncio_tcp_client_rst.py @@ -21,7 +21,7 @@ async def handle_connection(reader, writer): writer.close() await writer.wait_closed() except OSError as er: - print("OSError", er.errno) + print("OSError", er.errno in (22, 104)) ev.set() diff --git a/tests/multi_net/asyncio_tcp_client_rst.py.exp b/tests/multi_net/asyncio_tcp_client_rst.py.exp index 920d1bb8d7296..8dee3c14a350a 100644 --- a/tests/multi_net/asyncio_tcp_client_rst.py.exp +++ b/tests/multi_net/asyncio_tcp_client_rst.py.exp @@ -1,5 +1,5 @@ --- instance0 --- b'GET / HTTP' -OSError 104 +OSError True --- instance1 --- diff --git a/tests/multi_net/tcp_accept_recv.py b/tests/multi_net/tcp_accept_recv.py index dee14e3b9774c..d9b700fb62b80 100644 --- a/tests/multi_net/tcp_accept_recv.py +++ b/tests/multi_net/tcp_accept_recv.py @@ -17,7 +17,7 @@ def instance0(): try: print("recv", s.recv(10)) # should raise Errno 107 ENOTCONN except OSError as er: - print(er.errno in (107, 128)) + print(er.errno in (107, 128, 10057)) s.close() diff --git a/tests/net_hosted/accept_nonblock.py b/tests/net_hosted/accept_nonblock.py index 30d2033e652e1..915174a8082c1 100644 --- a/tests/net_hosted/accept_nonblock.py +++ b/tests/net_hosted/accept_nonblock.py @@ -9,5 +9,5 @@ try: s.accept() except OSError as er: - print(er.errno == 11) # 11 is EAGAIN + print(er.errno in (11, 10035)) # 11 is EAGAIN s.close() diff --git a/tests/net_hosted/accept_timeout.py b/tests/net_hosted/accept_timeout.py index 865d2aad26f25..2f047ee84399e 100644 --- a/tests/net_hosted/accept_timeout.py +++ b/tests/net_hosted/accept_timeout.py @@ -15,5 +15,7 @@ try: s.accept() except OSError as er: - print(er.errno in (errno.ETIMEDOUT, "timed out")) # CPython uses a string instead of errno + print( + er.errno == errno.ETIMEDOUT or str(er) == "timed out" + ) # CPython uses a string instead of errno s.close() diff --git a/tests/net_hosted/connect_nonblock.py b/tests/net_hosted/connect_nonblock.py index 781f1a4ee2cdc..4640dd5808b59 100644 --- a/tests/net_hosted/connect_nonblock.py +++ b/tests/net_hosted/connect_nonblock.py @@ -9,7 +9,7 @@ def test(peer_addr): try: s.connect(peer_addr) except OSError as er: - print(er.errno == errno.EINPROGRESS) + print(er.errno in (errno.EINPROGRESS, 10035)) s.close() diff --git a/tests/net_hosted/connect_timeout.py b/tests/net_hosted/connect_timeout.py index 5f35047c8c892..a2d6c78b3cb80 100644 --- a/tests/net_hosted/connect_timeout.py +++ b/tests/net_hosted/connect_timeout.py @@ -20,5 +20,5 @@ def test(peer_addr, timeout, expected_exc): # 1.1.1.1:8000 seem to reliably timeout, so use that. addr = socket.getaddrinfo("1.1.1.1", 8000)[0][-1] - test(addr, 0, (errno.EINPROGRESS,)) + test(addr, 0, (errno.EINPROGRESS, 10035)) test(addr, 1, (errno.ETIMEDOUT, "timed out")) # CPython uses a string instead of errno diff --git a/tests/net_inet/ssl_errors.py b/tests/net_inet/ssl_errors.py index bc4e5910bccd8..5798a62f4640f 100644 --- a/tests/net_inet/ssl_errors.py +++ b/tests/net_inet/ssl_errors.py @@ -12,7 +12,7 @@ def test(addr, hostname, block=True): s.connect(addr) print("connected") except OSError as e: - if e.errno != errno.EINPROGRESS: + if e.errno not in (errno.EINPROGRESS, 10035): raise print("EINPROGRESS") From e18693d533ca6080b0dd8cd768c3deaccfd6e3e2 Mon Sep 17 00:00:00 2001 From: stijn Date: Wed, 25 Oct 2023 11:55:38 +0200 Subject: [PATCH 2/9] tests: Add socket timeout tests for connect and recv. Signed-off-by: stijn --- tests/multi_net/tcp_recv_timeout.py | 37 +++++++++++++++++++++++++ tests/multi_net/tcp_recv_timeout.py.exp | 6 ++++ 2 files changed, 43 insertions(+) create mode 100644 tests/multi_net/tcp_recv_timeout.py create mode 100644 tests/multi_net/tcp_recv_timeout.py.exp diff --git a/tests/multi_net/tcp_recv_timeout.py b/tests/multi_net/tcp_recv_timeout.py new file mode 100644 index 0000000000000..415ee9dff16c3 --- /dev/null +++ b/tests/multi_net/tcp_recv_timeout.py @@ -0,0 +1,37 @@ +import errno +import socket + +PORT = 8000 + + +# Server +def instance0(): + multitest.globals(IP=multitest.get_network_ip()) + s = socket.socket() + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind(socket.getaddrinfo("0.0.0.0", PORT)[0][-1]) + s.listen() + multitest.next() + s2, _ = s.accept() + s2.settimeout(0.2) + try: + s2.recv(1) + except OSError as er: + print(er.errno in (errno.ETIMEDOUT, errno.EAGAIN) or str(er) == "timed out") + multitest.next() + s2.close() + s.close() + + +# Client +def instance1(): + multitest.next() + s = socket.socket() + s.connect(socket.getaddrinfo(IP, PORT)[0][-1]) + s.settimeout(0.2) + try: + s.recv(1) + except OSError as er: + print(er.errno in (errno.ETIMEDOUT, errno.EAGAIN) or str(er) == "timed out") + multitest.next() + s.close() diff --git a/tests/multi_net/tcp_recv_timeout.py.exp b/tests/multi_net/tcp_recv_timeout.py.exp new file mode 100644 index 0000000000000..6c19840fddab4 --- /dev/null +++ b/tests/multi_net/tcp_recv_timeout.py.exp @@ -0,0 +1,6 @@ +--- instance0 --- +True +NEXT +--- instance1 --- +True +NEXT From a0e15f13762a8db9fc2741bc8f73eb70b3fc946a Mon Sep 17 00:00:00 2001 From: stijn Date: Wed, 25 Oct 2023 16:53:28 +0200 Subject: [PATCH 3/9] unix/modsocket: Introduce typedefs and macros. Upcoming commits are going to port this file to work with sockets on windows so prepare for that with the platform-specific bits. Signed-off-by: stijn --- ports/unix/modsocket.c | 68 ++++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/ports/unix/modsocket.c b/ports/unix/modsocket.c index 6d6059ae44c54..22eb6496f9bfd 100644 --- a/ports/unix/modsocket.c +++ b/ports/unix/modsocket.c @@ -53,6 +53,16 @@ #include "extmod/vfs.h" #include +typedef socklen_t sock_len_t; +typedef int socket_t; +typedef ssize_t socket_size_t; + +#define socket_errno errno +#define socket_eintr EINTR +#define read_socket read +#define write_socket write +#define close_socket close + /* The idea of this module is to implement reasonable minimum of socket-related functions to write typical clients and servers. @@ -71,18 +81,18 @@ // fields should have the same layout. typedef struct _mp_obj_socket_t { mp_obj_base_t base; - int fd; + socket_t fd; bool blocking; } mp_obj_socket_t; const mp_obj_type_t mp_type_socket; // Helper functions -static inline mp_obj_t mp_obj_from_sockaddr(const struct sockaddr *addr, socklen_t len) { +static inline mp_obj_t mp_obj_from_sockaddr(const struct sockaddr *addr, sock_len_t len) { return mp_obj_new_bytes((const byte *)addr, len); } -static mp_obj_socket_t *socket_new(int fd) { +static mp_obj_socket_t *socket_new(socket_t fd) { mp_obj_socket_t *o = mp_obj_malloc(mp_obj_socket_t, &mp_type_socket); o->fd = fd; o->blocking = true; @@ -98,8 +108,8 @@ static void socket_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kin static mp_uint_t socket_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errcode) { mp_obj_socket_t *o = MP_OBJ_TO_PTR(o_in); - ssize_t r; - MP_HAL_RETRY_SYSCALL(r, read(o->fd, buf, size), { + socket_size_t r; + MP_HAL_RETRY_SYSCALL(r, read_socket(o->fd, buf, size), { // On blocking socket, we get EAGAIN in case SO_RCVTIMEO/SO_SNDTIMEO // timed out, and need to convert that to ETIMEDOUT. if (err == EAGAIN && o->blocking) { @@ -114,8 +124,8 @@ static mp_uint_t socket_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errc static mp_uint_t socket_write(mp_obj_t o_in, const void *buf, mp_uint_t size, int *errcode) { mp_obj_socket_t *o = MP_OBJ_TO_PTR(o_in); - ssize_t r; - MP_HAL_RETRY_SYSCALL(r, write(o->fd, buf, size), { + socket_size_t r; + MP_HAL_RETRY_SYSCALL(r, write_socket(o->fd, buf, size), { // On blocking socket, we get EAGAIN in case SO_RCVTIMEO/SO_SNDTIMEO // timed out, and need to convert that to ETIMEDOUT. if (err == EAGAIN && o->blocking) { @@ -141,7 +151,7 @@ static mp_uint_t socket_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, i // file descriptor. If you're interested to catch I/O errors before // closing fd, fsync() it. MP_THREAD_GIL_EXIT(); - close(self->fd); + close_socket(self->fd); MP_THREAD_GIL_ENTER(); return 0; @@ -204,9 +214,9 @@ static mp_obj_t socket_connect(mp_obj_t self_in, mp_obj_t addr_in) { int r = connect(self->fd, (const struct sockaddr *)bufinfo.buf, bufinfo.len); MP_THREAD_GIL_ENTER(); if (r == -1) { - int err = errno; + int err = socket_errno; if (self->blocking) { - if (err == EINTR) { + if (err == socket_eintr) { mp_handle_pending(true); continue; } @@ -229,7 +239,7 @@ static mp_obj_t socket_bind(mp_obj_t self_in, mp_obj_t addr_in) { MP_THREAD_GIL_EXIT(); int r = bind(self->fd, (const struct sockaddr *)bufinfo.buf, bufinfo.len); MP_THREAD_GIL_ENTER(); - RAISE_ERRNO(r, errno); + RAISE_ERRNO(r, socket_errno); return mp_const_none; } static MP_DEFINE_CONST_FUN_OBJ_2(socket_bind_obj, socket_bind); @@ -247,7 +257,7 @@ static mp_obj_t socket_listen(size_t n_args, const mp_obj_t *args) { MP_THREAD_GIL_EXIT(); int r = listen(self->fd, backlog); MP_THREAD_GIL_ENTER(); - RAISE_ERRNO(r, errno); + RAISE_ERRNO(r, socket_errno); return mp_const_none; } static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(socket_listen_obj, 1, 2, socket_listen); @@ -257,8 +267,8 @@ static mp_obj_t socket_accept(mp_obj_t self_in) { // sockaddr_storage isn't stack-friendly (129 bytes or so) // struct sockaddr_storage addr; byte addr[32]; - socklen_t addr_len = sizeof(addr); - int fd; + sock_len_t addr_len = sizeof(addr); + socket_t fd; MP_HAL_RETRY_SYSCALL(fd, accept(self->fd, (struct sockaddr *)&addr, &addr_len), { // EAGAIN on a blocking socket means the operation timed out if (self->blocking && err == EAGAIN) { @@ -288,7 +298,7 @@ static mp_obj_t socket_recv(size_t n_args, const mp_obj_t *args) { } byte *buf = m_new(byte, sz); - ssize_t out_sz; + socket_size_t out_sz; MP_HAL_RETRY_SYSCALL(out_sz, recv(self->fd, buf, sz, flags), mp_raise_OSError(err)); mp_obj_t ret = mp_obj_new_str_of_type(&mp_type_bytes, buf, out_sz); m_del(char, buf, sz); @@ -306,10 +316,10 @@ static mp_obj_t socket_recvfrom(size_t n_args, const mp_obj_t *args) { } struct sockaddr_storage addr; - socklen_t addr_len = sizeof(addr); + sock_len_t addr_len = sizeof(addr); byte *buf = m_new(byte, sz); - ssize_t out_sz; + socket_size_t out_sz; MP_HAL_RETRY_SYSCALL(out_sz, recvfrom(self->fd, buf, sz, flags, (struct sockaddr *)&addr, &addr_len), mp_raise_OSError(err)); mp_obj_t buf_o = mp_obj_new_str_of_type(&mp_type_bytes, buf, out_sz); @@ -336,7 +346,7 @@ static mp_obj_t socket_send(size_t n_args, const mp_obj_t *args) { mp_buffer_info_t bufinfo; mp_get_buffer_raise(args[1], &bufinfo, MP_BUFFER_READ); - ssize_t out_sz; + socket_size_t out_sz; MP_HAL_RETRY_SYSCALL(out_sz, send(self->fd, bufinfo.buf, bufinfo.len, flags), mp_raise_OSError(err)); return MP_OBJ_NEW_SMALL_INT(out_sz); @@ -356,7 +366,7 @@ static mp_obj_t socket_sendto(size_t n_args, const mp_obj_t *args) { mp_buffer_info_t bufinfo, addr_bi; mp_get_buffer_raise(args[1], &bufinfo, MP_BUFFER_READ); mp_get_buffer_raise(dst_addr, &addr_bi, MP_BUFFER_READ); - ssize_t out_sz; + socket_size_t out_sz; MP_HAL_RETRY_SYSCALL(out_sz, sendto(self->fd, bufinfo.buf, bufinfo.len, flags, (struct sockaddr *)addr_bi.buf, addr_bi.len), mp_raise_OSError(err)); return MP_OBJ_NEW_SMALL_INT(out_sz); @@ -370,7 +380,7 @@ static mp_obj_t socket_setsockopt(size_t n_args, const mp_obj_t *args) { int option = mp_obj_get_int(args[2]); const void *optval; - socklen_t optlen; + sock_len_t optlen; int val; if (mp_obj_is_int(args[3])) { val = mp_obj_int_get_truncated(args[3]); @@ -385,7 +395,7 @@ static mp_obj_t socket_setsockopt(size_t n_args, const mp_obj_t *args) { MP_THREAD_GIL_EXIT(); int r = setsockopt(self->fd, level, option, optval, optlen); MP_THREAD_GIL_ENTER(); - RAISE_ERRNO(r, errno); + RAISE_ERRNO(r, socket_errno); return mp_const_none; } static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(socket_setsockopt_obj, 4, 4, socket_setsockopt); @@ -397,7 +407,7 @@ static mp_obj_t socket_setblocking(mp_obj_t self_in, mp_obj_t flag_in) { int flags = fcntl(self->fd, F_GETFL, 0); if (flags == -1) { MP_THREAD_GIL_ENTER(); - RAISE_ERRNO(flags, errno); + RAISE_ERRNO(flags, socket_errno); } if (val) { flags &= ~O_NONBLOCK; @@ -406,7 +416,7 @@ static mp_obj_t socket_setblocking(mp_obj_t self_in, mp_obj_t flag_in) { } flags = fcntl(self->fd, F_SETFL, flags); MP_THREAD_GIL_ENTER(); - RAISE_ERRNO(flags, errno); + RAISE_ERRNO(flags, socket_errno); self->blocking = val; return mp_const_none; } @@ -442,11 +452,11 @@ static mp_obj_t socket_settimeout(mp_obj_t self_in, mp_obj_t timeout_in) { r = setsockopt(self->fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(struct timeval)); if (r == -1) { MP_THREAD_GIL_ENTER(); - RAISE_ERRNO(r, errno); + RAISE_ERRNO(r, socket_errno); } r = setsockopt(self->fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(struct timeval)); MP_THREAD_GIL_ENTER(); - RAISE_ERRNO(r, errno); + RAISE_ERRNO(r, socket_errno); } if (self->blocking != new_blocking) { @@ -491,9 +501,9 @@ static mp_obj_t socket_make_new(const mp_obj_type_t *type_in, size_t n_args, siz } MP_THREAD_GIL_EXIT(); - int fd = socket(family, type, proto); + socket_t fd = socket(family, type, proto); MP_THREAD_GIL_ENTER(); - RAISE_ERRNO(fd, errno); + RAISE_ERRNO(fd, socket_errno); return MP_OBJ_FROM_PTR(socket_new(fd)); } @@ -541,7 +551,7 @@ static mp_obj_t mod_socket_inet_pton(mp_obj_t family_in, mp_obj_t addr_in) { int family = mp_obj_get_int(family_in); byte binaddr[BINADDR_MAX_LEN]; int r = inet_pton(family, mp_obj_str_get_str(addr_in), binaddr); - RAISE_ERRNO(r, errno); + RAISE_ERRNO(r, socket_errno); if (r == 0) { mp_raise_OSError(MP_EINVAL); } @@ -565,7 +575,7 @@ static mp_obj_t mod_socket_inet_ntop(mp_obj_t family_in, mp_obj_t binaddr_in) { vstr_t vstr; vstr_init_len(&vstr, family == AF_INET ? INET_ADDRSTRLEN : INET6_ADDRSTRLEN); if (inet_ntop(family, bufinfo.buf, vstr.buf, vstr.len) == NULL) { - mp_raise_OSError(errno); + mp_raise_OSError(socket_errno); } vstr.len = strlen(vstr.buf); return mp_obj_new_str_from_utf8_vstr(&vstr); From fd60536cb0b7452a092617a892162f30a17553bd Mon Sep 17 00:00:00 2001 From: stijn Date: Tue, 24 Oct 2023 15:26:00 +0200 Subject: [PATCH 4/9] unix/modsocket: Remove obsolete comment. Signed-off-by: stijn --- ports/unix/modsocket.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/ports/unix/modsocket.c b/ports/unix/modsocket.c index 22eb6496f9bfd..c4306c049e619 100644 --- a/ports/unix/modsocket.c +++ b/ports/unix/modsocket.c @@ -77,8 +77,6 @@ typedef ssize_t socket_size_t; should be add to separate modules (C or Python level). */ -// This type must "inherit" from mp_obj_fdfile_t, i.e. matching subset of -// fields should have the same layout. typedef struct _mp_obj_socket_t { mp_obj_base_t base; socket_t fd; From fc6a12153bd98665a126572a01c894c4267a34b8 Mon Sep 17 00:00:00 2001 From: stijn Date: Wed, 25 Oct 2023 13:42:52 +0200 Subject: [PATCH 5/9] windows: Implement socket module. Signed-off-by: stijn --- ports/unix/modsocket.c | 182 +++++++++++++++++++++++++++++- ports/windows/Makefile | 1 + ports/windows/init.c | 12 ++ ports/windows/micropython.vcxproj | 1 + ports/windows/mpconfigport.h | 1 + tests/run-tests.py | 6 + 6 files changed, 197 insertions(+), 6 deletions(-) diff --git a/ports/unix/modsocket.c b/ports/unix/modsocket.c index c4306c049e619..cb067dc622108 100644 --- a/ports/unix/modsocket.c +++ b/ports/unix/modsocket.c @@ -36,10 +36,26 @@ #include #include #include +#ifdef _WIN32 +// To get inet_pton and inet_ntop. +#ifdef __MINGW32__ +#ifdef _WIN32_WINNT +#undef _WIN32_WINNT +#endif +#define _WIN32_WINNT 0x0600 +#endif +#include +#include +#ifndef __MINGW32__ +#pragma comment(lib, "Ws2_32.lib") +#endif +#else #include #include #include #include +#include +#endif #include #include @@ -51,8 +67,97 @@ #include "py/mphal.h" #include "py/mpthread.h" #include "extmod/vfs.h" -#include +#ifdef _WIN32 +// Some extra info regarding the windows-specific code: +// - like CPython, most of the error codes raised as OSError will be the WSA error codes, not errno. +// - places where EAGAIN is turned into MP_ETIMEOUT are generally not needed On windows since the +// socket calls already return WSAETIMEDOUT. +// - makefile() and poll functionality are not implemented. + +typedef int sock_len_t; +typedef int socket_size_t; +typedef SOCKET socket_t; + +// Just to make sure since we rely on this. +#if SOCKET_ERROR != -1 +#error socket functions must return -1 for errors +#endif +#if NO_ERROR != 0 +#error socket functions must return 0 for no errors +#endif + +#define socket_errno WSAGetLastError() +#define socket_eintr WSAEINTR +#define read_socket(fd, buf, size) recv(fd, buf, size, 0) +#define write_socket(fd, buf, size) send(fd, buf, size, 0) +#define close_socket(fd) closesocket(fd) + +void wsa_startup() { + WSADATA wsaData; + (void)WSAStartup(MAKEWORD(1, 1), &wsaData); +} + +void wsa_cleanup() { + (void)WSACleanup(); +} + +// Socket calls are syscalls but with WSA error codes, not errno. +#ifdef MP_HAL_RETRY_SYSCALL +#undef MP_HAL_RETRY_SYSCALL +#endif + +#define MP_HAL_RETRY_SYSCALL(ret, syscall, raise) { \ + for (;;) { \ + MP_THREAD_GIL_EXIT(); \ + ret = syscall; \ + MP_THREAD_GIL_ENTER(); \ + if (ret == -1) { \ + int err = WSAGetLastError(); \ + if (err == WSAEINTR) { \ + mp_handle_pending(true); \ + continue; \ + } \ + raise; \ + } \ + break; \ + } \ +} + +// Get SO_RCVTIMEO or SO_SNDTIMEO values. +DWORD get_socket_timeout(socket_t sock, bool read_or_write) { + MP_THREAD_GIL_EXIT(); + DWORD timeout = 0; + int opt_len = sizeof(timeout); + const int opt_name = read_or_write ? SO_RCVTIMEO : SO_SNDTIMEO; + const int r = getsockopt(sock, SOL_SOCKET, opt_name, (char *)&timeout, &opt_len); + MP_THREAD_GIL_ENTER(); + RAISE_ERRNO(r, WSAGetLastError()); + return timeout; +} + +// Perform select() call, raising timeout error if timed out. +void select_on_socket(socket_t sock, bool read_or_write, DWORD timeout) { + fd_set fdset; + FD_ZERO(&fdset); + FD_SET(sock, &fdset); + struct timeval tv; + tv.tv_sec = timeout / 1000; + tv.tv_usec = (timeout * 1000) % 1000; + int r; + fd_set *read_fd = read_or_write ? &fdset : NULL; + fd_set *write_fd = read_or_write ? NULL : &fdset; + // First argument is ignored: "The nfds parameter is included only for + // compatibility with Berkeley sockets". + MP_HAL_RETRY_SYSCALL(r, select(1, read_fd, write_fd, NULL, &tv), { + // r < 0 is error, r == 0 is timeout, r > 0 = no timeout. + RAISE_ERRNO(r, WSAGetLastError()); + }); + if (r == 0) { + mp_raise_OSError(MP_ETIMEDOUT); + } +} +#else typedef socklen_t sock_len_t; typedef int socket_t; typedef ssize_t socket_size_t; @@ -63,6 +168,9 @@ typedef ssize_t socket_size_t; #define write_socket write #define close_socket close +#define initialize_socket_system() {} +#endif + /* The idea of this module is to implement reasonable minimum of socket-related functions to write typical clients and servers. @@ -153,6 +261,7 @@ static mp_uint_t socket_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, i MP_THREAD_GIL_ENTER(); return 0; + #ifndef _WIN32 case MP_STREAM_GET_FILENO: return self->fd; @@ -187,6 +296,7 @@ static mp_uint_t socket_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, i return ret; } #endif + #endif default: *errcode = MP_EINVAL; @@ -194,11 +304,13 @@ static mp_uint_t socket_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, i } } +#ifndef _WIN32 static mp_obj_t socket_fileno(mp_obj_t self_in) { mp_obj_socket_t *self = MP_OBJ_TO_PTR(self_in); return MP_OBJ_NEW_SMALL_INT(self->fd); } static MP_DEFINE_CONST_FUN_OBJ_1(socket_fileno_obj, socket_fileno); +#endif static mp_obj_t socket_connect(mp_obj_t self_in, mp_obj_t addr_in) { mp_obj_socket_t *self = MP_OBJ_TO_PTR(self_in); @@ -208,6 +320,15 @@ static mp_obj_t socket_connect(mp_obj_t self_in, mp_obj_t addr_in) { // special case of PEP 475 to retry only if blocking so we can't use // MP_HAL_RETRY_SYSCALL() here for (;;) { + #ifdef _WIN32 + // The connect() call has no timeout so implement it by first calling select(). + // In theory there's a race condition here between select() returning and connect() + // being called, in practice doesn't seem worth fixing. + const DWORD timeout = get_socket_timeout(self->fd, true); + if (timeout > 0 && self->blocking) { + select_on_socket(self->fd, true, timeout); + } + #endif MP_THREAD_GIL_EXIT(); int r = connect(self->fd, (const struct sockaddr *)bufinfo.buf, bufinfo.len); MP_THREAD_GIL_ENTER(); @@ -267,6 +388,26 @@ static mp_obj_t socket_accept(mp_obj_t self_in) { byte addr[32]; sock_len_t addr_len = sizeof(addr); socket_t fd; + #ifdef _WIN32 + // The accept() call has no timeout so manually implement it by first calling select(). + const DWORD timeout = get_socket_timeout(self->fd, true); + if (timeout > 0 && self->blocking) { + select_on_socket(self->fd, true, timeout); + } + // The accept() call returns a socket, not an int, so cannot use MP_HAL_RETRY_SYSCALL. + for (;;) { + fd = accept(self->fd, (struct sockaddr *)&addr, &addr_len); + if (fd == INVALID_SOCKET) { + const int err = WSAGetLastError(); + if (err == WSAEINTR) { + mp_handle_pending(1); + continue; + } + mp_raise_OSError(err); + } + break; + } + #else MP_HAL_RETRY_SYSCALL(fd, accept(self->fd, (struct sockaddr *)&addr, &addr_len), { // EAGAIN on a blocking socket means the operation timed out if (self->blocking && err == EAGAIN) { @@ -274,6 +415,7 @@ static mp_obj_t socket_accept(mp_obj_t self_in) { } mp_raise_OSError(err); }); + #endif mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(2, NULL)); t->items[0] = MP_OBJ_FROM_PTR(socket_new(fd)); @@ -297,7 +439,7 @@ static mp_obj_t socket_recv(size_t n_args, const mp_obj_t *args) { byte *buf = m_new(byte, sz); socket_size_t out_sz; - MP_HAL_RETRY_SYSCALL(out_sz, recv(self->fd, buf, sz, flags), mp_raise_OSError(err)); + MP_HAL_RETRY_SYSCALL(out_sz, recv(self->fd, (char *)buf, sz, flags), mp_raise_OSError(err)); mp_obj_t ret = mp_obj_new_str_of_type(&mp_type_bytes, buf, out_sz); m_del(char, buf, sz); return ret; @@ -318,7 +460,7 @@ static mp_obj_t socket_recvfrom(size_t n_args, const mp_obj_t *args) { byte *buf = m_new(byte, sz); socket_size_t out_sz; - MP_HAL_RETRY_SYSCALL(out_sz, recvfrom(self->fd, buf, sz, flags, (struct sockaddr *)&addr, &addr_len), + MP_HAL_RETRY_SYSCALL(out_sz, recvfrom(self->fd, (char *)buf, sz, flags, (struct sockaddr *)&addr, &addr_len), mp_raise_OSError(err)); mp_obj_t buf_o = mp_obj_new_str_of_type(&mp_type_bytes, buf, out_sz); m_del(char, buf, sz); @@ -402,6 +544,11 @@ static mp_obj_t socket_setblocking(mp_obj_t self_in, mp_obj_t flag_in) { mp_obj_socket_t *self = MP_OBJ_TO_PTR(self_in); int val = mp_obj_is_true(flag_in); MP_THREAD_GIL_EXIT(); + #ifdef _WIN32 + u_long mode = val ? 0 : 1; + // Called 'flags' but here it's just the return value. + const int flags = ioctlsocket(self->fd, FIONBIO, &mode); + #else int flags = fcntl(self->fd, F_GETFL, 0); if (flags == -1) { MP_THREAD_GIL_ENTER(); @@ -413,6 +560,7 @@ static mp_obj_t socket_setblocking(mp_obj_t self_in, mp_obj_t flag_in) { flags |= O_NONBLOCK; } flags = fcntl(self->fd, F_SETFL, flags); + #endif MP_THREAD_GIL_ENTER(); RAISE_ERRNO(flags, socket_errno); self->blocking = val; @@ -423,6 +571,14 @@ static MP_DEFINE_CONST_FUN_OBJ_2(socket_setblocking_obj, socket_setblocking); static mp_obj_t socket_settimeout(mp_obj_t self_in, mp_obj_t timeout_in) { mp_obj_socket_t *self = MP_OBJ_TO_PTR(self_in); struct timeval tv = {0, }; + #ifdef _WIN32 + DWORD time_val = 0; + const char *ptv = (const char *)&time_val; + const int opt_len = sizeof(DWORD); + #else + struct timeval *ptv = &tv; + const int opt_len = sizeof(struct timeval); + #endif bool new_blocking = true; // Timeout of None means no timeout, which in POSIX is signified with 0 timeout, @@ -436,6 +592,9 @@ static mp_obj_t socket_settimeout(mp_obj_t self_in, mp_obj_t timeout_in) { #else tv.tv_sec = mp_obj_get_int(timeout_in); #endif + #ifdef _WIN32 + time_val = tv.tv_usec / 1000 + tv.tv_sec * 1000; + #endif // For SO_RCVTIMEO/SO_SNDTIMEO, zero timeout means infinity, but // for Python API it means non-blocking. @@ -447,12 +606,12 @@ static mp_obj_t socket_settimeout(mp_obj_t self_in, mp_obj_t timeout_in) { if (new_blocking) { int r; MP_THREAD_GIL_EXIT(); - r = setsockopt(self->fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(struct timeval)); + r = setsockopt(self->fd, SOL_SOCKET, SO_RCVTIMEO, ptv, opt_len); if (r == -1) { MP_THREAD_GIL_ENTER(); RAISE_ERRNO(r, socket_errno); } - r = setsockopt(self->fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(struct timeval)); + r = setsockopt(self->fd, SOL_SOCKET, SO_SNDTIMEO, ptv, opt_len); MP_THREAD_GIL_ENTER(); RAISE_ERRNO(r, socket_errno); } @@ -465,6 +624,7 @@ static mp_obj_t socket_settimeout(mp_obj_t self_in, mp_obj_t timeout_in) { } static MP_DEFINE_CONST_FUN_OBJ_2(socket_settimeout_obj, socket_settimeout); +#ifndef _WIN32 static mp_obj_t socket_makefile(size_t n_args, const mp_obj_t *args) { // TODO: CPython explicitly says that closing returned object doesn't close // the original socket (Python2 at all says that fd is dup()ed). But we @@ -476,6 +636,7 @@ static mp_obj_t socket_makefile(size_t n_args, const mp_obj_t *args) { return mp_vfs_open(n_args, new_args, (mp_map_t *)&mp_const_empty_map); } static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(socket_makefile_obj, 1, 3, socket_makefile); +#endif static mp_obj_t socket_make_new(const mp_obj_type_t *type_in, size_t n_args, size_t n_kw, const mp_obj_t *args) { (void)type_in; @@ -500,14 +661,21 @@ static mp_obj_t socket_make_new(const mp_obj_type_t *type_in, size_t n_args, siz MP_THREAD_GIL_EXIT(); socket_t fd = socket(family, type, proto); + #ifdef _WIN32 + const int err = fd == INVALID_SOCKET ? -1 : 0; + #else + const int err = fd; + #endif MP_THREAD_GIL_ENTER(); - RAISE_ERRNO(fd, socket_errno); + RAISE_ERRNO(err, socket_errno); return MP_OBJ_FROM_PTR(socket_new(fd)); } static const mp_rom_map_elem_t socket_locals_dict_table[] = { + #ifndef _WIN32 { MP_ROM_QSTR(MP_QSTR_fileno), MP_ROM_PTR(&socket_fileno_obj) }, { MP_ROM_QSTR(MP_QSTR_makefile), MP_ROM_PTR(&socket_makefile_obj) }, + #endif { MP_ROM_QSTR(MP_QSTR_read), MP_ROM_PTR(&mp_stream_read_obj) }, { MP_ROM_QSTR(MP_QSTR_readinto), MP_ROM_PTR(&mp_stream_readinto_obj) }, { MP_ROM_QSTR(MP_QSTR_readline), MP_ROM_PTR(&mp_stream_unbuffered_readline_obj) }, @@ -708,7 +876,9 @@ static const mp_rom_map_elem_t mp_module_socket_globals_table[] = { C(SOCK_RAW), C(MSG_DONTROUTE), + #ifndef _WIN32 C(MSG_DONTWAIT), + #endif C(SOL_SOCKET), C(SO_BROADCAST), diff --git a/ports/windows/Makefile b/ports/windows/Makefile index cf0a927014b1a..4f789417437bd 100644 --- a/ports/windows/Makefile +++ b/ports/windows/Makefile @@ -56,6 +56,7 @@ SRC_C = \ shared/runtime/gchelper_generic.c \ ports/unix/main.c \ ports/unix/input.c \ + ports/unix/modsocket.c \ ports/unix/gccollect.c \ windows_mphal.c \ realpath.c \ diff --git a/ports/windows/init.c b/ports/windows/init.c index 87d581c30493e..fba8d2c4293f3 100644 --- a/ports/windows/init.c +++ b/ports/windows/init.c @@ -24,6 +24,7 @@ * THE SOFTWARE. */ +#include "py/mpconfig.h" #include #include #include @@ -34,6 +35,11 @@ extern BOOL WINAPI console_sighandler(DWORD evt); +#if MICROPY_PY_SOCKET +extern void wsa_startup(); +extern void wsa_cleanup(); +#endif + #ifdef _MSC_VER void invalid_param_handler(const wchar_t *expr, const wchar_t *fun, const wchar_t *file, unsigned int line, uintptr_t p) { } @@ -61,8 +67,14 @@ void init() { _set_output_format(_TWO_DIGIT_EXPONENT); #endif set_fmode_binary(); + #if MICROPY_PY_SOCKET + wsa_startup(); + #endif } void deinit() { SetConsoleCtrlHandler(console_sighandler, FALSE); + #if MICROPY_PY_SOCKET + wsa_cleanup(); + #endif } diff --git a/ports/windows/micropython.vcxproj b/ports/windows/micropython.vcxproj index 9326f3f4cde18..bd647f6736cf1 100644 --- a/ports/windows/micropython.vcxproj +++ b/ports/windows/micropython.vcxproj @@ -94,6 +94,7 @@ + diff --git a/ports/windows/mpconfigport.h b/ports/windows/mpconfigport.h index fabc9072d6c70..ef6d5d6479364 100644 --- a/ports/windows/mpconfigport.h +++ b/ports/windows/mpconfigport.h @@ -107,6 +107,7 @@ #define MICROPY_PY_ARRAY_SLICE_ASSIGN (1) #define MICROPY_PY_BUILTINS_SLICE_ATTRS (1) #define MICROPY_PY_SYS_PATH_ARGV_DEFAULTS (0) +#define MICROPY_PY_SOCKET (1) #define MICROPY_PY_SYS_EXIT (1) #define MICROPY_PY_SYS_ATEXIT (1) #define MICROPY_PY_SYS_PLATFORM "win32" diff --git a/tests/run-tests.py b/tests/run-tests.py index 83344714c0375..a5e1ff2b5f8ec 100755 --- a/tests/run-tests.py +++ b/tests/run-tests.py @@ -711,6 +711,12 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): if not sysconfig.get_platform().startswith("mingw"): # Works but CPython uses '\' path separator skip_tests.add("import/import_file.py") + # These are usually skipped because either the socket or select module is missing, + # but variants can have them both and select support for sockets is not implemented. + skip_tests.add("extmod/select_ipoll.py") + skip_tests.add("extmod/select_poll_basic.py") + skip_tests.add("extmod/select_poll_custom.py") + skip_tests.add("extmod/select_poll_udp.py") # Some tests are known to fail with native emitter # Remove them from the below when they work From 1580f442917079370bbdb9e9282a7529f048a7fc Mon Sep 17 00:00:00 2001 From: stijn Date: Mon, 25 Mar 2024 14:33:50 +0100 Subject: [PATCH 6/9] windows/msvc: Add a build target for updating submodules. This is more consistent with the Makefile/CMake-based builds. Signed-off-by: stijn --- .github/workflows/ports_windows.yml | 2 +- ports/windows/README.md | 1 + ports/windows/micropython.vcxproj | 2 +- ports/windows/msvc/common.props | 12 ++++++++++++ ports/windows/variants/dev/mpconfigvariant.props | 3 +++ .../windows/variants/standard/mpconfigvariant.props | 3 +++ 6 files changed, 21 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ports_windows.yml b/.github/workflows/ports_windows.yml index 91a3192a10a48..fd108df7238f6 100644 --- a/.github/workflows/ports_windows.yml +++ b/.github/workflows/ports_windows.yml @@ -63,7 +63,7 @@ jobs: - name: Build mpy-cross.exe run: msbuild mpy-cross\mpy-cross.vcxproj -maxcpucount -property:Configuration=${{ matrix.configuration }} -property:Platform=${{ matrix.platform }} - name: Update submodules - run: git submodule update --init lib/micropython-lib + run: msbuild ports\windows\micropython.vcxproj -target:UpdateSubmodules -property:Configuration=${{ matrix.configuration }} -property:Platform=${{ matrix.platform }} -property:PyVariant=${{ matrix.variant }} - name: Build micropython.exe run: msbuild ports\windows\micropython.vcxproj -maxcpucount -property:Configuration=${{ matrix.configuration }} -property:Platform=${{ matrix.platform }} -property:PyVariant=${{ matrix.variant }} - name: Get micropython.exe path diff --git a/ports/windows/README.md b/ports/windows/README.md index 57ec0e19149be..4906f13486274 100644 --- a/ports/windows/README.md +++ b/ports/windows/README.md @@ -69,6 +69,7 @@ In the IDE, open `micropython-cross.vcxproj` and `micropython.vcxproj` and build To build from the command line: msbuild ../../mpy-cross/mpy-cross.vcxproj + msbuild micropython.vcxproj /t:UpdateSubmodules msbuild micropython.vcxproj __Variants__ diff --git a/ports/windows/micropython.vcxproj b/ports/windows/micropython.vcxproj index bd647f6736cf1..462821a6a0869 100644 --- a/ports/windows/micropython.vcxproj +++ b/ports/windows/micropython.vcxproj @@ -108,7 +108,7 @@ - + diff --git a/ports/windows/msvc/common.props b/ports/windows/msvc/common.props index 55c9c934f22ea..ecb6641371da0 100644 --- a/ports/windows/msvc/common.props +++ b/ports/windows/msvc/common.props @@ -62,4 +62,16 @@ + + + + + + + + + + diff --git a/ports/windows/variants/dev/mpconfigvariant.props b/ports/windows/variants/dev/mpconfigvariant.props index be1f1ae51e5d2..be1ee1eb29fbb 100644 --- a/ports/windows/variants/dev/mpconfigvariant.props +++ b/ports/windows/variants/dev/mpconfigvariant.props @@ -8,4 +8,7 @@ %(PreprocessorDefinitions);MICROPY_ROM_TEXT_COMPRESSION=1 + + + diff --git a/ports/windows/variants/standard/mpconfigvariant.props b/ports/windows/variants/standard/mpconfigvariant.props index ea8bf0bc827bb..d9279940871ce 100644 --- a/ports/windows/variants/standard/mpconfigvariant.props +++ b/ports/windows/variants/standard/mpconfigvariant.props @@ -3,4 +3,7 @@ $(PyWinDir)\variants\manifest.py + + + From 8275f32f01436df6e7091eea162328e7b48d99ad Mon Sep 17 00:00:00 2001 From: stijn Date: Mon, 25 Mar 2024 14:38:08 +0100 Subject: [PATCH 7/9] windows/msvc: Introduce an msbuild variable for third-party sources. This is consistent with extmod.mk's split between SRC_EXTMOD_C and SRC_THIRDPARTY_C, the latter specifying source files which get compiled but not included in qstr generation. Signed-off-by: stijn --- ports/windows/micropython.vcxproj | 1 + ports/windows/msvc/genhdr.targets | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ports/windows/micropython.vcxproj b/ports/windows/micropython.vcxproj index 462821a6a0869..bbdb0a1b7f078 100644 --- a/ports/windows/micropython.vcxproj +++ b/ports/windows/micropython.vcxproj @@ -87,6 +87,7 @@ + diff --git a/ports/windows/msvc/genhdr.targets b/ports/windows/msvc/genhdr.targets index 7bd8364a9ed08..00afeed6f09f5 100644 --- a/ports/windows/msvc/genhdr.targets +++ b/ports/windows/msvc/genhdr.targets @@ -66,7 +66,7 @@ using(var outFile = System.IO.File.CreateText(OutputFile)) { - + False $([System.String]::new('%(FullPath)').Replace('$(PyBaseDir)', '$(DestDir)qstr\')) From 249b90396cc68f6d1251d9547664ac8e1335d732 Mon Sep 17 00:00:00 2001 From: stijn Date: Mon, 25 Mar 2024 14:32:57 +0100 Subject: [PATCH 8/9] windows/msvc: Fix preprocessing command for quoted definitions. Supports preprocessor definitions like /Dval="quotedvalue" by turning that into /Dval=\"quotedvalue\". Signed-off-by: stijn --- ports/windows/msvc/genhdr.targets | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ports/windows/msvc/genhdr.targets b/ports/windows/msvc/genhdr.targets index 00afeed6f09f5..526b1a0a712e8 100644 --- a/ports/windows/msvc/genhdr.targets +++ b/ports/windows/msvc/genhdr.targets @@ -82,7 +82,9 @@ using(var outFile = System.IO.File.CreateText(OutputFile)) { - $(PyClTool) /nologo /I@(PyIncDirs, ' /I') /D@(PreProcDefs, ' /D') + + /D@(PreProcDefs, ' /D') + $(PyClTool) /nologo /I@(PyIncDirs, ' /I') $([System.String]::Copy('$(UnescapedPreProcDefs)').Replace('"','\"')) @(QstrDependencies->AnyHaveMetadataValue('Changed', 'True')) @(PyQstrSourceFiles->AnyHaveMetadataValue('Changed', 'True')) From 1f4266dd25368d7e849910d34455f53a27091f1b Mon Sep 17 00:00:00 2001 From: stijn Date: Wed, 25 Oct 2023 16:12:39 +0200 Subject: [PATCH 9/9] windows/msvc: Add ssl module using mbedtls in standard variant. Signed-off-by: stijn --- .../variants/standard/mpconfigvariant.h | 4 + .../variants/standard/mpconfigvariant.props | 82 +++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/ports/windows/variants/standard/mpconfigvariant.h b/ports/windows/variants/standard/mpconfigvariant.h index 3cdcfa8e9b8c9..0e0f4cc0ae27c 100644 --- a/ports/windows/variants/standard/mpconfigvariant.h +++ b/ports/windows/variants/standard/mpconfigvariant.h @@ -26,3 +26,7 @@ #define MICROPY_PY_BUILTINS_HELP (1) #define MICROPY_PY_BUILTINS_HELP_MODULES (1) +#ifndef __MINGW32__ +#define MICROPY_PY_SSL (1) +#define MICROPY_SSL_MBEDTLS (1) +#endif diff --git a/ports/windows/variants/standard/mpconfigvariant.props b/ports/windows/variants/standard/mpconfigvariant.props index d9279940871ce..2577b346316e8 100644 --- a/ports/windows/variants/standard/mpconfigvariant.props +++ b/ports/windows/variants/standard/mpconfigvariant.props @@ -2,8 +2,90 @@ $(PyWinDir)\variants\manifest.py + $(PyIncDirs);$(PyBaseDir)lib\mbedtls\include + + + MBEDTLS_CONFIG_FILE="$(PyBaseDir)ports\unix\mbedtls\mbedtls_config_port.h";%(PreprocessorDefinitions) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +