From 70b2484ed7e62993fbeb463399014bc4cf33623e Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 3 Oct 2023 13:47:48 +0200 Subject: [PATCH 1/6] Added Python 3.12 to test matrix (for common tests only, no tests of integrations) --- .github/workflows/test-common.yml | 2 +- tox.ini | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-common.yml b/.github/workflows/test-common.yml index 03117b7db1..7204c5d7d7 100644 --- a/.github/workflows/test-common.yml +++ b/.github/workflows/test-common.yml @@ -31,7 +31,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.5","3.6","3.7","3.8","3.9","3.10","3.11"] + python-version: ["3.5","3.6","3.7","3.8","3.9","3.10","3.11","3.12"] # python3.6 reached EOL and is no longer being supported on # new versions of hosted runners on Github Actions # ubuntu-20.04 is the last version that supported python3.6 diff --git a/tox.ini b/tox.ini index be4c5141f1..8650367378 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,7 @@ [tox] envlist = # === Common === - {py2.7,py3.5,py3.6,py3.7,py3.8,py3.9,py3.10,py3.11}-common + {py2.7,py3.5,py3.6,py3.7,py3.8,py3.9,py3.10,py3.11,py3.12}-common # === Integrations === # General format is {pythonversion}-{integrationname}-v{frameworkversion} @@ -187,7 +187,7 @@ deps = linters: werkzeug<2.3.0 # Common - {py3.6,py3.7,py3.8,py3.9,py3.10,py3.11}-common: pytest-asyncio + {py3.6,py3.7,py3.8,py3.9,py3.10,py3.11,py3.12}-common: pytest-asyncio # AIOHTTP aiohttp-v3.4: aiohttp>=3.4.0,<3.5.0 @@ -330,7 +330,7 @@ deps = # See https://stackoverflow.com/questions/51496550/runtime-warning-greenlet-greenlet-size-changed # for justification why greenlet is pinned here py3.5-gevent: greenlet==0.4.17 - {py2.7,py3.6,py3.7,py3.8,py3.9,py3.10,py3.11}-gevent: gevent>=22.10.0, <22.11.0 + {py2.7,py3.6,py3.7,py3.8,py3.9,py3.10,py3.11,py3.12}-gevent: gevent>=22.10.0, <22.11.0 # GQL gql: gql[all] @@ -566,6 +566,7 @@ basepython = py3.9: python3.9 py3.10: python3.10 py3.11: python3.11 + py3.12: python3.12 # Python version is pinned here because flake8 actually behaves differently # depending on which version is used. You can patch this out to point to @@ -592,7 +593,7 @@ commands = ; when loading tests in scenarios. In particular, django fails to ; load the settings from the test module. {py2.7}: python -m pytest --ignore-glob='*py3.py' -rsx -s --durations=5 -vvv {env:TESTPATH} {posargs} - {py3.5,py3.6,py3.7,py3.8,py3.9,py3.10,py3.11}: python -m pytest -rsx -s --durations=5 -vvv {env:TESTPATH} {posargs} + {py3.5,py3.6,py3.7,py3.8,py3.9,py3.10,py3.11,py3.12}: python -m pytest -rsx -s --durations=5 -vvv {env:TESTPATH} {posargs} [testenv:linters] commands = From 4c2cd8aeebf445aa495848308f3ace4ba645cb10 Mon Sep 17 00:00:00 2001 From: Ivana Kellyerova Date: Wed, 25 Oct 2023 13:26:35 +0200 Subject: [PATCH 2/6] Add taskName to COMMON_RECORD_ATTRS --- sentry_sdk/integrations/logging.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sentry_sdk/integrations/logging.py b/sentry_sdk/integrations/logging.py index 4162f90aef..895f09f780 100644 --- a/sentry_sdk/integrations/logging.py +++ b/sentry_sdk/integrations/logging.py @@ -130,6 +130,7 @@ class _BaseHandler(logging.Handler, object): "relativeCreated", "stack", "tags", + "taskName", "thread", "threadName", "stack_info", From b543541f838e0a5a90c011e637734d9b41b1b919 Mon Sep 17 00:00:00 2001 From: Ivana Kellyerova Date: Wed, 25 Oct 2023 16:35:22 +0200 Subject: [PATCH 3/6] Use main thread atexit if one is not available --- sentry_sdk/client.py | 8 +++++--- sentry_sdk/integrations/atexit.py | 2 +- sentry_sdk/transport.py | 4 ++++ sentry_sdk/worker.py | 24 ++++++++++++++++++------ 4 files changed, 28 insertions(+), 10 deletions(-) diff --git a/sentry_sdk/client.py b/sentry_sdk/client.py index 749ab23cfe..7749c76ae1 100644 --- a/sentry_sdk/client.py +++ b/sentry_sdk/client.py @@ -637,6 +637,7 @@ def close( self, timeout=None, # type: Optional[float] callback=None, # type: Optional[Callable[[int, float], None]] + shutdown=False, # type: bool ): # type: (...) -> None """ @@ -644,7 +645,7 @@ def close( semantics as :py:meth:`Client.flush`. """ if self.transport is not None: - self.flush(timeout=timeout, callback=callback) + self.flush(timeout=timeout, callback=callback, shutdown=shutdown) self.session_flusher.kill() if self.metrics_aggregator is not None: self.metrics_aggregator.kill() @@ -657,14 +658,15 @@ def flush( self, timeout=None, # type: Optional[float] callback=None, # type: Optional[Callable[[int, float], None]] + shutdown=False, # type: bool ): # type: (...) -> None """ Wait for the current events to be sent. :param timeout: Wait for at most `timeout` seconds. If no `timeout` is provided, the `shutdown_timeout` option value is used. - :param callback: Is invoked with the number of pending events and the configured timeout. + :param shutdown: `flush` has been invoked on interpreter shutdown. """ if self.transport is not None: if timeout is None: @@ -672,7 +674,7 @@ def flush( self.session_flusher.flush() if self.metrics_aggregator is not None: self.metrics_aggregator.flush() - self.transport.flush(timeout=timeout, callback=callback) + self.transport.flush(timeout=timeout, callback=callback, shutdown=shutdown) def __enter__(self): # type: () -> _Client diff --git a/sentry_sdk/integrations/atexit.py b/sentry_sdk/integrations/atexit.py index af70dd9fc9..00656f93ee 100644 --- a/sentry_sdk/integrations/atexit.py +++ b/sentry_sdk/integrations/atexit.py @@ -58,4 +58,4 @@ def _shutdown(): # If an integration is there, a client has to be there. client = hub.client # type: Any - client.close(callback=integration.callback) + client.close(callback=integration.callback, shutdown=True) diff --git a/sentry_sdk/transport.py b/sentry_sdk/transport.py index 4b12287ec9..9b8363ac64 100644 --- a/sentry_sdk/transport.py +++ b/sentry_sdk/transport.py @@ -544,10 +544,14 @@ def flush( self, timeout, # type: float callback=None, # type: Optional[Any] + shutdown=False, # type: bool ): # type: (...) -> None logger.debug("Flushing HTTP transport") + if shutdown: + self._worker.prepare_shutdown() + if timeout > 0: self._worker.submit(lambda: self._flush_client_reports(force=True)) self._worker.flush(timeout, callback) diff --git a/sentry_sdk/worker.py b/sentry_sdk/worker.py index 2fe81a8d70..518a3aa627 100644 --- a/sentry_sdk/worker.py +++ b/sentry_sdk/worker.py @@ -26,6 +26,7 @@ def __init__(self, queue_size=DEFAULT_QUEUE_SIZE): self._lock = threading.Lock() self._thread = None # type: Optional[threading.Thread] self._thread_for_pid = None # type: Optional[int] + self._can_start_threads = True # type: bool @property def is_alive(self): @@ -63,11 +64,15 @@ def start(self): # type: () -> None with self._lock: if not self.is_alive: - self._thread = threading.Thread( - target=self._target, name="raven-sentry.BackgroundWorker" - ) - self._thread.daemon = True - self._thread.start() + if self._can_start_threads: + self._thread = threading.Thread( + target=self._target, name="raven-sentry.BackgroundWorker" + ) + self._thread.daemon = True + self._thread.start() + else: + self._thread = threading.main_thread() + self._thread_for_pid = os.getpid() def kill(self): @@ -78,7 +83,7 @@ def kill(self): """ logger.debug("background worker got kill request") with self._lock: - if self._thread: + if self._thread and self._thread != threading.main_thread(): try: self._queue.put_nowait(_TERMINATOR) except FullError: @@ -135,3 +140,10 @@ def _target(self): finally: self._queue.task_done() sleep(0) + + def prepare_shutdown(self): + # If the Python 3.12+ interpreter is shutting down, trying to start a new + # thread throws a RuntimeError. If we're shutting down and the worker has + # no active thread, use the main thread instead of spawning a new one. + # See https://github.com/python/cpython/pull/104826 + self._can_start_threads = False From 244a548672b204a54bac0e06f9fa0f23e6302266 Mon Sep 17 00:00:00 2001 From: Ivana Kellyerova Date: Wed, 25 Oct 2023 19:49:30 +0200 Subject: [PATCH 4/6] add missing arg --- sentry_sdk/transport.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sentry_sdk/transport.py b/sentry_sdk/transport.py index 9b8363ac64..ea5f725542 100644 --- a/sentry_sdk/transport.py +++ b/sentry_sdk/transport.py @@ -87,6 +87,7 @@ def flush( self, timeout, # type: float callback=None, # type: Optional[Any] + shutdown=False, # type: bool ): # type: (...) -> None """Wait `timeout` seconds for the current events to be sent out.""" From a64d54576e0705d6a8c3cab349554fb52f19ed4d Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 31 Oct 2023 10:05:42 +0100 Subject: [PATCH 5/6] Handling main thread in Python 3.12 --- sentry_sdk/worker.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/sentry_sdk/worker.py b/sentry_sdk/worker.py index 518a3aa627..6f7dfcf51a 100644 --- a/sentry_sdk/worker.py +++ b/sentry_sdk/worker.py @@ -42,6 +42,21 @@ def _ensure_thread(self): if not self.is_alive: self.start() + def _get_main_thread(self): + # type: () -> Optional[threading.Thread] + main_thread = None + + try: + main_thread = threading.main_thread() + except AttributeError: + # Python 2.7 doesn't have threading.main_thread() + for thread in threading.enumerate(): + if isinstance(thread, threading._MainThread): + main_thread = thread + break + + return main_thread + def _timed_queue_join(self, timeout): # type: (float) -> bool deadline = time() + timeout @@ -71,7 +86,7 @@ def start(self): self._thread.daemon = True self._thread.start() else: - self._thread = threading.main_thread() + self._thread = self._get_main_thread() self._thread_for_pid = os.getpid() @@ -83,7 +98,7 @@ def kill(self): """ logger.debug("background worker got kill request") with self._lock: - if self._thread and self._thread != threading.main_thread(): + if self._thread and self._thread != self._get_main_thread(): try: self._queue.put_nowait(_TERMINATOR) except FullError: From 335a0b60fe2827ad7ff1ddabf1f35af14479301a Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 31 Oct 2023 10:10:14 +0100 Subject: [PATCH 6/6] Linting --- sentry_sdk/worker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sentry_sdk/worker.py b/sentry_sdk/worker.py index 6f7dfcf51a..8f6cdda5b4 100644 --- a/sentry_sdk/worker.py +++ b/sentry_sdk/worker.py @@ -51,7 +51,7 @@ def _get_main_thread(self): except AttributeError: # Python 2.7 doesn't have threading.main_thread() for thread in threading.enumerate(): - if isinstance(thread, threading._MainThread): + if isinstance(thread, threading._MainThread): # type: ignore[attr-defined] main_thread = thread break @@ -157,6 +157,7 @@ def _target(self): sleep(0) def prepare_shutdown(self): + # type: () -> None # If the Python 3.12+ interpreter is shutting down, trying to start a new # thread throws a RuntimeError. If we're shutting down and the worker has # no active thread, use the main thread instead of spawning a new one.