fix event loop is already running bug on Linux

dlqqq · dlqqq · commit 9f5dc56aae1f · 2023-10-30T14:47:08.000-07:00
diff --git a/jupyter_scheduler/scheduler.py b/jupyter_scheduler/scheduler.py
@@ -1,7 +1,7 @@
+import multiprocessing as mp
 import os
 import random
 import shutil
-from multiprocessing import Process
 from typing import Dict, Optional, Type, Union
 
 import fsspec
@@ -403,7 +403,15 @@ def create_job(self, model: CreateJob) -> str:
             staging_paths = self.get_staging_paths(DescribeJob.from_orm(job))
             self.copy_input_file(model.input_uri, staging_paths["input"])
 
-            p = Process(
+            # The MP context forces new processes to not be forked on Linux.
+            # This is necessary because `asyncio.get_event_loop()` is bugged in
+            # forked processes in Python versions below 3.12. This method is
+            # called by `jupyter_core` by `nbconvert` in the default executor.
+            #
+            # See: https://github.com/python/cpython/issues/66285
+            # See also: https://github.com/jupyter/jupyter_core/pull/362
+            mp_ctx = mp.get_context("spawn")
+            p = mp_ctx.Process(
                 target=self.execution_manager_class(
                     job_id=job.job_id,
                     staging_paths=staging_paths,
