Parallel lib compilation(!?)

RustPython · coolreader18 · Mar 3, 2023 · Mar 3, 2023 · Mar 3, 2023 · Mar 3, 2023
commit dca98249734b22b11a78a301fdbe6e0242281188
diff --git a/derive-impl/src/compile_bytecode.rs b/derive-impl/src/compile_bytecode.rs
@@ -52,12 +52,13 @@
     LibPath(PathBuf),
 }
 
+#[derive(Clone)]
 struct CompiledModule {
     code: CodeObject,
     package: bool,
 }
 
-pub trait Compiler {
+pub trait Compiler: Sync {
     fn compile(
         &self,
         source: &str,
@@ -95,13 +96,15 @@
                 return Ok(vec![(module_name, module)]);
             }
         }
-        dir.modules
-            .into_iter()
-            .map(|(module_name, (path, package))| {
-                let code = Self::compile_file(&path, mode, &module_name, compiler)?;
-                Ok((module_name, CompiledModule { code, package }))
-            })
-            .collect()
+        let do_compile = |(module_name, (path, package)): (String, (PathBuf, _))| {
+            let code = Self::compile_file(&path, mode, &module_name, compiler)?;
+            Ok((module_name, CompiledModule { code, package }))
+        };
+        if dir.modules.len() > 32 {
+            par_map(dir.modules, do_compile).collect()
+        } else {
+            dir.modules.into_iter().map(do_compile).collect()
+        }
     }
 
     fn compile_file(
@@ -152,6 +155,32 @@
     }
 }
 
+fn par_map<T, U, I, F>(it: I, f: F) -> impl Iterator<Item = U>
+where
+    I: IntoIterator<Item = T, IntoIter: ExactSizeIterator + Send>,
+    F: Fn(T) -> U + Sync,
+    U: Send,
+{
+    let it = it.into_iter();
+    let mut out = Vec::from_iter(std::iter::repeat_with(|| None).take(it.len()));
+    let it = std::sync::Mutex::new(std::iter::zip(&mut out, it));
+    let task = || {
+        while let Some((out, x)) = { it.lock().unwrap().next() } {
+            *out = Some(f(x));
+        }
+    };
+    std::thread::scope(|s| {
+        let nproc = std::thread::available_parallelism().unwrap().get();
+        for _ in 0..nproc {
+            std::thread::Builder::new()
+                .stack_size(4 * 1024 * 1024)
+                .spawn_scoped(s, task)
+                .unwrap();
+        }
+    });
+    out.into_iter().map(Option::unwrap)
+D7EF
}
+
 #[derive(Default)]
 struct DirWalker<'a> {
     excludes: &'a [pattern::ModulePattern],
@@ -492,21 +521,21 @@
    fn test_pattern() {
        let pattern: ModulePattern = "x.bar.foo_*.a".parse().unwrap();
        assert!(pattern.matches("x.bar.foo_asdf.a"));
        assert!(pattern.matches("x.bar.foo_bazzzz.a"));
        assert!(pattern.matches("x.bar.foo_.a"));
        assert!(!pattern.matches("x.bar.foo_"));
        assert!(!pattern.matches("x.bar.foo_quxxx"));
        assert!(!pattern.matches("foo_b.a"));

        let pattern: ModulePattern = "**.foo.**".parse().unwrap();
        assert!(pattern.matches("ba.bazzz.foo.quux"));

        let pattern: ModulePattern = "*.foo.**".parse().unwrap();
        assert!(pattern.matches("ba.foo.baz.quux"));
        assert!(pattern.matches("asdf.foo.barrr"));

        let pattern: ModulePattern = "foo.**".parse().unwrap();
        assert!(pattern.matches("foo.baaar.qx"));
        assert!(!pattern.matches("asdf.foo.brrrr"));

        let pattern: ModulePattern = "foo.**.bar*".parse().unwrap();