8000 Parallel lib compilation(!?) · RustPython/RustPython@dca9824 · GitHub
[go: up one dir, main page]

Skip to content

Commit dca9824

Browse files
committed
Parallel lib compilation(!?)
1 parent a6cba69 commit dca9824

File tree

1 file changed

+37
-8
lines changed

1 file changed

+37
-8
lines changed

derive-impl/src/compile_bytecode.rs

Lines changed: 37 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -52,12 +52,13 @@ enum CompilationSource {
5252
LibPath(PathBuf),
5353
}
5454

55+
#[derive(Clone)]
5556
struct CompiledModule {
5657
code: CodeObject,
5758
package: bool,
5859
}
5960

60-
pub trait Compiler {
61+
pub trait Compiler: Sync {
6162
fn compile(
6263
&self,
6364
source: &str,
@@ -95,13 +96,15 @@ impl CompilationSource {
9596
return Ok(vec![(module_name, module)]);
9697
}
9798
}
98-
dir.modules
99-
.into_iter()
100-
.map(|(module_name, (path, package))| {
101-
let code = Self::compile_file(&path, mode, &module_name, compiler)?;
102-
Ok((module_name, CompiledModule { code, package }))
103-
})
104-
.collect()
99+
let do_compile = |(module_name, (path, package)): (String, (PathBuf, _))| {
100+
let code = Self::compile_file(&path, mode, &module_name, compiler)?;
101+
Ok((module_name, CompiledModule { code, package }))
102+
};
103+
if dir.modules.len() > 32 {
104+
par_map(dir.modules, do_compile).collect()
105+
} else {
106+
dir.modules.into_iter().map(do_compile).collect()
107+
}
105108
}
106109

107110
fn compile_file(
@@ -152,6 +155,32 @@ impl CompilationSource {
152155
}
153156
}
154157

158+
fn par_map<T, U, I, F>(it: I, f: F) -> impl Iterator<Item = U>
159+
where
160+
I: IntoIterator<Item = T, IntoIter: ExactSizeIterator + Send>,
161+
F: Fn(T) -> U + Sync,
162+
U: Send,
163+
{
164+
let it = it.into_iter();
165+
let mut out = Vec::from_iter(std::iter::repeat_with(|| None).take(it.len()));
166+
let it = std::sync::Mutex::new(std::iter::zip(&mut out, it));
167+
let task = || {
168+
while let Some((out, x)) = { it.lock().unwrap().next() } {
169+
*out = Some(f(x));
170+
}
171+
};
172+
std::thread::scope(|s| {
173+
let nproc = std::thread::available_parallelism().unwrap().get();
174+
for _ in 0..nproc {
175+
std::thread::Builder::new()
176+
.stack_size(4 * 1024 * 1024)
177+
.spawn_scoped(s, task)
178+
.unwrap();
179+
}
180+
});
181+
out.into_iter().map(Option::unwrap)
182+
}
183+
155184
#[derive(Default)]
156185
struct DirWalker<'a> {
157186
excludes: &'a [pattern::ModulePattern],

0 commit comments

Comments
 (0)
0