8000 fix: should rebuild chunk outgoings when incremental runtime change by JSerFeng · Pull Request #10577 · web-infra-dev/rspack · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions crates/rspack_core/src/artifacts/cgm_hash_artifact.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,13 @@ impl CgmHashArtifact {
self.module_to_hashes.is_empty()
}

pub fn get_runtime_map(
&self,
module: &ModuleIdentifier,
) -> Option<&RuntimeSpecMap<RspackHashDigest>> {
self.module_to_hashes.get(module)
}

pub fn get(&self, module: &ModuleIdentifier, runtime: &RuntimeSpec) -> Option<&RspackHashDigest> {
let hashes = self.module_to_hashes.get(module)?;
hashes.get(runtime)
Expand Down
60 changes: 38 additions & 22 deletions crates/rspack_core/src/build_chunk_graph/new_code_splitter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,7 @@ pub struct CacheableChunkItem {
pub struct CodeSplitter {
cache_chunk_desc: HashMap<CreateChunkRoot, Vec<CacheableChunkItem>>,
cache_chunks: UkeyMap<CacheUkey, Chunk>,

pub module_deps: ModuleDeps,
pub module_deps_without_runtime:
IdentifierDashMap<(Vec<ModuleIdentifier>, Vec<AsyncDependenciesBlockIdentifier>)>,
pub module_ordinal: IdentifierMap<u64>,
}

Expand Down Expand Up @@ -120,6 +117,13 @@ impl ChunkDesc {
}
}

pub(crate) fn chunk_modules(&self) -> &IdentifierSet {
match self {
ChunkDesc::Entry(entry) => &entry.chunk_modules,
ChunkDesc::Chunk(chunk) => &chunk.chunk_modules,
}
}

pub(crate) fn chunk_modules_ordinal(&self) -> &AvailableModules {
match self {
ChunkDesc::Entry(entry) => &entry.modules_ordinal,
Expand Down Expand Up @@ -233,7 +237,7 @@ impl CreateChunkRoot {
)
.filter_map(|dep_id| module_graph.module_identifier_by_dependency_id(dep_id))
{
splitter.fill_chunk_modules(*m, Some(runtime), &module_graph, &mut ctx);
splitter.fill_chunk_modules(*m, runtime, &module_graph, &mut ctx);
}

vec![ChunkDesc::Entry(Box::new(EntryChunkDesc {
Expand Down Expand Up @@ -287,7 +291,7 @@ impl CreateChunkRoot {
continue;
};

splitter.fill_chunk_modules(*m, Some(runtime), &module_graph, &mut ctx);
splitter.fill_chunk_modules(*m, runtime, &module_graph, &mut ctx);
}

if let Some(group_option) = block.get_group_options()
Expand Down Expand Up @@ -375,27 +379,29 @@ impl CodeSplitter {
cache_chunk_desc: Default::default(),
cache_chunks: Default::default(),
module_deps: Default::default(),
module_deps_without_runtime: Default::default(),
module_ordinal,
}
}

fn invalidate_outgoing_cache(&mut self, module: ModuleIdentifier) {
// refresh module traversal result in the last compilation
for map in self.module_deps.values() {
map.remove(&module);
}
}

// modify the module ordinal for changed_modules
pub fn invalidate(&mut self, changed_modules: impl Iterator<Item = ModuleIdentifier>) {
let module_ordinal = &mut self.module_ordinal;
let mut invalidate_outgoing_module = IdentifierSet::default();

for module in changed_modules {
// add ordinal for new modules
if !module_ordinal.contains_key(&module) {
module_ordinal.insert(module, module_ordinal.len() as u64);
}

// refresh module traversal result in the last compilation
for map in self.module_deps.values() {
map.remove(&module);
}

self.module_deps_without_runtime.remove(&module);
invalidate_outgoing_module.insert(module);

// remove chunks that contains changed module
self.cache_chunk_desc.retain(|_, chunks| {
Expand All @@ -405,9 +411,19 @@ impl CodeSplitter {
ChunkDesc::Chunk(normal_chunk_desc) => !normal_chunk_desc.chunk_modules.contains(&module),
});

if !can_reuse {
for chunk in chunks.iter() {
invalidate_outgoing_module.extend(chunk.chunk_desc.chunk_modules().iter().copied());
}
}

can_reuse
});
}

for m in invalidate_outgoing_module {
self.invalidate_outgoing_cache(m);
}
}

#[instrument(skip_all)]
Expand Down Expand Up @@ -485,7 +501,7 @@ impl CodeSplitter {
continue;
}

let guard = self.outgoings_modules(&module, Some(runtime.as_ref()), &module_graph);
let guard = self.outgoings_modules(&module, runtime.as_ref(), &module_graph);
let (modules, blocks) = guard.value();
let blocks = blocks.clone();
for m in modules {
Expand Down Expand Up @@ -731,14 +747,10 @@ impl CodeSplitter {
pub fn outgoings_modules(
&self,
module: &ModuleIdentifier,
runtime: Option<&RuntimeSpec>,
runtime: &RuntimeSpec,
module_graph: &ModuleGraph,
) -> Ref<ModuleIdentifier, (Vec<ModuleIdentifier>, Vec<AsyncDependenciesBlockIdentifier>)> {
let module_map = if let Some(runtime) = runtime {
self.module_deps.get(runtime).expect("should have value")
} else {
&self.module_deps_without_runtime
};
let module_map = self.module_deps.get(runtime).expect("should have value");

let guard = module_map.get(module);

Expand Down Expand Up @@ -771,7 +783,7 @@ impl CodeSplitter {

'outer: for (m, conns) in outgoings.iter() {
for conn in conns {
let conn_state = conn.active_state(module_graph, runtime);
let conn_state = conn.active_state(module_graph, Some(runtime));
match conn_state {
crate::ConnectionState::Active(true) => {
modules.insert(*m);
Expand All @@ -797,7 +809,7 @@ impl CodeSplitter {
fn fill_chunk_modules(
&self,
target_module: ModuleIdentifier,
runtime: Option<&RuntimeSpec>,
runtime: &RuntimeSpec,
module_graph: &ModuleGraph,
ctx: &mut FillCtx,
) {
Expand Down Expand Up @@ -971,7 +983,7 @@ impl CodeSplitter {
if !self.module_deps.contains_key(runtime) {
self.module_deps.insert(runtime.clone(), Default::default());
}
let guard = self.outgoings_modules(&m, Some(runtime), &module_graph);
let guard = self.outgoings_modules(&m, runtime, &module_graph);
let (modules, blocks) = guard.value();

for m in modules.iter().rev() {
Expand Down Expand Up @@ -1113,6 +1125,10 @@ impl CodeSplitter {
continue;
}

if !reuse {
chunk_desc.chunk_modules();
}

match chunk_desc {
ChunkDesc::Entry(entry_desc) => {
let box EntryChunkDesc {
Expand Down
55 changes: 54 additions & 1 deletion crates/rspack_core/src/compiler/compilation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ use crate::{
DependencyId, DependencyTemplate, DependencyTemplateType, DependencyType, Entry, EntryData,
EntryOptions, EntryRuntime, Entrypoint, ExecuteModuleId, Filename, ImportVarMap, Logger,
ModuleFactory, ModuleGraph, ModuleGraphPartial, ModuleIdentifier, ModuleIdsArtifact, PathData,
ResolverFactory, RuntimeGlobals, RuntimeModule, RuntimeSpecMap, RuntimeTemplate,
ResolverFactory, RuntimeGlobals, RuntimeMode, RuntimeModule, RuntimeSpecMap, RuntimeTemplate,
SharedPluginDriver, SideEffectsOptimizeArtifact, SourceType, Stats,
};

Expand Down Expand Up @@ -1624,13 +1624,66 @@ impl Compilation {
_ => {}
}
}

// check if module runtime changes
for mi in mg.modules().keys() {
let module_runtimes = self
.chunk_graph
.get_module_runtimes(*mi, &self.chunk_by_ukey);
let module_runtime_keys = module_runtimes
.values()
.map(get_runtime_key)
.collect::<Vec<_>>();

if let Some(runtime_map) = self.cgm_hash_artifact.get_runtime_map(mi) {
if module_runtimes.is_empty() {
// module has no runtime, skip
continue;
}
if module_runtimes.len() == 1 {
// single runtime
if !matches!(runtime_map.mode, RuntimeMode::SingleEntry)
|| runtime_map
.single_runtime
.as_ref()
.expect("should have single runtime for single entry")
!= module_runtimes
.values()
.next()
.expect("should have at least one runtime")
{
modules.insert(*mi);
}
} else {
// multiple runtimes
if matches!(runtime_map.mode, RuntimeMode::SingleEntry) {
modules.insert(*mi);
continue;
}

if runtime_map.map.len() != module_runtimes.len() {
modules.insert(*mi);
continue;
}

for runtime_key in runtime_map.map.keys() {
if !module_runtime_keys.contains(&runtime_key.as_str()) {
modules.insert(*mi);
break;
}
}
}
}
}

tracing::debug!(target: incremental::TRACING_TARGET, passes = %IncrementalPasses::MODULES_HASHES, %mutations, ?modules);
let logger = self.get_logger("rspack.incremental.modulesHashes");
logger.log(format!(
"{} modules are affected, {} in total",
modules.len(),
mg.modules().len()
));

modules
} else {
self.get_module_graph().modules().keys().copied().collect()
Expand Down
19 changes: 1 addition & 18 deletions crates/rspack_core/src/old_cache/local/code_splitting_cache.rs
Original file line number Diff line number Diff line change
Expand Up @@ -185,12 +185,7 @@ impl CodeSplittingCache {
return false;
}

if self.new_code_splitter.module_deps.is_empty()
&& self
.new_code_splitter
.module_deps_without_runtime
.is_empty()
{
if self.new_code_splitter.module_deps.is_empty() {
logger.log("no cache detected, rebuilding chunk graph");
return false;
}
Expand Down Expand Up @@ -259,18 +254,6 @@ impl CodeSplittingCache {
}
}

if let Some(outgoings) = self
.new_code_splitter
.module_deps_without_runtime
.get(&module)
{
newly_added_module = false;
let (outgoings, _blocks) = outgoings.value();
for out in outgoings {
previous_outgoings.insert(*out);
}
}

if newly_added_module {
logger.log(format!("new module: {module}"));
return false;
Expand Down
6 changes: 1 addition & 5 deletions packages/rspack-test-tools/tests/NewIncremental-node.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,7 @@ describeByWalk(
{
source: path.resolve(__dirname, "./hotCases"),
dist: path.resolve(__dirname, `./js/new-incremental/hot-async-node`),
exclude: [
/^css$/,
/move-between-runtime/,
/require-disposed-module-warning/
]
exclude: [/^css$/, /require-disposed-module-warning/]
}
);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,6 @@ describeByWalk(
{
source: path.resolve(__dirname, "../../../tests/webpack-test/hotCases"),
dist: path.resolve(__dirname, `./js/new-incremental/webpack-test/hot-web`),
exclude: [/move-between-runtime/, /require-disposed-module-warning/]
exclude: [/require-disposed-module-warning/]
}
);
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,7 @@ describeByWalk(
{
source: path.resolve(__dirname, "./hotCases"),
dist: path.resolve(__dirname, `./js/new-incremental/hot-worker`),
exclude: [
/^css$/,
/move-between-runtime/,
/require-disposed-module-warning/
]
exclude: [/^css$/, /require-disposed-module-warning/]
}
);

Expand Down
Loading
0