10000 Track in Emitter whether a module changed in an incremental run · scala-js/scala-js@d2b7322 · GitHub
[go: up one dir, main page]
Skip to content

Commit d2b7322

Browse files
gzm0gzm0
committed
Track in Emitter whether a module changed in an incremental run
In the next commit, we want to avoid caching entire classes because of the memory cost. However, the BasicLinkerBackend relies on the identity of the generated trees to detect changes: Since that identity will change if we stop caching them, we need to provide an explicit "changed" signal.
1 parent 393802d commit d2b7322

File tree

3 files changed

+68
-52
lines changed

3 files changed

+68
-52
lines changed

‎linker/jvm/src/main/scala/org/scalajs/linker/backend/closure/ClosureLinkerBackend.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,8 @@ final class ClosureLinkerBackend(config: LinkerBackendImpl.Config)
106106
sjsModule <- moduleSet.modules.headOption
107107
} yield {
108108
val closureChunk = logger.time("Closure: Create trees)") {
109-
buildChunk(emitterResult.body(sjsModule.id))
109+
val (trees, _) = emitterResult.body(sjsModule.id)
110+
buildChunk(trees)
110111
}
111112

112113
logger.time("Closure: Compiler pass") {

‎linker/shared/src/main/scala/org/scalajs/linker/backend/BasicLinkerBackend.scala

Lines changed: 2 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -88,9 +88,7 @@ final class BasicLinkerBackend(config: LinkerBackendImpl.Config)
8888
val writer = new OutputWriter(output, config, skipContentCheck) {
8989
protected def writeModuleWithoutSourceMap(moduleID: ModuleID, force: Boolean): Option[ByteBuffer] = {
9090
val cache = printedModuleSetCache.getModuleCache(moduleID)
91-
val printedTrees = emitterResult.body(moduleID)
92-
93-
val changed = cache.update(printedTrees)
91+
val (printedTrees, changed) = emitterResult.body(moduleID)
9492

9593
if (force || changed || allChanged) {
9694
rewrittenModules.incrementAndGet()
@@ -114,9 +112,7 @@ final class BasicLinkerBackend(config: LinkerBackendImpl.Config)
114112

115113
protected def writeModuleWithSourceMap(moduleID: ModuleID, force: Boolean): Option[(ByteBuffer, ByteBuffer)] = {
116114
val cache = printedModuleSetCache.getModuleCache(moduleID)
117-
val printedTrees = emitterResult.body(moduleID)
118-
119-
val changed = cache.update(printedTrees)
115+
val (printedTrees, changed) = emitterResult.body(moduleID)
120116

121117
if (force || changed || allChanged) {
122118
rewrittenModules.incrementAndGet()
@@ -220,8 +216,6 @@ private object BasicLinkerBackend {
220216

221217
private sealed class PrintedModuleCache {
222218
private var cacheUsed = false
223-
private var changed = false
224-
private var lastPrintedTrees: List[js.PrintedTree] = Nil
225219

226220
private var previousFinalJSFileSize: Int = 0
227221
private var previousFinalSourceMapSize: Int = 0
@@ -239,15 +233,6 @@ private object BasicLinkerBackend {
239233
previousFinalSourceMapSize = finalSourceMapSize
240234
}
241235

242-
def update(newPrintedTrees: List[js.PrintedTree]): Boolean = {
243-
val changed = !newPrintedTrees.corresponds(lastPrintedTrees)(_ eq _)
244-
this.changed = changed
245-
if (changed) {
246-
lastPrintedTrees = newPrintedTrees
247-
}
248-
changed
249-
}
250-
251236
def cleanAfterRun(): Boolean = {
252237
val wasUsed = cacheUsed
253238
cacheUsed = false

‎linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/Emitter.scala

Lines changed: 64 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ final class Emitter[E >: Null <: js.Tree](
112112
}
113113

114114
private def emitInternal(moduleSet: ModuleSet,
115-
logger: Logger): WithGlobals[Map[ModuleID, List[E]]] = {
115+
logger: Logger): WithGlobals[Map[ModuleID, (List[E], Boolean)]] = {
116116
// Reset caching stats.
117117
statsClassesReused = 0
118118
statsClassesInvalidated = 0
@@ -169,7 +169,7 @@ final class Emitter[E >: Null <: js.Tree](
169169
*/
170170
@tailrec
171171
private def emitAvoidGlobalClash(moduleSet: ModuleSet,
172-
logger: Logger, secondAttempt: Boolean): WithGlobals[Map[ModuleID, List[E]]] = {
172+
logger: Logger, secondAttempt: Boolean): WithGlobals[Map[ModuleID, (List[E], Boolean)]] = {
173173
val result = emitOnce(moduleSet, logger)
174174

175175
val mentionedDangerousGlobalRefs =
@@ -194,7 +194,7 @@ final class Emitter[E >: Null <: js.Tree](
194194
}
195195

196196
private def emitOnce(moduleSet: ModuleSet,
197-
logger: Logger): WithGlobals[Map[ModuleID, List[E]]] = {
197+
logger: Logger): WithGlobals[Map[ModuleID, (List[E], Boolean)]] = {
198198
// Genreate classes first so we can measure time separately.
199199
val generatedClasses = logger.time("Emitter: Generate Classes") {
200200
moduleSet.modules.map { module =>
@@ -212,18 +212,26 @@ final class Emitter[E >: Null <: js.Tree](
212212

213213
val moduleTrees = logger.time("Emitter: Write trees") {
214214
moduleSet.modules.map { module =>
215+
var changed = false
216+
def extractChangedAndWithGlobals[T](x: (WithGlobals[T], Boolean)): T = {
217+
changed ||= x._2
218+
extractWithGlobals(x._1)
219+
}
220+
215221
val moduleContext = ModuleContext.fromModule(module)
216222
val moduleCache = state.moduleCaches.getOrElseUpdate(module.id, new ModuleCache)
217223

218224
val moduleClasses = generatedClasses(module.id)
219225

220-
val moduleImports = extractWithGlobals {
226+
changed ||= moduleClasses.exists(_.changed)
227+
228+
val moduleImports = extractChangedAndWithGlobals {
221229
moduleCache.getOrComputeImports(module.externalDependencies, module.internalDependencies) {
222230
genModuleImports(module).map(postTransform(_, 0))
223231
}
224232
}
225233

226-
val topLevelExports = extractWithGlobals {
234+
val topLevelExports = extractChangedAndWithGlobals {
227235
/* We cache top level exports all together, rather than individually,
228236
* since typically there are few.
229237
*/
@@ -233,7 +241,7 @@ final class Emitter[E >: Null <: js.Tree](
233241
}
234242
}
235243

236-
val moduleInitializers = extractWithGlobals {
244+
val moduleInitializers = extractChangedAndWithGlobals {
237245
val initializers = module.initializers.toList
238246
moduleCache.getOrComputeInitializers(initializers) {
239247
WithGlobals.list(initializers.map { initializer =>
@@ -324,7 +332,7 @@ final class Emitter[E >: Null <: js.Tree](
324332
trackedGlobalRefs = unionPreserveEmpty(trackedGlobalRefs, genClass.trackedGlobalRefs)
325333
}
326334

327-
module.id -> allTrees
335+
module.id -> (allTrees, changed)
328336
}
329337
}
330338

@@ -382,8 +390,14 @@ final class Emitter[E >: Null <: js.Tree](
382390
val classCache = classCaches.getOrElseUpdate(
383391
new ClassID(linkedClass.ancestors, moduleContext), new ClassCache)
384392

393+
var changed = false
394+
def extractChanged[T](x: (T, Boolean)): T = {
395+
changed ||= x._2
396+
x._1
397+
}
398+
385399
val classTreeCache =
386-
classCache.getCache(linkedClass.version)
400+
extractChanged(classCache.getCache(linkedClass.version))
387401

388402
val kind = linkedClass.kind
389403

@@ -396,6 +410,9 @@ final class Emitter[E >: Null <: js.Tree](
396410
withGlobals.value
397411
}
398412

413+
def extractWithGlobalsAndChanged[T](x: (WithGlobals[T], Boolean)): T =
414+
extractWithGlobals(extractChanged(x))
415+
399416
// Main part
400417

401418
val main = List.newBuilder[E]
@@ -426,7 +443,7 @@ final class Emitter[E >: Null <: js.Tree](
426443
val methodCache =
427444
classCache.getStaticLikeMethodCache(namespace, methodDef.methodName)
428445

429-
main ++= extractWithGlobals(methodCache.getOrElseUpdate(methodDef.version, {
446+
main ++= extractWithGlobalsAndChanged(methodCache.getOrElseUpdate(methodDef.version, {
430447
classEmitter.genStaticLikeMethod(className, methodDef)(moduleContext, methodCache)
431448
.map(postTransform(_, 0))
432449
}))
@@ -486,7 +503,7 @@ final class Emitter[E >: Null <: js.Tree](
486503
}
487504

488505
// JS constructor
489-
val ctorWithGlobals = {
506+
val ctorWithGlobals = extractChanged {
490507
/* The constructor depends both on the class version, and the version
491508
* of the inlineable init, if there is one.
492509
*
@@ -571,13 +588,13 @@ final class Emitter[E >: Null <: js.Tree](
571588
classCache.getMemberMethodCache(method.methodName)
572589

573590
val version = Version.combine(isJSClassVersion, method.version)
574-
methodCache.getOrElseUpdate(version,
591+
extractChanged(methodCache.getOrElseUpdate(version,
575592
classEmitter.genMemberMethod(
576593
className, // invalidated by overall class cache
577594
isJSClass, // invalidated by isJSClassVersion
578595
useESClass, // invalidated by isJSClassVersion
579596
method // invalidated by method.version
580-
)(moduleContext, methodCache).map(postTransform(_, memberIndent)))
597+
)(moduleContext, methodCache).map(postTransform(_, memberIndent))))
581598
}
582599

583600
// Exported Members
@@ -586,13 +603,13 @@ final class Emitter[E >: Null <: js.Tree](
586603
} yield {
587604
val memberCache = classCache.getExportedMemberCache(idx)
588605
val version = Version.combine(isJSClassVersion, member.version)
589-
memberCache.getOrElseUpdate(version,
606+
extractChanged(memberCache.getOrElseUpdate(version,
590607
classEmitter.genExportedMember(
591608
className, // invalidated by overall class cache
592609
isJSClass, // invalidated by isJSClassVersion
593610
useESClass, // invalidated by isJSClassVersion
594611
member // invalidated by version
595-
)(moduleContext, memberCache).map(postTransform(_, memberIndent)))
612+
)(moduleContext, memberCache).map(postTransform(_, memberIndent))))
596613
}
597614

598615
val hasClassInitializer: Boolean = {
@@ -602,7 +619,7 @@ final class Emitter[E >: Null <: js.Tree](
602619
}
603620
}
604621

605-
val fullClass = {
622+
val fullClass = extractChanged {
606623
val fullClassCache = classCache.getFullClassCache()
607624

608625
fullClassCache.getOrElseUpdate(linkedClass.version, ctorWithGlobals,
@@ -714,7 +731,8 @@ final class Emitter[E >: Null <: js.Tree](
714731
main.result(),
715732
staticFields,
716733
staticInitialization,
717-
trackedGlobalRefs
734+
trackedGlobalRefs,
735+
changed
718736
)
719737
}
720738

@@ -751,28 +769,33 @@ final class Emitter[E >: Null <: js.Tree](
751769
}
752770

753771
def getOrComputeImports(externalDependencies: Set[String], internalDependencies: Set[ModuleID])(
754-
compute: => WithGlobals[List[E]]): WithGlobals[List[E]] = {
772+
compute: => WithGlobals[List[E]]): (WithGlobals[List[E]], Boolean) = {
755773

756774
_cacheUsed = true
757775

758776
if (externalDependencies != _lastExternalDependencies || internalDependencies != _lastInternalDependencies) {
759777
_importsCache = compute
760778
_lastExternalDependencies = externalDependencies
761779
_lastInternalDependencies = internalDependencies
780+
(_importsCache, true)
781+
} else {
782+
(_importsCache, false)
762783
}
763-
_importsCache
784+
764785
}
765786

766787
def getOrComputeTopLevelExports(topLevelExports: List[LinkedTopLevelExport])(
767-
compute: => WithGlobals[List[E]]): WithGlobals[List[E]] = {
788+
compute: => WithGlobals[List[E]]): (WithGlobals[List[E]], Boolean) = {
768789

769790
_cacheUsed = true
770791

771792
if (!sameTopLevelExports(topLevelExports, _lastTopLevelExports)) {
772793
_topLevelExportsCache = compute
773794
_lastTopLevelExports = topLevelExports
795+
(_topLevelExportsCache, true)
796+
} else {
797+
(_topLevelExportsCache, false)
774798
}
775-
_topLevelExportsCache
776799
}
777800

778801
private def sameTopLevelExports(tles1: List[LinkedTopLevelExport], tles2: List[LinkedTopLevelExport]): Boolean = {
@@ -803,15 +826,17 @@ final class Emitter[E >: Null <: js.Tree](
803826
}
804827

805828
def getOrComputeInitializers(initializers: List[ModuleInitializer.Initializer])(
806-
compute: => WithGlobals[List[E]]): WithGlobals[List[E]] = {
829+
compute: => WithGlobals[List[E]]): (WithGlobals[List[E]], Boolean) = {
807830

808831
_cacheUsed = true
809832

810833
if (initializers != _lastInitializers) {
811834
_initializersCache = compute
812835
_lastInitializers = initializers
836+
(_initializersCache, true)
837+
} else {
838+
(_initializersCache, false)
813839
}
814-
_initializersCache
815840
}
816841

817842
def cleanAfterRun(): Boolean = {
@@ -856,17 +881,18 @@ final class Emitter[E >: Null <: js.Tree](
856881
_fullClassCache.foreach(_.startRun())
857882
}
858883

859-
def getCache(version: Version): DesugaredClassCache[List[E]] = {
884+
def getCache(version: Version): (DesugaredClassCache[List[E]], Boolean) = {
885+
_cacheUsed = true
860886
if (_cache == null || !_lastVersion.sameVersion(version)) {
861887
invalidate()
862888
statsClassesInvalidated += 1
863889
_lastVersion = version
864890
_cache = new DesugaredClassCache[List[E]]
891+
(_cache, true)
865892
} else {
866893
statsClassesReused += 1
894+
(_cache, false)
867895
}
868-
_cacheUsed = true
869-
_cache
870896
}
871897

872898
def getMemberMethodCache(
@@ -932,17 +958,18 @@ final class Emitter[E >: Null <: js.Tree](
932958
def startRun(): Unit = _cacheUsed = false
933959

934960
def getOrElseUpdate(version: Version,
935-
v: => WithGlobals[T]): WithGlobals[T] = {
961+
v: => WithGlobals[T]): (WithGlobals[T], Boolean) = {
962+
_cacheUsed = true
936963
if (_tree == null || !_lastVersion.sameVersion(version)) {
937964
invalidate()
938965
statsMethodsInvalidated += 1
939966
_tree = v
940967
_lastVersion = version
968+
(_tree, true)
941969
} else {
942970
statsMethodsReused += 1
971+
(_tree, false)
943972
}
944-
_cacheUsed = true
945-
_tree
946973
}
947974

948975
def cleanAfterRun(): Boolean = {
@@ -974,7 +1001,7 @@ final class Emitter[E >: Null <: js.Tree](
9741001

9751002
def getOrElseUpdate(version: Version, ctor: WithGlobals[List[E]],
9761003
memberMethods: List[WithGlobals[List[E]]], exportedMembers: List[WithGlobals[List[E]]],
977-
compute: => WithGlobals[List[E]]): WithGlobals[List[E]] = {
1004+
compute: => WithGlobals[List[E]]): (WithGlobals[List[E]], Boolean) = {
9781005

9791006
@tailrec
9801007
def allSame[A <: AnyRef](xs: List[A], ys: List[A]): Boolean = {
@@ -984,6 +1011,8 @@ final class Emitter[E >: Null <: js.Tree](
9841011
}
9851012
}
9861013

1014+
_cacheUsed = true
1015+
9871016
if (_tree == null || !version.sameVersion(_lastVersion) || (_lastCtor ne ctor) ||
9881017
!allSame(_lastMemberMethods, memberMethods) ||
9891018
!allSame(_lastExportedMembers, exportedMembers)) {
@@ -993,10 +1022,10 @@ final class Emitter[E >: Null <: js.Tree](
9931022
_lastCtor = ctor
9941023
_lastMemberMethods = memberMethods
9951024
_lastExportedMembers = exportedMembers
1025+
(_tree, true)
1026+
} else {
1027+
(_tree, false)
9961028
}
997-
998-
_cacheUsed = true
999-
_tree
10001029
}
10011030

10021031
def cleanAfterRun(): Boolean = {
@@ -1030,7 +1059,7 @@ object Emitter {
10301059
/** Result of an emitter run. */
10311060
final class Result[E] private[Emitter](
10321061
val header: String,
1033-
val body: Map[ModuleID, List[E]],
1062+
val body: Map[ModuleID, (List[E], Boolean)],
10341063
val footer: String,
10351064
val topLevelVarDecls: List[String],
10361065
val globalRefs: Set[String]
@@ -1121,7 +1150,8 @@ object Emitter {
11211150
val main: List[E],
11221151
val staticFields: List[E],
11231152
val staticInitialization: List[E],
1124-
val trackedGlobalRefs: Set[String]
1153+
val trackedGlobalRefs: Set[String],
1154+
val changed: Boolean
11251155
)
11261156

11271157
private final class OneTimeCache[A >: Null] {

0 commit comments

Comments
 (0)
0