Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit ea5a7c4

Browse files
gzm0gzm0
committed
Track in Emitter whether a module changed in an incremental run
In the next commit, we want to avoid caching entire classes because of the memory cost. However, the BasicLinkerBackend relies on the identity of the generated trees to detect changes: Since that identity will change if we stop caching them, we need to provide an explicit "changed" signal.
1 parent 6a4b945 commit ea5a7c4

File tree

3 files changed

+68
-52
lines changed

3 files changed

+68
-52
lines changed

‎linker/jvm/src/main/scala/org/scalajs/linker/backend/closure/ClosureLinkerBackend.scala

+2-1
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,8 @@ final class ClosureLinkerBackend(config: LinkerBackendImpl.Config)
109109
sjsModule <- moduleSet.modules.headOption
110110
} yield {
111111
val closureChunk = logger.time("Closure: Create trees)") {
112-
buildChunk(emitterResult.body(sjsModule.id))
112+
val (trees, _) = emitterResult.body(sjsModule.id)
113+
buildChunk(trees)
113114
}
114115

115116
logger.time("Closure: Compiler pass") {

‎linker/shared/src/main/scala/org/scalajs/linker/backend/BasicLinkerBackend.scala

+2-17
Original file line numberDiff line numberDiff line change
@@ -90,9 +90,7 @@ final class BasicLinkerBackend(config: LinkerBackendImpl.Config)
9090
val writer = new OutputWriter(output, config, skipContentCheck) {
9191
protected def writeModuleWithoutSourceMap(moduleID: ModuleID, force: Boolean): Option[ByteBuffer] = {
9292
val cache = printedModuleSetCache.getModuleCache(moduleID)
93-
val printedTrees = emitterResult.body(moduleID)
94-
95-
val changed = cache.update(printedTrees)
93+
val (printedTrees, changed) = emitterResult.body(moduleID)
9694

9795
if (force || changed || allChanged) {
9896
rewrittenModules.incrementAndGet()
@@ -116,9 +114,7 @@ final class BasicLinkerBackend(config: LinkerBackendImpl.Config)
116114

117115
protected def writeModuleWithSourceMap(moduleID: ModuleID, force: Boolean): Option[(ByteBuffer, ByteBuffer)] = {
118116
val cache = printedModuleSetCache.getModuleCache(moduleID)
119-
val printedTrees = emitterResult.body(moduleID)
120-
121-
val changed = cache.update(printedTrees)
117+
val (printedTrees, changed) = emitterResult.body(moduleID)
122118

123119
if (force || changed || allChanged) {
124120
rewrittenModules.incrementAndGet()
@@ -222,8 +218,6 @@ private object BasicLinkerBackend {
222218

223219
private sealed class PrintedModuleCache {
224220
private var cacheUsed = false
225-
private var changed = false
226-
private var lastPrintedTrees: List[PrintedTree] = Nil
227221

228222
private var previousFinalJSFileSize: Int = 0
229223
private var previousFinalSourceMapSize: Int = 0
@@ -241,15 +235,6 @@ private object BasicLinkerBackend {
241235
previousFinalSourceMapSize = finalSourceMapSize
242236
}
243237

244-
def update(newPrintedTrees: List[PrintedTree]): Boolean = {
245-
val changed = !newPrintedTrees.corresponds(lastPrintedTrees)(_ eq _)
246-
this.changed = changed
247-
if (changed) {
248-
lastPrintedTrees = newPrintedTrees
249-
}
250-
changed
251-
}
252-
253238
def cleanAfterRun(): Boolean = {
254239
val wasUsed = cacheUsed
255240
cacheUsed = false

‎linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/Emitter.scala

+64-34
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ final class Emitter[E >: Null <: js.Transformed.Value](
112112
}
113113

114114
private def emitInternal(moduleSet: ModuleSet,
115-
logger: Logger): WithGlobals[Map[ModuleID, List[E]]] = {
115+
logger: Logger): WithGlobals[Map[ModuleID, (List[E], Boolean)]] = {
116116
// Reset caching stats.
117117
statsClassesReused = 0
118118
statsClassesInvalidated = 0
@@ -172,7 +172,7 @@ final class Emitter[E >: Null <: js.Transformed.Value](
172172
*/
173173
@tailrec
174174
private def emitAvoidGlobalClash(moduleSet: ModuleSet,
175-
logger: Logger, secondAttempt: Boolean): WithGlobals[Map[ModuleID, List[E]]] = {
175+
logger: Logger, secondAttempt: Boolean): WithGlobals[Map[ModuleID, (List[E], Boolean)]] = {
176176
val result = emitOnce(moduleSet, logger)
177177

178178
val mentionedDangerousGlobalRefs =
@@ -197,7 +197,7 @@ final class Emitter[E >: Null <: js.Transformed.Value](
197197
}
198198

199199
private def emitOnce(moduleSet: ModuleSet,
200-
logger: Logger): WithGlobals[Map[ModuleID, List[E]]] = {
200+
logger: Logger): WithGlobals[Map[ModuleID, (List[E], Boolean)]] = {
201201
// Genreate classes first so we can measure time separately.
202202
val generatedClasses = logger.time("Emitter: Generate Classes") {
203203
moduleSet.modules.map { module =>
@@ -215,18 +215,26 @@ final class Emitter[E >: Null <: js.Transformed.Value](
215215

216216
val moduleTrees = logger.time("Emitter: Write trees") {
217217
moduleSet.modules.map { module =>
218+
var changed = false
219+
def extractChangedAndWithGlobals[T](x: (WithGlobals[T], Boolean)): T = {
220+
changed ||= x._2
221+
extractWithGlobals(x._1)
222+
}
223+
218224
val moduleContext = ModuleContext.fromModule(module)
219225
val moduleCache = state.moduleCaches.getOrElseUpdate(module.id, new ModuleCache)
220226

221227
val moduleClasses = generatedClasses(module.id)
222228

223-
val moduleImports = extractWithGlobals {
229+
changed ||= moduleClasses.exists(_.changed)
230+
231+
val moduleImports = extractChangedAndWithGlobals {
224232
moduleCache.getOrComputeImports(module.externalDependencies, module.internalDependencies) {
225233
genModuleImports(module).map(postTransform(_, 0))
226234
}
227235
}
228236

229-
val topLevelExports = extractWithGlobals {
237+
val topLevelExports = extractChangedAndWithGlobals {
230238
/* We cache top level exports all together, rather than individually,
231239
* since typically there are few.
232240
*/
@@ -236,7 +244,7 @@ final class Emitter[E >: Null <: js.Transformed.Value](
236244
}
237245
}
238246

239-
val moduleInitializers = extractWithGlobals {
247+
val moduleInitializers = extractChangedAndWithGlobals {
240248
val initializers = module.initializers.toList
241249
moduleCache.getOrComputeInitializers(initializers) {
242250
WithGlobals.list(initializers.map { initializer =>
@@ -327,7 +335,7 @@ final class Emitter[E >: Null <: js.Transformed.Value](
327335
trackedGlobalRefs = unionPreserveEmpty(trackedGlobalRefs, genClass.trackedGlobalRefs)
328336
}
329337

330-
module.id -> allTrees
338+
module.id -> (allTrees, changed)
331339
}
332340
}
333341

@@ -385,8 +393,14 @@ final class Emitter[E >: Null <: js.Transformed.Value](
385393
val classCache = classCaches.getOrElseUpdate(
386394
new ClassID(linkedClass.ancestors, moduleContext), new ClassCache)
387395

396+
var changed = false
397+
def extractChanged[T](x: (T, Boolean)): T = {
398+
changed ||= x._2
399+
x._1
400+
}
401+
388402
val classTreeCache =
389-
classCache.getCache(linkedClass.version)
403+
extractChanged(classCache.getCache(linkedClass.version))
390404

391405
val kind = linkedClass.kind
392406

@@ -399,6 +413,9 @@ final class Emitter[E >: Null <: js.Transformed.Value](
399413
withGlobals.value
400414
}
401415

416+
def extractWithGlobalsAndChanged[T](x: (WithGlobals[T], Boolean)): T =
417+
extractWithGlobals(extractChanged(x))
418+
402419
// Main part
403420

404421
val main = List.newBuilder[E]
@@ -429,7 +446,7 @@ final class Emitter[E >: Null <: js.Transformed.Value](
429446
val methodCache =
430447
classCache.getStaticLikeMethodCache(namespace, methodDef.methodName)
431448

432-
main += extractWithGlobals(methodCache.getOrElseUpdate(methodDef.version,
449+
main += extractWithGlobalsAndChanged(methodCache.getOrElseUpdate(methodDef.version,
433450
classEmitter.genStaticLikeMethod(className, methodDef)(moduleContext, methodCache).map(postTransform(_, 0))))
434451
}
435452
}
@@ -482,7 +499,7 @@ final class Emitter[E >: Null <: js.Transformed.Value](
482499
}
483500

484501
// JS constructor
485-
val ctorWithGlobals = {
502+
val ctorWithGlobals = extractChanged {
486503
/* The constructor depends both on the class version, and the version
487504
* of the inlineable init, if there is one.
488505
*
@@ -567,13 +584,13 @@ final class Emitter[E >: Null <: js.Transformed.Value](
567584
classCache.getMemberMethodCache(method.methodName)
568585

569586
val version = Version.combine(isJSClassVersion, method.version)
570-
methodCache.getOrElseUpdate(version,
587+
extractChanged(methodCache.getOrElseUpdate(version,
571588
classEmitter.genMemberMethod(
572589
className, // invalidated by overall class cache
573590
isJSClass, // invalidated by isJSClassVersion
574591
useESClass, // invalidated by isJSClassVersion
575592
method, // invalidated by method.version
576-
)(moduleContext, methodCache).map(postTransform(_, memberIndent)))
593+
)(moduleContext, methodCache).map(postTransform(_, memberIndent))))
577594
}
578595

579596
// Exported Members
@@ -582,13 +599,13 @@ final class Emitter[E >: Null <: js.Transformed.Value](
582599
} yield {
583600
val memberCache = classCache.getExportedMemberCache(idx)
584601
val version = Version.combine(isJSClassVersion, member.version)
585-
memberCache.getOrElseUpdate(version,
602+
extractChanged(memberCache.getOrElseUpdate(version,
586603
classEmitter.genExportedMember(
587604
className, // invalidated by overall class cache
588605
isJSClass, // invalidated by isJSClassVersion
589606
useESClass, // invalidated by isJSClassVersion
590607
member // invalidated by version
591-
)(moduleContext, memberCache).map(postTransform(_, memberIndent)))
608+
)(moduleContext, memberCache).map(postTransform(_, memberIndent))))
592609
}
593610

594611
val hasClassInitializer: Boolean = {
@@ -598,7 +615,7 @@ final class Emitter[E >: Null <: js.Transformed.Value](
598615
}
599616
}
600617

601-
val fullClass = {
618+
val fullClass = extractChanged {
602619
val fullClassCache = classCache.getFullClassCache()
603620

604621
fullClassCache.getOrElseUpdate(linkedClass.version, ctorWithGlobals,
@@ -701,7 +718,8 @@ final class Emitter[E >: Null <: js.Transformed.Value](
701718
main.result(),
702719
staticFields,
703720
staticInitialization,
704-
trackedGlobalRefs
721+
trackedGlobalRefs,
722+
changed
705723
)
706724
}
707725

@@ -738,28 +756,33 @@ final class Emitter[E >: Null <: js.Transformed.Value](
738756
}
739757

740758
def getOrComputeImports(externalDependencies: Set[String], internalDependencies: Set[ModuleID])(
741-
compute: => WithGlobals[E]): WithGlobals[E] = {
759+
compute: => WithGlobals[E]): (WithGlobals[E], Boolean) = {
742760

743761
_cacheUsed = true
744762

745763
if (externalDependencies != _lastExternalDependencies || internalDependencies != _lastInternalDependencies) {
746764
_importsCache = compute
747765
_lastExternalDependencies = externalDependencies
748766
_lastInternalDependencies = internalDependencies
767+
(_importsCache, true)
768+
} else {
769+
(_importsCache, false)
749770
}
750-
_importsCache
771+
751772
}
752773

753774
def getOrComputeTopLevelExports(topLevelExports: List[LinkedTopLevelExport])(
754-
compute: => WithGlobals[E]): WithGlobals[E] = {
775+
compute: => WithGlobals[E]): (WithGlobals[E], Boolean) = {
755776

756777
_cacheUsed = true
757778

758779
if (!sameTopLevelExports(topLevelExports, _lastTopLevelExports)) {
759780
_topLevelExportsCache = compute
760781
_lastTopLevelExports = topLevelExports
782+
(_topLevelExportsCache, true)
783+
} else {
784+
(_topLevelExportsCache, false)
761785
}
762-
_topLevelExportsCache
763786
}
764787

765788
private def sameTopLevelExports(tles1: List[LinkedTopLevelExport], tles2: List[LinkedTopLevelExport]): Boolean = {
@@ -790,15 +813,17 @@ final class Emitter[E >: Null <: js.Transformed.Value](
790813
}
791814

792815
def getOrComputeInitializers(initializers: List[ModuleInitializer.Initializer])(
793-
compute: => WithGlobals[E]): WithGlobals[E] = {
816+
compute: => WithGlobals[E]): (WithGlobals[E], Boolean) = {
794817

795818
_cacheUsed = true
796819

797820
if (initializers != _lastInitializers) {
798821
_initializersCache = compute
799822
_lastInitializers = initializers
823+
(_initializersCache, true)
824+
} else {
825+
(_initializersCache, false)
800826
}
801-
_initializersCache
802827
}
803828

804829
def cleanAfterRun(): Boolean = {
@@ -843,17 +868,18 @@ final class Emitter[E >: Null <: js.Transformed.Value](
843868
_fullClassCache.foreach(_.startRun())
844869
}
845870

846-
def getCache(version: Version): DesugaredClassCache[E] = {
871+
def getCache(version: Version): (DesugaredClassCache[E], Boolean) = {
872+
_cacheUsed = true
847873
if (_cache == null || !_lastVersion.sameVersion(version)) {
848874
invalidate()
849875
statsClassesInvalidated += 1
850876
_lastVersion = version
851877
_cache = new DesugaredClassCache[E]
878+
(_cache, true)
852879
} else {
853880
statsClassesReused += 1
881+
(_cache, false)
854882
}
855-
_cacheUsed = true
856-
_cache
857883
}
858884

859885
def getMemberMethodCache(
@@ -919,17 +945,18 @@ final class Emitter[E >: Null <: js.Transformed.Value](
919945
def startRun(): Unit = _cacheUsed = false
920946

921947
def getOrElseUpdate(version: Version,
922-
v: => WithGlobals[T]): WithGlobals[T] = {
948+
v: => WithGlobals[T]): (WithGlobals[T], Boolean) = {
949+
_cacheUsed = true
923950
if (_tree == null || !_lastVersion.sameVersion(version)) {
924951
invalidate()
925952
statsMethodsInvalidated += 1
926953
_tree = v
927954
_lastVersion = version
955+
(_tree, true)
928956
} else {
929957
statsMethodsReused += 1
958+
(_tree, false)
930959
}
931-
_cacheUsed = true
932-
_tree
933960
}
934961

935962
def cleanAfterRun(): Boolean = {
@@ -961,7 +988,7 @@ final class Emitter[E >: Null <: js.Transformed.Value](
961988

962989
def getOrElseUpdate(version: Version, ctor: WithGlobals[E],
963990
memberMethods: List[WithGlobals[E]], exportedMembers: List[WithGlobals[E]],
964-
compute: => WithGlobals[List[E]]): WithGlobals[List[E]] = {
991+
compute: => WithGlobals[List[E]]): (WithGlobals[List[E]], Boolean) = {
965992

966993
@tailrec
967994
def allSame[A <: AnyRef](xs: List[A], ys: List[A]): Boolean = {
@@ -971,6 +998,8 @@ final class Emitter[E >: Null <: js.Transformed.Value](
971998
}
972999
}
9731000

1001+
_cacheUsed = true
1002+
9741003
if (_tree == null || !version.sameVersion(_lastVersion) || (_lastCtor ne ctor) ||
9751004
!allSame(_lastMemberMethods, memberMethods) ||
9761005
!allSame(_lastExportedMembers, exportedMembers)) {
@@ -980,10 +1009,10 @@ final class Emitter[E >: Null <: js.Transformed.Value](
9801009
_lastCtor = ctor
9811010
_lastMemberMethods = memberMethods
9821011
_lastExportedMembers = exportedMembers
1012+
(_tree, true)
1013+
} else {
1014+
(_tree, false)
9831015
}
984-
985-
_cacheUsed = true
986-
_tree
9871016
}
9881017

9891018
def cleanAfterRun(): Boolean = {
@@ -1017,7 +1046,7 @@ object Emitter {
10171046
/** Result of an emitter run. */
10181047
final class Result[E] private[Emitter](
10191048
val header: String,
1020-
val body: Map[ModuleID, List[E]],
1049+
val body: Map[ModuleID, (List[E], Boolean)],
10211050
val footer: String,
10221051
val topLevelVarDecls: List[String],
10231052
val globalRefs: Set[String]
@@ -1108,7 +1137,8 @@ object Emitter {
11081137
val main: List[E],
11091138
val staticFields: E,
11101139
val staticInitialization: E,
1111-
val trackedGlobalRefs: Set[String]
1140+
val trackedGlobalRefs: Set[String],
1141+
val changed: Boolean
11121142
)
11131143

11141144
private final class OneTimeCache[A >: Null] {

0 commit comments

Comments
 (0)