8000 Optimize memory consumption of Infos by gzm0 · Pull Request #4978 · scala-js/scala-js · GitHub
[go: up one dir, main page]
Skip to content

Optimize memory consumption of Infos #4978

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Apr 14, 2024
Merged

Optimize memory consumption of Infos #4978

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
175ec62
Use Array for byClass reachability info
gzm0 Apr 12, 2024
6dcd1b7
Move static module dependency tracking to class level info
gzm0 Apr 12, 2024
a54c022
Add a flag to class level info for dynamic import
gzm0 Apr 12, 2024
0d81e85
Store member reachability in a single field
gzm0 Apr 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 31 additions & 65 deletions linker/shared/src/main/scala/org/scalajs/linker/analyzer/Analyzer.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1079,7 +1079,8 @@ private class AnalyzerRun(config: CommonPhaseConfig, initial: Boolean,
} else if (!_isInstantiated.getAndSet(true)) {

// TODO: Why is this not in subclassInstantiated()?
referenceFieldClasses(fieldsRead ++ fieldsWritten)
fieldsRead.foreach(referenceFieldClasses(_))
fieldsWritten.foreach(referenceFieldClasses(_))

if (isScalaClass) {
accessData()
Expand Down Expand Up @@ -1208,12 +1209,6 @@ private class AnalyzerRun(config: CommonPhaseConfig, initial: Boolean,
}
}

def callMethodStatically(namespacedMethodName: NamespacedMethodName)(
implicit from: From): Unit = {
callMethodStatically(namespacedMethodName.namespace,
namespacedMethodName.methodName)
}

def callMethodStatically(namespace: MemberNamespace,
methodName: MethodName)(
implicit from: From): Unit = {
Expand All @@ -1225,16 +1220,14 @@ private class AnalyzerRun(config: CommonPhaseConfig, initial: Boolean,
lookupMethod(methodName).reachStatic()
}

def readFields(names: List[FieldName])(implicit from: From): Unit = {
names.foreach(_fieldsRead.update(_, ()))
if (isInstantiated)
referenceFieldClasses(names)
}

def writeFields(names: List[FieldName])(implicit from: From): Unit = {
names.foreach(_fieldsWritten.update(_, ()))
def reachField(info: Infos.FieldReachable)(implicit from: From): Unit = {
val fieldName = info.fieldName
if (info.read)
_fieldsRead.update(fieldName, ())
if (info.written)
_fieldsWritten.update(fieldName, ())
if (isInstantiated)
referenceFieldClasses(names)
referenceFieldClasses(fieldName)
}

def useJSNativeMember(name: MethodName)(
Expand All @@ -1251,8 +1244,7 @@ private class AnalyzerRun(config: CommonPhaseConfig, initial: Boolean,
maybeJSNativeLoadSpec
}

private def referenceFieldClasses(fieldNames: Iterable[FieldName])(
implicit from: From): Unit = {
private def referenceFieldClasses(fieldName: FieldName)(implicit from: From): Unit = {
assert(isInstantiated)

/* Reach referenced classes of non-static fields
Expand All @@ -1261,7 +1253,6 @@ private class AnalyzerRun(config: CommonPhaseConfig, initial: Boolean,
* site will not reference the classes in the final JS code.
*/
for {
fieldName <- fieldNames
className <- data.referencedFieldClasses.get(fieldName)
} {
lookupClass(className)(_ => ())
Expand Down Expand Up @@ -1432,61 +1423,36 @@ private class AnalyzerRun(config: CommonPhaseConfig, initial: Boolean,
if ((flags & ReachabilityInfoInClass.FlagStaticallyReferenced) != 0) {
moduleUnit.addStaticDependency(className)
}
}

/* Since many of the lists below are likely to be empty, we always
* test `!list.isEmpty` before calling `foreach` or any other
* processing, avoiding closure allocations.
*/

if (!dataInClass.fieldsRead.isEmpty) {
clazz.readFields(dataInClass.fieldsRead)
}

if (!dataInClass.fieldsWritten.isEmpty) {
clazz.writeFields(dataInClass.fieldsWritten)
if ((flags & ReachabilityInfoInClass.FlagDynamicallyReferenced) != 0) {
if (isNoModule)
_errors ::= DynamicImportWithoutModuleSupport(from)
else
moduleUnit.addDynamicDependency(className)
}
}

if (!dataInClass.staticFieldsRead.isEmpty) {
moduleUnit.addStaticDependency(className)
dataInClass.staticFieldsRead.foreach(
clazz._staticFieldsRead.update(_, ()))
}
if (dataInClass.memberInfos != null) {
dataInClass.memberInfos.foreach {
case field: Infos.FieldReachable =>
clazz.reachField(field)

if (!dataInClass.staticFieldsWritten.isEmpty) {
moduleUnit.addStaticDependency(className)
dataInClass.staticFieldsWritten.foreach(
clazz._staticFieldsWritten.update(_, ()))
}
case Infos.StaticFieldReachable(fieldName, read, written) =>
if (read)
clazz._staticFieldsRead.update(fieldName, ())
if (written)
clazz._staticFieldsWritten.update(fieldName, ())

if (!dataInClass.methodsCalled.isEmpty) {
// Do not add to staticDependencies: We call these on the object.
for (methodName <- dataInClass.methodsCalled)
clazz.callMethod(methodName)
}
case Infos.MethodReachable(methodName) =>
clazz.callMethod(methodName)

if (!dataInClass.methodsCalledStatically.isEmpty) {
moduleUnit.addStaticDependency(className)
for (methodName <- dataInClass.methodsCalledStatically)
clazz.callMethodStatically(methodName)
}
case Infos.MethodStaticallyReachable(namespace, methodName) =>
clazz.callMethodStatically(namespace, methodName)

if (!dataInClass.methodsCalledDynamicImport.isEmpty) {
if (isNoModule) {
_errors ::= DynamicImportWithoutModuleSupport(from)
} else {
moduleUnit.addDynamicDependency(className)
// In terms of reachability, a dynamic import call is just a static call.
for (methodName <- dataInClass.methodsCalledDynamicImport)
clazz.callMethodStatically(methodName)
case Infos.JSNativeMemberReachable(methodName) =>
clazz.useJSNativeMember(methodName).foreach(addLoadSpec(moduleUnit, _))
}
}

if (!dataInClass.jsNativeMembersUsed.isEmpty) {
for (member <- dataInClass.jsNativeMembersUsed)
clazz.useJSNativeMember(member)
.foreach(addLoadSpec(moduleUnit, _))
}
}
}

Expand Down
111 changes: 75 additions & 36 deletions linker/shared/src/main/scala/org/scalajs/linker/analyzer/Infos.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ object Infos {
)

final class ReachabilityInfo private[Infos] (
val byClass: List[ReachabilityInfoInClass],
val byClass: Array[ReachabilityInfoInClass],
val globalFlags: ReachabilityInfo.Flags
)

Expand All @@ -109,14 +109,12 @@ object Infos {
/** Things from a given class that are reached by one method. */
final class ReachabilityInfoInClass private[Infos] (
val className: ClassName,
val fieldsRead: List[FieldName],
val fieldsWritten: List[FieldName],
val staticFieldsRead: List[FieldName],
val staticFieldsWritten: List[FieldName],
val methodsCalled: List[MethodName],
val methodsCalledStatically: List[NamespacedMethodName],
val methodsCalledDynamicImport: List[NamespacedMethodName],
val jsNativeMembersUsed: List[MethodName],
/* We use a single field for all members to reduce memory consumption:
* Typically, there are very few members reached in a single
* ReachabilityInfoInClass, so the overhead of having a field per type
* becomes significant in terms of memory usage.
*/
val memberInfos: Array[MemberReachabilityInfo], // nullable!
val flags: ReachabilityInfoInClass.Flags
)

Expand All @@ -132,8 +130,41 @@ object Infos {
final val FlagInstanceTestsUsed = 1 << 2
final val FlagClassDataAccessed = 1 << 3
final val FlagStaticallyReferenced = 1 << 4
final val FlagDynamicallyReferenced = 1 << 5
}

sealed trait MemberReachabilityInfo

final case class FieldReachable private[Infos] (
val fieldName: FieldName,
val read: Boolean = false,
val written: Boolean = false
) extends MemberReachabilityInfo

final case class StaticFieldReachable private[Infos] (
val fieldName: FieldName,
val read: Boolean = false,
val written: Boolean = false
) extends MemberReachabilityInfo

final case class MethodReachable private[Infos] (
val methodName: MethodName
) extends MemberReachabilityInfo

final case class MethodStaticallyReachable private[Infos] (
val namespace: MemberNamespace,
val methodName: MethodName
) extends MemberReachabilityInfo

object MethodStaticallyReachable {
private[Infos] def apply(m: NamespacedMethodName): MethodStaticallyReachable =
MethodStaticallyReachable(m.namespace, m.methodName)
}

final case class JSNativeMemberReachable private[Infos] (
val methodName: MethodName
) extends MemberReachabilityInfo

final class ClassInfoBuilder(
private val className: ClassName,
private val kind: ClassKind,
Expand Down Expand Up @@ -374,52 +405,63 @@ object Infos {
setFlag(ReachabilityInfo.FlagUsedExponentOperator)

def result(): ReachabilityInfo =
new ReachabilityInfo(byClass.valuesIterator.map(_.result()).toList, flags)
new ReachabilityInfo(byClass.valuesIterator.map(_.result()).toArray, flags)
}

final class ReachabilityInfoInClassBuilder(val className: ClassName) {
private val fieldsRead = mutable.Set.empty[FieldName]
private val fieldsWritten = mutable.Set.empty[FieldName]
private val staticFieldsRead = mutable.Set.empty[FieldName]
private val staticFieldsWritten = mutable.Set.empty[FieldName]
private val fieldsUsed = mutable.Map.empty[FieldName, FieldReachable]
private val staticFieldsUsed = mutable.Map.empty[FieldName, StaticFieldReachable]
private val methodsCalled = mutable.Set.empty[MethodName]
private val methodsCalledStatically = mutable.Set.empty[NamespacedMethodName]
private val methodsCalledDynamicImport = mutable.Set.empty[NamespacedMethodName]
private val jsNativeMembersUsed = mutable.Set.empty[MethodName]
private var flags: ReachabilityInfoInClass.Flags = 0

def addFieldRead(field: FieldName): this.type = {
fieldsRead += field
fieldsUsed(field) = fieldsUsed
.getOrElse(field, FieldReachable(field))
.copy(read = true)
this
}

def addFieldWritten(field: FieldName): this.type = {
fieldsWritten += field
fieldsUsed(field) = fieldsUsed
.getOrElse(field, FieldReachable(field))
.copy(written = true)
this
}

def addStaticFieldRead(field: FieldName): this.type = {
staticFieldsRead += field
staticFieldsUsed(field) = staticFieldsUsed
.getOrElse(field, StaticFieldReachable(field))
.copy(read = true)
setStaticallyReferenced()
this
}

def addStaticFieldWritten(field: FieldName): this.type = {
staticFieldsWritten += field
staticFieldsUsed(field) = staticFieldsUsed
.getOrElse(field, StaticFieldReachable(field))
.copy(written = true)
setStaticallyReferenced()
this
}

def addMethodCalled(method: MethodName): this.type = {
methodsCalled += method
// Do not call setStaticallyReferenced: We call these methods on the object.
this
}

def addMethodCalledStatically(method: NamespacedMethodName): this.type = {
methodsCalledStatically += method
setStaticallyReferenced()
this
}

def addMethodCalledDynamicImport(method: NamespacedMethodName): this.type = {
methodsCalledDynamicImport += method
// In terms of reachability, a dynamic import call is just a static call.
methodsCalledStatically += method
setFlag(ReachabilityInfoInClass.FlagDynamicallyReferenced)
this
}

Expand Down Expand Up @@ -449,23 +491,20 @@ object Infos {
setFlag(ReachabilityInfoInClass.FlagStaticallyReferenced)

def result(): ReachabilityInfoInClass = {
new ReachabilityInfoInClass(
className,
fieldsRead = toLikelyEmptyList(fieldsRead),
fieldsWritten = toLikelyEmptyList(fieldsWritten),
staticFieldsRead = toLikelyEmptyList(staticFields 80FB Read),
staticFieldsWritten = toLikelyEmptyList(staticFieldsWritten),
methodsCalled = toLikelyEmptyList(methodsCalled),
methodsCalledStatically = toLikelyEmptyList(methodsCalledStatically),
methodsCalledDynamicImport = toLikelyEmptyList(methodsCalledDynamicImport),
jsNativeMembersUsed = toLikelyEmptyList(jsNativeMembersUsed),
flags = flags
)
}
val memberInfos: Array[MemberReachabilityInfo] = (
fieldsUsed.valuesIterator ++
staticFieldsUsed.valuesIterator ++
methodsCalled.iterator.map(MethodReachable(_)) ++
methodsCalledStatically.iterator.map(MethodStaticallyReachable(_)) ++
jsNativeMembersUsed.iterator.map(JSNativeMemberReachable(_))
).toArray

val memberInfosOrNull =
if (memberInfos.isEmpty) null
else memberInfos

private def toLikelyEmptyList[A](set: mutable.Set[A]): List[A] =
if (set.isEmpty) Nil
else set.toList
new ReachabilityInfoInClass(className, memberInfosOrNull, flags)
}
}

/** Generates the [[MethodInfo]] of a
Expand Down
0