8000 DO NOT MERGE: On disk cache by gzm0 · Pull Request #4963 · scala-js/scala-js · GitHub
[go: up one dir, main page]
Skip to content

DO NOT MERGE: On disk cache #4963

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 4 commits into
base: main
Choose a base branch
from
Draft

DO NOT MERGE: On disk cache #4963

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
fac3b52
Remove `force` parameter from OutputWriter methods
gzm0 Mar 10, 2024
788b7b5
Move module changed check to OutputWriter
gzm0 Mar 10, 2024
143a24d
No nested post transforms
gzm0 Mar 10, 2024
639d93b
wip
gzm0 Mar 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Move module changed check to OutputWriter
  • Loading branch information
@gzm0
gzm0 committed Mar 10, 2024
commit 788b7b5ddbe6ce4048032aa24a88f0606c1c3a11
10 changes: 6 additions & 4 deletions linker/jvm/src/main/scala/org/scalajs/linker/backend/closure/ClosureLinkerBackend.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,15 +216,17 @@ final class ClosureLinkerBackend(config: LinkerBackendImpl.Config)
writer.write(footer)
}

protected def writeModuleWithoutSourceMap(moduleID: ModuleID): Option[ByteBuffer] = {
protected def moduleChanged(moduleID: ModuleID): Boolean = true // no incremental support

protected def writeModuleWithoutSourceMap(moduleID: ModuleID): ByteBuffer = {
val jsFileWriter = new ByteArrayOutputStream()
val jsFileStrWriter = new java.io.OutputStreamWriter(jsFileWriter, StandardCharsets.UTF_8)
writeCode(jsFileStrWriter)
jsFileStrWriter.flush()
Some(ByteBuffer.wrap(jsFileWriter.toByteArray()))
ByteBuffer.wrap(jsFileWriter.toByteArray())
}

protected def writeModuleWithSourceMap(moduleID: ModuleID): Option[(ByteBuffer, ByteBuffer)] = {
protected def writeModuleWithSourceMap(moduleID: ModuleID): (ByteBuffer, ByteBuffer) = {
val jsFileURI = OutputPatternsImpl.jsFileURI(config.outputPatterns, moduleID.id)
val sourceMapURI = OutputPatternsImpl.sourceMapURI(config.outputPatterns, moduleID.id)

Expand All @@ -241,7 +243,7 @@ final class ClosureLinkerBackend(config: LinkerBackendImpl.Config)
sourceMap.appendTo(sourceMapStrWriter, jsFileURI)
sourceMapStrWriter.flush()

Some((ByteBuffer.wrap(jsFileWriter.toByteArray()), ByteBuffer.wrap(sourceMapWriter.toByteArray())))
(ByteBuffer.wrap(jsFileWriter.toByteArray()), ByteBuffer.wrap(sourceMapWriter.toByteArray()))
}
}

Expand Down
71 changes: 33 additions & 38 deletions linker/shared/src/main/scala/org/scalajs/linker/backend/BasicLinkerBackend.scala
View file
Open in desktop
10000
Original file line number Diff line number Diff line change
Expand Up @@ -90,64 +90,59 @@ final class BasicLinkerBackend(config: LinkerBackendImpl.Config)
val allChanged = allChanged0 || config.minify

val writer = new OutputWriter(output, config, skipContentCheck) {
protected def writeModuleWithoutSourceMap(moduleID: ModuleID): Option[ByteBuffer] = {
protected def moduleChanged(moduleID: ModuleID): Boolean =
allChanged || emitterResult.body(moduleID)._2

protected def writeModuleWithoutSourceMap(moduleID: ModuleID): ByteBuffer = {
val cache = printedModuleSetCache.getModuleCache(moduleID)
val (trees, changed) = emitterResult.body(moduleID)
val (trees, _) = emitterResult.body(moduleID)

if (changed || allChanged) {
rewrittenModules.incrementAndGet()
rewrittenModules.incrementAndGet()

val jsFileWriter = new ByteArrayWriter(sizeHintFor(cache.getPreviousFinalJSFileSize()))
val jsFileWriter = new ByteArrayWriter(sizeHintFor(cache.getPreviousFinalJSFileSize()))

jsFileWriter.write(printedModuleSetCache.headerBytes)
jsFileWriter.writeASCIIString("'use strict';\n")
jsFileWriter.write(printedModuleSetCache.headerBytes)
jsFileWriter.writeASCIIString("'use strict';\n")

bodyPrinter.printWithoutSourceMap(trees, jsFileWriter)
bodyPrinter.printWithoutSourceMap(trees, jsFileWriter)

jsFileWriter.write(printedModuleSetCache.footerBytes)
jsFileWriter.write(printedModuleSetCache.footerBytes)

cache.recordFinalSizes(jsFileWriter.currentSize, 0)
Some(jsFileWriter.toByteBuffer())
} else {
None
}
cache.recordFinalSizes(jsFileWriter.currentSize, 0)
jsFileWriter.toByteBuffer()
}

protected def writeModuleWithSourceMap(moduleID: ModuleID): Option[(ByteBuffer, ByteBuffer)] = {
protected def writeModuleWithSourceMap(moduleID: ModuleID): (ByteBuffer, ByteBuffer) = {
val cache = printedModuleSetCache.getModuleCache(moduleID)
val (trees, changed) = emitterResult.body(moduleID)

if (changed || allChanged) {
rewrittenModules.incrementAndGet()
val (trees, _) = emitterResult.body(moduleID)

val jsFileWriter = new ByteArrayWriter(sizeHintFor(cache.getPreviousFinalJSFileSize()))
val sourceMapWriter = new ByteArrayWriter(sizeHintFor(cache.getPreviousFinalSourceMapSize()))
rewrittenModules.incrementAndGet()

val jsFileURI = OutputPatternsImpl.jsFileURI(config.outputPatterns, moduleID.id)
val sourceMapURI = OutputPatternsImpl.sourceMapURI(config.outputPatterns, moduleID.id)
val jsFileWriter = new ByteArrayWriter(sizeHintFor(cache.getPreviousFinalJSFileSize()))
val sourceMapWriter = new ByteArrayWriter(sizeHintFor(cache.getPreviousFinalSourceMapSize()))

val smWriter = new SourceMapWriter(sourceMapWriter, jsFileURI,
config.relativizeSourceMapBase)
val jsFileURI = OutputPatternsImpl.jsFileURI(config.outputPatterns, moduleID.id)
val sourceMapURI = OutputPatternsImpl.sourceMapURI(config.outputPatterns, moduleID.id)

jsFileWriter.write(printedModuleSetCache.headerBytes)
for (_ <- 0 until printedModuleSetCache.headerNewLineCount)
smWriter.nextLine()
val smWriter = new SourceMapWriter(sourceMapWriter, jsFileURI,
config.relativizeSourceMapBase)

jsFileWriter.writeASCIIString("'use strict';\n")
jsFileWriter.write(printedModuleSetCache.headerBytes)
for (_ <- 0 until printedModuleSetCache.headerNewLineCount)
smWriter.nextLine()

bodyPrinter.printWithSourceMap(trees, jsFileWriter, smWriter)
jsFileWriter.writeASCIIString("'use strict';\n")
smWriter.nextLine()

jsFileWriter.write(printedModuleSetCache.footerBytes)
jsFileWriter.write(("//# sourceMappingURL=" + sourceMapURI + "\n").getBytes(StandardCharsets.UTF_8))
bodyPrinter.printWithSourceMap(trees, jsFileWriter, smWriter)

smWriter.complete()
jsFileWriter.write(printedModuleSetCache.footerBytes)
jsFileWriter.write(("//# sourceMappingURL=" + sourceMapURI + "\n").getBytes(StandardCharsets.UTF_8))

cache.recordFinalSizes(jsFileWriter.currentSize, sourceMapWriter.currentSize)
Some((jsFileWriter.toByteBuffer(), sourceMapWriter.toByteBuffer()))
} else {
None
}
smWriter.complete()

cache.recordFinalSizes(jsFileWriter.currentSize, sourceMapWriter.currentSize)
(jsFileWriter.toByteBuffer(), sourceMapWriter.toByteBuffer())
}

private def sizeHintFor(previousSize: Int): Int =
Expand Down
44 changes: 23 additions & 21 deletions linker/shared/src/main/scala/org/scalajs/linker/backend/OutputWriter.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,11 @@ private[backend] abstract class OutputWriter(output: OutputDirectory,
private val outputImpl = OutputDirectoryImpl.fromOutputDirectory(output)
private val moduleKind = config.commonConfig.coreSpec.moduleKind

protected def writeModuleWithoutSourceMap(moduleID: ModuleID): Option[ByteBuffer]
protected def moduleChanged(moduleID: ModuleID): Boolean

protected def writeModuleWithSourceMap(moduleID: ModuleID): Option[(ByteBuffer, ByteBuffer)]
protected def writeModuleWithoutSourceMap(moduleID: ModuleID): ByteBuffer

protected def writeModuleWithSourceMap(moduleID: ModuleID): (ByteBuffer, ByteBuffer)

def write(moduleSet: ModuleSet)(implicit ec: ExecutionContext): Future[Report] = {
val ioThrottler = new IOThrottler(config.maxConcurrentWrites)
Expand Down Expand Up @@ -69,29 +71,29 @@ private[backend] abstract class OutputWriter(output: OutputDirectory,
val sourceMapFileName = OutputPatternsImpl.sourceMapFile(config.outputPatterns, moduleID.id)
val report = new ReportImpl.ModuleImpl(moduleID.id, jsFileName, Some(sourceMapFileName), moduleKind)

writeModuleWithSourceMap(moduleID) match {
case Some((code, sourceMap)) =>
for {
_ <- outputImpl.writeFull(jsFileName, code, skipContentCheck)
_ <- outputImpl.writeFull(sourceMapFileName, sourceMap, skipContentCheck)
} yield {
report
}
case None =>
Future.successful(report)
if (moduleChanged(moduleID)) {
val (code, sourceMap) = writeModuleWithSourceMap(moduleID)
for {
_ <- outputImpl.writeFull(jsFileName, code, skipContentCheck)
_ <- outputImpl.writeFull(sourceMapFileName, sourceMap, skipContentCheck)
} yield {
report
}
} else {
Future.successful(report)
}
} else {
val report = new ReportImpl.ModuleImpl(moduleID.id, jsFileName, None, moduleKind)

writeModuleWithoutSourceMap(moduleID) match {
case Some(code) =>
for {
_ <- outputImpl.writeFull(jsFileName, code, skipContentCheck)
} yield {
report
}
case None =>
Future.successful(report)
if (moduleChanged(moduleID)) {
val code = writeModuleWithoutSourceMap(moduleID)
for {
_ <- outputImpl.writeFull(jsFileName, code, skipContentCheck)
} yield {
report
}
} else {
Future.successful(report)
}
}
}
Expand Down
0