diff --git a/metals/src/main/scala/scala/meta/internal/metals/Configs.scala b/metals/src/main/scala/scala/meta/internal/metals/Configs.scala index 272608b8094..4b079f23e42 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/Configs.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/Configs.scala @@ -371,6 +371,28 @@ object Configs { } } + final case class MbtConfig( + importGeneratedSources: Boolean, + semanticdbCacheEnabled: Boolean, + semanticdbCacheMaxSize: Int, + ) + + object MbtConfig { + val defaultMaxSize: Int = Int.MaxValue + def default: MbtConfig = + MbtConfig(false, false, defaultMaxSize) + def fromConfig( + importGeneratedSources: Option[Boolean], + semanticdbCacheEnabled: Option[Boolean], + semanticdbCacheMaxSize: Option[Int], + ): MbtConfig = + MbtConfig( + importGeneratedSources.getOrElse(false), + semanticdbCacheEnabled.getOrElse(false), + semanticdbCacheMaxSize.getOrElse(defaultMaxSize), + ) + } + final case class ProtoOutlineProviderConfig(val value: String) { require(List("v1", "v2").contains(value), value) def isV1: Boolean = diff --git a/metals/src/main/scala/scala/meta/internal/metals/Directories.scala b/metals/src/main/scala/scala/meta/internal/metals/Directories.scala index f2f95de8614..5566c2cc92f 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/Directories.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/Directories.scala @@ -40,6 +40,8 @@ object Directories { RelativePath(".metals").resolve("rules") def explainedDiagnostics: RelativePath = RelativePath(".metals").resolve("explained-diagnostics") + def semanticdbCache: RelativePath = + RelativePath(".metals").resolve("semanticdb-cache") val stacktraceFilename = "stacktrace.scala" val dependenciesName = "dependencies" diff --git a/metals/src/main/scala/scala/meta/internal/metals/MetalsLspService.scala b/metals/src/main/scala/scala/meta/internal/metals/MetalsLspService.scala index d967282bbeb..652b8ff1fb8 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/MetalsLspService.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/MetalsLspService.scala @@ -592,6 +592,8 @@ abstract class MetalsLspService( languageClient, workDoneProgress, metrics, + folder, + () => userConfig, ) val referencesProvider: ReferenceProvider = new ReferenceProvider( diff --git a/metals/src/main/scala/scala/meta/internal/metals/UserConfiguration.scala b/metals/src/main/scala/scala/meta/internal/metals/UserConfiguration.scala index 37721afbeb0..c4614dc5c7d 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/UserConfiguration.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/UserConfiguration.scala @@ -20,6 +20,7 @@ import scala.meta.internal.metals.Configs.FallbackClasspathConfig import scala.meta.internal.metals.Configs.FallbackSourcepathConfig import scala.meta.internal.metals.Configs.JavaSymbolLoaderConfig import scala.meta.internal.metals.Configs.JavacServicesOverrides +import scala.meta.internal.metals.Configs.MbtConfig import scala.meta.internal.metals.Configs.ProtoOutlineProviderConfig import scala.meta.internal.metals.Configs.ProtobufLspConfig import scala.meta.internal.metals.Configs.RangeFormattingProviders @@ -123,7 +124,7 @@ case class UserConfiguration( defaultShell: Option[String] = None, startMcpServer: Boolean = false, mcpClient: Option[String] = None, - importGeneratedSourcesMbt: Boolean = false, + mbtConfig: MbtConfig = MbtConfig.default, ) { def isMbtDefinitionProviderEnabled: Boolean = @@ -368,8 +369,12 @@ case class UserConfiguration( optStringField("mcpClient", mcpClient), Some( ( - "importGeneratedSourcesMbt", - importGeneratedSourcesMbt, + "mbt", + Map( + "importGeneratedSources" -> mbtConfig.importGeneratedSources, + "semanticdbCacheEnabled" -> mbtConfig.semanticdbCacheEnabled, + "semanticdbCacheMaxSize" -> mbtConfig.semanticdbCacheMaxSize, + ).asJava, ) ), Some( @@ -931,7 +936,7 @@ object UserConfiguration { |""".stripMargin, ), UserConfigurationOption( - "import-generated-sources-mbt", + "mbt.import-generated-sources", "false", "true", "Import Generated Sources In MBT Builds", @@ -940,6 +945,28 @@ object UserConfiguration { |""".stripMargin, isBoolean = true, ), + UserConfigurationOption( + "mbt.semanticdb-cache-enabled", + "false", + "true", + "Enable MBT Semanticdb Cache", + """|If enabled, Metals will persist semanticdb documents to disk in the + |`.metals/semanticdb-cache` directory. This can improve performance for + |find references and implementations operations in MBT mode by avoiding + |recalculating semanticdb when files haven't changed. + |""".stripMargin, + isBoolean = true, + ), + UserConfigurationOption( + "mbt.semanticdb-cache-max-size", + Int.MaxValue.toString(), + "2000", + "MBT Semanticdb In-Memory Cache Size Limit", + """|Maximum number of semanticdb documents to keep in the in-memory cache. + |When this limit is exceeded, the least recently used documents are evicted. + |A higher value uses more memory but can improve performance for large projects. + |""".stripMargin, + ), ) def listOptions: String = @@ -1029,9 +1056,15 @@ object UserConfiguration { ) def getBooleanKey(key: String): Option[Boolean] = + getBooleanKeyOnObj(key, json) + + def getBooleanKeyOnObj( + key: String, + currentObject: JsonObject, + ): Option[Boolean] = getKey( key, - json, + currentObject, { value => Try(value.getAsBoolean()) .fold( @@ -1043,8 +1076,12 @@ object UserConfiguration { ) }, ) + def getIntKey(key: String): Option[Int] = - getStringKey(key).flatMap { value => + getIntKeyOnObj(key, json) + + def getIntKeyOnObj(key: String, currentObject: JsonObject): Option[Int] = + getStringKeyOnObj(key, currentObject).flatMap { value => Try(value.toInt) match { case Failure(_) => errors += s"Not a number: '$value'" @@ -1470,8 +1507,18 @@ object UserConfiguration { val mcpClient = getStringKey("mcp-client") - val importGeneratedSourcesMbt = - getBooleanKey("import-generated-sources-mbt").getOrElse(false) + val mbtSubKey = getSubKey("mbt") + val mbtConfig = MbtConfig.fromConfig( + mbtSubKey + .flatMap(getBooleanKeyOnObj("import-generated-sources", _)) + .orElse(getBooleanKey("import-generated-sources-mbt")), + mbtSubKey + .flatMap(getBooleanKeyOnObj("semanticdb-cache-enabled", _)) + .orElse(getBooleanKey("mbt-semanticdb-cache")), + mbtSubKey + .flatMap(getIntKeyOnObj("semanticdb-cache-max-size", _)) + .orElse(getIntKey("mbt-semanticdb-cache-max-size")), + ) if (errors.isEmpty) { Right( @@ -1537,7 +1584,7 @@ object UserConfiguration { defaultShell, startMcpServer, mcpClient, - importGeneratedSourcesMbt, + mbtConfig, ) ) } else { diff --git a/metals/src/main/scala/scala/meta/internal/metals/mbt/MbtReferenceProvider.scala b/metals/src/main/scala/scala/meta/internal/metals/mbt/MbtReferenceProvider.scala index dfd8fef217a..1431ec5407a 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/mbt/MbtReferenceProvider.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/mbt/MbtReferenceProvider.scala @@ -1,11 +1,13 @@ package scala.meta.internal.metals.mbt +import java.nio.file.Files import java.time.Duration +import java.util.Collections import java.util.concurrent.CancellationException import java.util.concurrent.TimeoutException +import java.util.{LinkedHashMap => JLinkedHashMap} import scala.annotation.tailrec -import scala.collection.concurrent.TrieMap import scala.collection.mutable import scala.collection.mutable.Buffer import scala.concurrent.Await @@ -20,6 +22,7 @@ import scala.meta.internal.metals.AdjustRange import scala.meta.internal.metals.Buffers import scala.meta.internal.metals.BuildTargets import scala.meta.internal.metals.Compilers +import scala.meta.internal.metals.Directories import scala.meta.internal.metals.EmptyCancelToken import scala.meta.internal.metals.MetalsEnrichments._ import scala.meta.internal.metals.ReferencesResult @@ -28,6 +31,7 @@ import scala.meta.internal.metals.Synthetics import scala.meta.internal.metals.TaskProgress import scala.meta.internal.metals.Time import scala.meta.internal.metals.Timer +import scala.meta.internal.metals.UserConfiguration import scala.meta.internal.metals.WorkDoneProgress import scala.meta.internal.metals.clients.language.MetalsLanguageClient import scala.meta.internal.metals.noAdjustRange @@ -55,8 +59,14 @@ class MbtReferenceProvider( languageClient: MetalsLanguageClient, workDoneProgress: WorkDoneProgress, metrics: MonitoringClient, + workspace: AbsolutePath, + userConfig: () => UserConfiguration, )(implicit ec: ExecutionContext) { - private val cache = new TextDocumentCache() + private val cache = new TextDocumentCache( + workspace.resolve(Directories.semanticdbCache), + () => userConfig().mbtConfig.semanticdbCacheEnabled, + () => userConfig().mbtConfig.semanticdbCacheMaxSize, + ) // When looking for usages of a method, we don't visit supermethods from these // types because it would result in a ton of noisy results. This list should @@ -683,8 +693,67 @@ class MbtReferenceProvider( // presentation compilers when MBT is available and provides all sources and // dependencies through the fallback compiler. In BSP mode, SemanticDBs are // tried first and per-target presentation compilers are used as a fallback. - private class TextDocumentCache { - private val cache = TrieMap.empty[AbsolutePath, s.TextDocument] + private class TextDocumentCache( + cacheDir: AbsolutePath, + isPersistenceEnabled: () => Boolean, + maxCacheSize: () => Int, + ) { + private val cache: java.util.Map[AbsolutePath, s.TextDocument] = + Collections.synchronizedMap( + new JLinkedHashMap[AbsolutePath, s.TextDocument]( + 16, // initial capacity + 0.75f, // load factor + true, // accessOrder - enables LRU ordering + ) { + override def removeEldestEntry( + eldest: java.util.Map.Entry[AbsolutePath, s.TextDocument] + ): Boolean = size() > maxCacheSize() + } + ) + + private def diskCachePath(path: AbsolutePath): AbsolutePath = { + val relativePath = path.parent.toRelative(workspace) + val semanticdbPath = cacheDir.resolve(relativePath) + val name = path.filename + ".semanticdb" + semanticdbPath.resolve(name) + } + + private def loadFromDisk( + path: AbsolutePath, + md5: String, + ): Option[s.TextDocument] = { + if (!isPersistenceEnabled()) None + else { + val cachePath = diskCachePath(path) + if (cachePath.exists) { + try { + val bytes = Files.readAllBytes(cachePath.toNIO) + val doc = s.TextDocument.parseFrom(bytes) + if (doc.md5 == md5) Some(doc) + else None + } catch { + case e: Exception => + scribe.debug(s"Failed to load cached document for $path", e) + None + } + } else None + } + } + private def saveToDisk( + path: AbsolutePath, + doc: s.TextDocument, + ): Unit = { + if (isPersistenceEnabled()) + try { + val cachePath = diskCachePath(path) + cachePath.parent.createDirectories() + Files.write(cachePath.toNIO, doc.toByteArray) + } catch { + case e: Exception => + scribe.debug(s"Failed to save cached document for $path", e) + } + } + def indexSingle(path: AbsolutePath): s.TextDocument = { index(Seq(path)).documents.headOption.getOrElse { scribe.warn(s"references: no document found for $path") @@ -700,12 +769,19 @@ class MbtReferenceProvider( }.toMap val toIndex = pathsWithMd5.filter { case (path, md5) => - cache.get(path) match { + Option(cache.get(path)) match { case Some(doc) if doc.md5 == md5 => docs += doc false case _ => - true + loadFromDisk(path, md5) match { + case Some(doc) => + docs += doc + cache.put(path, doc) + false + case None => + true + } } } if (toIndex.isEmpty) { @@ -719,6 +795,7 @@ class MbtReferenceProvider( val docAdjusted = doc.withUri(path.toURI.toString) docs += docAdjusted cache.put(path, docAdjusted) + saveToDisk(path, docAdjusted) case _ => needCompiler += path } @@ -753,13 +830,12 @@ class MbtReferenceProvider( } } val result = fetchDocs(maxRetries = 3) - result.foreach { doc => - doc.uri.toAbsolutePathSafe match { - case Some(path) => - cache.put(path, doc) - case None => - scribe.warn(s"references: no path found for ${doc.uri}") - } + for { + doc <- result + path <- doc.uri.toAbsolutePathSafe + } { + cache.put(path, doc) + saveToDisk(path, doc) } docs ++= result s.TextDocuments(documents = docs.toSeq) diff --git a/metals/src/main/scala/scala/meta/internal/metals/mbt/importer/BazelMbtImporter.scala b/metals/src/main/scala/scala/meta/internal/metals/mbt/importer/BazelMbtImporter.scala index 37c83c8527c..b07ac2c379f 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/mbt/importer/BazelMbtImporter.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/mbt/importer/BazelMbtImporter.scala @@ -80,7 +80,7 @@ abstract class BazelMbtImporter( targetsXmlDump = new BazelTargetsXmlDump(targetsXmlQueryOutput) srcs = targetsXmlDump.getLabels("srcs") (genSrcOutputsByTarget, genSrcLabels) <- - if (userConfig().importGeneratedSourcesMbt) + if (userConfig().mbtConfig.importGeneratedSources) queryGenSrcOutputsByTarget(srcs) else Future.successful( diff --git a/tests/unit/src/test/scala/tests/UserConfigurationSuite.scala b/tests/unit/src/test/scala/tests/UserConfigurationSuite.scala index 7a32c01c173..87bc3b1b0ca 100644 --- a/tests/unit/src/test/scala/tests/UserConfigurationSuite.scala +++ b/tests/unit/src/test/scala/tests/UserConfigurationSuite.scala @@ -14,6 +14,7 @@ import scala.meta.internal.metals.Configs.BatchSemanticdbConfig import scala.meta.internal.metals.Configs.FallbackClasspathConfig import scala.meta.internal.metals.Configs.FallbackSourcepathConfig import scala.meta.internal.metals.Configs.JavacServicesOverrides +import scala.meta.internal.metals.Configs.MbtConfig import scala.meta.internal.metals.Configs.TurbineRecompileDelayConfig import scala.meta.internal.metals.Configs.WorkspaceSymbolProviderConfig import scala.meta.internal.metals.InlayHintsOption @@ -410,6 +411,7 @@ class UserConfigurationSuite extends BaseSuite { scalaImportsPlacement = ScalaImportsPlacement.SMART, batchSemanticdbCompilerInstances = BatchSemanticdbConfig(4), promptBuildImport = true, + mbtConfig = MbtConfig(false, true, 1000), ) val json = nonDefault.toString() @@ -523,7 +525,11 @@ class UserConfigurationSuite extends BaseSuite { }, "enableBestEffort": false, "startMcpServer": false, - "importGeneratedSourcesMbt": false + "mbt": { + "importGeneratedSources": false, + "semanticdbCacheEnabled": true, + "semanticdbCacheMaxSize": 1000 + } }""", ) val roundtripJson = UserConfiguration.parse(json) @@ -602,7 +608,9 @@ class UserConfigurationSuite extends BaseSuite { |default-shell string "" Full path to the shell executable to be used as the default |start-mcp-server boolean false Start MCP server |mcp-client string "" MCP Client Name - |import-generated-sources-mbt boolean false Import Generated Sources In MBT Builds""".stripMargin + |mbt.import-generated-sources boolean false Import Generated Sources In MBT Builds + |mbt.semanticdb-cache-enabled boolean false Enable MBT Semanticdb Cache + |mbt.semanticdb-cache-max-size string ${Int.MaxValue.toString.padTo(15, ' ')} MBT Semanticdb In-Memory Cache Size Limit""".stripMargin assertNoDiff(obtained, expected) } @@ -696,4 +704,36 @@ class UserConfigurationSuite extends BaseSuite { ) { obtained => assert(obtained.targetBuildTool == Some("sbt")) } + + checkOK( + "mbt-config-old-format", + """ + |{ + | "import-generated-sources-mbt": true, + | "mbt-semanticdb-cache": true, + | "mbt-semanticdb-cache-max-size": "500" + |} + """.stripMargin, + ) { obtained => + assert(obtained.mbtConfig.importGeneratedSources == true) + assert(obtained.mbtConfig.semanticdbCacheEnabled == true) + assert(obtained.mbtConfig.semanticdbCacheMaxSize == 500) + } + + checkOK( + "mbt-config-new-format", + """ + |{ + | "mbt": { + | "import-generated-sources": true, + | "semanticdb-cache-enabled": true, + | "semanticdb-cache-max-size": "500" + | } + |} + """.stripMargin, + ) { obtained => + assert(obtained.mbtConfig.importGeneratedSources == true) + assert(obtained.mbtConfig.semanticdbCacheEnabled == true) + assert(obtained.mbtConfig.semanticdbCacheMaxSize == 500) + } } diff --git a/tests/unit/src/test/scala/tests/mbt/MbtReferenceSpec.scala b/tests/unit/src/test/scala/tests/mbt/MbtReferenceSpec.scala index 60b47255e30..d5ac23c404a 100644 --- a/tests/unit/src/test/scala/tests/mbt/MbtReferenceSpec.scala +++ b/tests/unit/src/test/scala/tests/mbt/MbtReferenceSpec.scala @@ -1,7 +1,10 @@ package tests.mbt import scala.meta.internal.metals.AutoImportBuildKind +import scala.meta.internal.metals.Configs.MbtConfig +import scala.meta.internal.metals.Directories import scala.meta.internal.metals.InitializationOptions +import scala.meta.internal.metals.MetalsEnrichments._ import scala.meta.internal.metals.UserConfiguration import scala.meta.internal.metals.mbt.MbtBuildServer @@ -20,6 +23,70 @@ class MbtReferenceSuite extends MbtReferenceSpec { ) } +class MbtReferenceCacheSuite + extends BaseMbtReferenceSuite("mbt-reference-cache") { + + override def userConfig: UserConfiguration = + super.userConfig.copy( + fallbackScalaVersion = Some(BuildInfo.scalaVersion), + preferredBuildServer = Some(MbtBuildServer.name), + automaticImportBuild = AutoImportBuildKind.All, + mbtConfig = MbtConfig( + importGeneratedSources = false, + semanticdbCacheEnabled = true, + semanticdbCacheMaxSize = 1000, + ), + ) + + testLSP("semanticdb-cache-files-created") { + cleanWorkspace() + val a = "a/src/main/scala/a/Upstream.java" + val b = "a/src/main/scala/a/Downstream.java" + for { + _ <- initialize( + s"""|/.metals/mbt.json + |{} + |/$a + |package a; + |public class Upstream { + | public static String greeting = "Hello, World!"; + |} + |/$b + |package a; + |public class Downstream { + | public static void main(String[] args) { + | System.out.println(Upstream.greeting); + | } + |} + |""".stripMargin + ) + _ <- server.didOpenAndFocus(a) + _ <- server.assertReferencesSubquery( + a, + "String greeti@@ng", + """|a/src/main/scala/a/Downstream.java:4:33: reference + | System.out.println(Upstream.greeting); + | ^^^^^^^^ + |a/src/main/scala/a/Upstream.java:3:24: reference + | public static String greeting = "Hello, World!"; + | ^^^^^^^^ + |""".stripMargin, + ) + cacheDir = workspace.resolve(Directories.semanticdbCache) + _ = assert(cacheDir.exists, s"Cache directory should exist: $cacheDir") + cacheFiles = cacheDir.listRecursive.filter(_.isFile).toList + _ = println(s"Cache files: ${cacheFiles.mkString("\n")}") + _ = assertNoDiff( + cacheFiles.sortBy(_.toString).mkString("\n"), + s""" + |${cacheDir}/a/src/main/scala/a/Downstream.java.semanticdb + |${cacheDir}/a/src/main/scala/a/Upstream.java.semanticdb + |""".stripMargin, + ) + } yield () + } +} + class BspReferenceSuite extends MbtReferenceSpec { override def withMbt: Boolean = false