Use the new stdlib when generating scaladoc (#23744)

In this PR, we use the new stdlib project to generate the scaladoc and
website of scala 3.
The previous build used `scala2-library-bootstrapped` to fetch have
tasty files available. We change this by now using the real, and to be,
TASTy files for the stdlib.
This changes will already take effect in the first run after the merge
of this PR.

Author: hamzaremmal

.github/workflows/scaladoc.yaml

@@ -44,6 +44,7 @@ jobs:
       - name: Compile and test
         run: |
           ./project/scripts/sbt scaladoc/test
+          ./project/scripts/sbt dist/Universal/stage
           ./project/scripts/cmdScaladocTests
 
       - name: Locally publish self

project/Build.scala

@@ -3199,7 +3199,7 @@ object Build {
       }.value,
 
       generateScalaDocumentation := Def.inputTaskDyn {
-        val majorVersion = (LocalProject("scala3-library-bootstrapped") / scalaBinaryVersion).value
+        val majorVersion = (`scala-library-bootstrapped` / scalaBinaryVersion).value
 
         val extraArgs = spaceDelimited("[<output-dir>] [--justAPI]").parsed
         val outputDirOverride = extraArgs.headOption.fold(identity[GenerationConfig](_))(newDir => {
@@ -3816,25 +3816,12 @@ object ScaladocConfigs {
 
   def dottyExternalMapping = ".*scala/.*::scaladoc3::https://dotty.epfl.ch/api/"
   def javaExternalMapping = ".*java/.*::javadoc::https://docs.oracle.com/javase/8/docs/api/"
-  def scalaSrcLink(v: String, s: String) = s"${s}github://scala/scala/v$v#src/library"
-  def dottySrcLink(v: String, sourcesPrefix: String = "", outputPrefix: String = "") =
-    sys.env.get("GITHUB_SHA") match {
-      case Some(sha) =>
-        s"${sourcesPrefix}github://${sys.env("GITHUB_REPOSITORY")}/$sha$outputPrefix"
-      case None => s"${sourcesPrefix}github://scala/scala3/$v$outputPrefix"
+  def defaultSourceLinks(version: String) = {
+    def dottySrcLink(v: String) = sys.env.get("GITHUB_SHA") match {
+      case Some(sha) => s"github://scala/scala3/$sha"
+      case None => s"github://scala/scala3/$v"
     }
-
-  def defaultSourceLinks(version: String = dottyNonBootstrappedVersion, refVersion: String = dottyVersion) = Def.task {
-    def stdLibVersion = stdlibVersion(NonBootstrapped)
-    def srcManaged(v: String, s: String) = s"out/bootstrap/scala2-library-bootstrapped/scala-$v/src_managed/main/$s-library-src"
-    SourceLinks(
-      List(
-        scalaSrcLink(stdLibVersion, srcManaged(version, "scala") + "="),
-        dottySrcLink(refVersion, "library/src=", "#library/src"),
-        dottySrcLink(refVersion),
-        "docs=github://scala/scala3/main#docs"
-      )
-    )
+    SourceLinks(List(dottySrcLink(version), "docs=github://scala/scala3/main#docs"))
   }
 
   lazy val DefaultGenerationSettings = Def.task {
@@ -3849,17 +3836,14 @@ object ScaladocConfigs {
     def skipById = SkipById(List(
       "scala.runtime.stdLibPatches",
       "scala.runtime.MatchCase",
-      "dotty.tools.tasty",
-      "dotty.tools.tasty.util",
-      "dotty.tools.tasty.besteffort"
     ))
     def projectFooter = ProjectFooter(s"Copyright (c) 2002-$currentYear, LAMP/EPFL")
     def defaultTemplate = DefaultTemplate("static-site-main")
     GenerationConfig(
       List(),
       ProjectVersion(projectVersion),
       GenerateInkuire(true),
-      defaultSourceLinks().value,
+      defaultSourceLinks(version = dottyVersion),
       skipByRegex,
       skipById,
       projectLogo,
@@ -3881,13 +3865,8 @@ object ScaladocConfigs {
     )
   }
 
-  lazy val DefaultGenerationConfig = Def.task {
-    def distLocation = (dist / Universal / stage).value
-    DefaultGenerationSettings.value
-  }
-
   lazy val Scaladoc = Def.task {
-    DefaultGenerationConfig.value
+    DefaultGenerationSettings.value
       .add(UseJavacp(true))
       .add(ProjectName("scaladoc"))
       .add(OutputDir("scaladoc/output/self"))
@@ -3898,7 +3877,7 @@ object ScaladocConfigs {
 
   lazy val Testcases = Def.task {
     val tastyRoots = (Test / Build.testcasesOutputDir).value
-    DefaultGenerationConfig.value
+    DefaultGenerationSettings.value
       .add(UseJavacp(true))
       .add(OutputDir("scaladoc/output/testcases"))
       .add(ProjectName("scaladoc testcases"))
@@ -3914,56 +3893,35 @@ object ScaladocConfigs {
   }
 
   lazy val Scala3 = Def.task {
-    val dottyJars: Seq[java.io.File] = Seq(
-      (`scala2-library-bootstrapped`/Compile/products).value,
-      (`scala3-library-bootstrapped`/Compile/products).value,
-      (`scala3-interfaces`/Compile/products).value,
-      (`tasty-core-bootstrapped`/Compile/products).value,
-    ).flatten
-
-    val roots = dottyJars.map(_.getAbsolutePath)
-
-    val managedSources =
-      (`scala2-library-bootstrapped`/Compile/sourceManaged).value / "scala-library-src"
-    val projectRoot = (ThisBuild/baseDirectory).value.toPath
-    val stdLibRoot = projectRoot.relativize(managedSources.toPath.normalize())
-    val docRootFile = stdLibRoot.resolve("rootdoc.txt")
-
-    val dottyManagesSources = (`scala3-library-bootstrapped`/Compile/baseDirectory).value
-
-    val tastyCoreSources = projectRoot.relativize((`tasty-core-bootstrapped`/Compile/scalaSource).value.toPath().normalize())
-
-    val dottyLibRoot = projectRoot.relativize(dottyManagesSources.toPath.normalize())
-    DefaultGenerationConfig.value
+    DefaultGenerationSettings.value
       .add(ProjectName("Scala 3"))
       .add(OutputDir(file("scaladoc/output/scala3").getAbsoluteFile.getAbsolutePath))
       .add(Revision("main"))
       .add(ExternalMappings(List(javaExternalMapping)))
-      .add(DocRootContent(docRootFile.toString))
+      .add(DocRootContent(((`scala-library-bootstrapped` / baseDirectory).value / "src" / "rootdoc.txt").toString))
       .add(CommentSyntax(List(
-        s"${dottyLibRoot}=markdown",
-        s"${stdLibRoot}=wiki",
-        s"${tastyCoreSources}=markdown",
+        //s"${dottyLibRoot}=markdown",
+        //s"${stdLibRoot}=wiki",
         "wiki"
       )))
       .add(VersionsDictionaryUrl("https://scala-lang.org/api/versions.json"))
       .add(DocumentSyntheticTypes(true))
-      .add(SnippetCompiler(List(
-        s"$dottyLibRoot/src/scala=compile",
-        s"$dottyLibRoot/src/scala/compiletime=compile",
-        s"$dottyLibRoot/src/scala/util=compile",
-        s"$dottyLibRoot/src/scala/util/control=compile"
-      )))
+      //.add(SnippetCompiler(List(
+        //s"$dottyLibRoot/src/scala=compile",
+        //s"$dottyLibRoot/src/scala/compiletime=compile",
+        //s"$dottyLibRoot/src/scala/util=compile",
+        //s"$dottyLibRoot/src/scala/util/control=compile"
+      //)))
       .add(SiteRoot("docs"))
       .add(ApiSubdirectory(true))
-      .withTargets(roots)
+      .withTargets((`scala-library-bootstrapped` / Compile / products).value.map(_.getAbsolutePath))
   }
 
   def stableScala3(version: String) = Def.task {
     val scalaLibrarySrc = s"out/bootstrap/scala2-library-bootstrapped/scala-$version-bin-SNAPSHOT-nonbootstrapped/src_managed"
     val dottyLibrarySrc = "library/src"
     Scala3.value
-      .add(defaultSourceLinks(version + "-bin-SNAPSHOT-nonbootstrapped", version).value)
+      .add(defaultSourceLinks(version = version))
       .add(ProjectVersion(version))
       .add(SnippetCompiler(
         List(

scaladoc/src/dotty/tools/scaladoc/tasty/ClassLikeSupport.scala

@@ -140,7 +140,7 @@ trait ClassLikeSupport:
     if summon[DocContext].args.generateInkuire then doInkuireStuff(classDef)
 
     if signatureOnly then baseMember else baseMember.copy(
-        members = classDef.extractPatchedMembers.sortBy(m => (m.name, m.kind.name)),
+        members = classDef.extractMembers.sortBy(m => (m.name, m.kind.name)),
         selfType = selfType,
         companion = classDef.getCompanion
     )
@@ -278,31 +278,6 @@ trait ClassLikeSupport:
         inherited.flatMap(s => parseInheritedMember(c)(s))
     }
 
-    /** Extracts members while taking Dotty logic for patching the stdlib into account. */
-    def extractPatchedMembers: Seq[Member] = {
-      val ownMembers = c.extractMembers
-      def extractPatchMembers(sym: Symbol) = {
-        // NOTE for some reason scala.language$.experimental$ class doesn't show up here, so we manually add the name
-        val ownMemberDRIs = ownMembers.iterator.map(_.name).toSet + "experimental$"
-        sym.tree.asInstanceOf[ClassDef]
-          .membersToDocument.filterNot(m => ownMemberDRIs.contains(m.symbol.name))
-          .flatMap(parseMember(c))
-      }
-      c.symbol.fullName match {
-        case "scala.Predef$" =>
-          ownMembers ++
-          extractPatchMembers(qctx.reflect.Symbol.requiredClass("scala.runtime.stdLibPatches.Predef$"))
-        case "scala.language$" =>
-          ownMembers ++
-          extractPatchMembers(qctx.reflect.Symbol.requiredModule("scala.runtime.stdLibPatches.language").moduleClass)
-        case "scala.language$.experimental$" =>
-          ownMembers ++
-          extractPatchMembers(qctx.reflect.Symbol.requiredModule("scala.runtime.stdLibPatches.language.experimental").moduleClass)
-        case _ => ownMembers
-      }
-
-    }
-
     def getTreeOfFirstParent: Option[Tree] =
       c.getParentsAsTreeSymbolTuples.headOption.map(_._1)
 

scaladoc/src/dotty/tools/scaladoc/tasty/TastyParser.scala

@@ -135,7 +135,7 @@ case class ScaladocTastyInspector()(using ctx: DocContext) extends Inspector:
   def mergeAnyRefAliasAndObject(parser: TastyParser) =
     import parser.qctx.reflect._
     val javaLangObjectDef = defn.ObjectClass.tree.asInstanceOf[ClassDef]
-    val objectMembers = parser.extractPatchedMembers(javaLangObjectDef)
+    val objectMembers = javaLangObjectDef.extractMembers
     val aM = parser.parseTypeDef(
       defn.AnyRefClass.tree.asInstanceOf[TypeDef],
       defn.AnyClass.tree.asInstanceOf[ClassDef],