Add a dependency tracking mechanism

modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-no-compile/build.sbt

@@ -4,5 +4,5 @@ lazy val foo = (project in file("foo"))
 
 lazy val bar = (project in file("bar"))
   .enablePlugins(Smithy4sCodegenPlugin)
-  .settings(Compile / smithy4sLocalJars := Nil)
+  .settings(Compile / smithy4sInternalDependenciesAsJars := Nil)
   .dependsOn(foo)

modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/bar/src/main/scala/Test.scala

@@ -20,6 +20,8 @@ import foo._
 
 object BarTest {
 
-  def main(args: Array[String]): Unit = println(Bar(Some(Foo(Some(1)))))
+  def main(args: Array[String]): Unit = {
+    println(Bar(Some(Foo(Some(1)))))
+  }
 
 }

modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/baz/src/main/scala/Test.scala

@@ -0,0 +1,28 @@
+/*
+ *  Copyright 2021-2022 Disney Streaming
+ *
+ *  Licensed under the Tomorrow Open Source Technology License, Version 1.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *     https://disneystreaming.github.io/TOST-1.0.txt
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package baz
+
+import foo._
+import bar._
+
+object BarTest {
+
+  def main(args: Array[String]): Unit = {
+    println(Baz(Some(Foo(Some(1)))))
+  }
+
+}

modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/baz/src/main/smithy/bar.smithy

@@ -0,0 +1,12 @@
+$version: "2.0"
+
+namespace baz
+
+use foo#Foo
+
+// Checking that Foo can be found by virtue of the upstream `bar` project
+// defined as a compile-scope library dependency was published with an indication
+// in the manifest that it used the `foo` project for code generation.
+structure Baz {
+  foo: Foo
+}

modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/build.sbt

@@ -1,3 +1,5 @@
+import smithy4s.codegen.BuildInfo.smithyVersion
+
 ThisBuild / scalaVersion := "2.13.10"
 ThisBuild / version := "0.0.1-SNAPSHOT"
 ThisBuild / organization := "foobar"
@@ -17,3 +19,11 @@ lazy val bar = (project in file("bar"))
       "foobar" %% "foo" % version.value % Smithy4sCompile
     )
   )
+
+lazy val baz = (project in file("baz"))
+  .enablePlugins(Smithy4sCodegenPlugin)
+  .settings(
+    libraryDependencies ++= Seq(
+      "foobar" %% "bar" % version.value
+    )
+  )

modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/test

@@ -3,6 +3,12 @@
 > bar/compile
 $ exists bar/target/scala-2.13/src_managed/main/bar/Bar.scala
 $ absent bar/target/scala-2.13/src_managed/main/foo/Foo.scala
+> bar/publishLocal
+> baz/compile
+$ exists baz/target/scala-2.13/src_managed/main/baz/Baz.scala
+$ absent baz/target/scala-2.13/src_managed/main/foo/Foo.scala
+$ absent baz/target/scala-2.13/src_managed/main/bar/Bar.scala
 
 # check if code can run, this can reveal runtime issues# such as initialization errors
 > bar/run
+> baz/run

modules/codegen-plugin/src/smithy4s/codegen/Smithy4sCodegenPlugin.scala

@@ -17,6 +17,7 @@
 package smithy4s.codegen
 
 import sbt.Keys._
+import java.util.jar.JarFile
 import sbt.util.CacheImplicits._
 import sbt.{fileJsonFormatter => _, _}
 import JsonConverters._
@@ -62,13 +63,44 @@ object Smithy4sCodegenPlugin extends AutoPlugin {
         "Sets whether this project should be used as a Smithy library by packaging the Smithy specs in the resulting jar"
       )
 
-    val smithy4sLocalJars =
+    val smithy4sExternalDependenciesAsJars =
       taskKey[Seq[File]](
         List(
-          "List of jars for local dependencies that should be used as sources of Smithy specs.",
-          "Namespaces that were used for code generation in the upstream dependencies will be excluded from code generation in this project.",
-          "By default, this includes the jars produced by packaging your project's build dependencies, so they'll need to be compiled for the codegen task to run.",
-          "You can clear this (set to an empty list) if your Smithy specs don't have dependencies on other module."
+          "List of jars for external dependencies that should be added to the classpath used by Smithy4s during code-generation",
+          "The smithy files and smithy validators contained by these jars are included in the Smithy4s code-generation process",
+          "Namespaces that were used for code generation in these dependencies will be excluded from code generation in this project.",
+          "By default, this includes the jars resulting from the resolution of library dependencies annotated with the `Smithy4s` configuration"
+        ).mkString(" ")
+      )
+
+    val smithy4sExternalCodegenDependenciesAsJars =
+      taskKey[Seq[File]](
+        List(
+          "List of jars that external dependencies indicate having used during their own code-generation process",
+          "If a project using Smithy4s depends on a library that contains Smithy4s generated code, local Smithy files might need the jars",
+          "that were used by Smithy4s when the library in question was built. By default, Smithy4s adds a line in the jar manifests of the",
+          "projects it is enabled on to inform downstream projects of the jars they might want to pull during their own code-generation.",
+          "This is different from a transitive compile dependency, as the jars used during code-generation might not necessarily end up",
+          "on the compile class-path of a project"
+        ).mkString(" ")
+      )
+
+    val smithy4sInternalDependenciesAsJars =
+      taskKey[Seq[File]](
+        List(
+          "List of jars of internal dependencies that should be added to the classpath used by Smithy4s during code-generation",
+          "The smithy files and smithy validators contained by these jars are included in the Smithy4s code-generation process",
+          "Namespaces that were used for code generation in these dependencies will be excluded from code generation in this project.",
+          "By default, this includes the jars produced by packaging this project's local dependencies, which implies these should compile for the codegen task to run",
+          "This can be set to an empty list to prevent the inclusion of local dependencies during the code-gen process"
+        ).mkString(" ")
+      )
+
+    val smithy4sAllDependenciesAsJars =
+      taskKey[Seq[File]](
+        List(
+          "List of all jars for internal and external dependencies that should be used as sources of Smithy specs.",
+          "Namespaces that were used for code generation in these upstream dependencies will be excluded from code generation in this project."
         ).mkString(" ")
       )
 
@@ -93,6 +125,8 @@ object Smithy4sCodegenPlugin extends AutoPlugin {
     smithy4sVersion := BuildInfo.version
   )
 
+  private val SMITHY4S_DEPENDENCIES = "smithy4sDependencies"
+
   override def projectConfigurations: Seq[Configuration] = Seq(Smithy4s)
 
   // Use this with any configuration to enable the codegen in it.
@@ -103,8 +137,22 @@ object Smithy4sCodegenPlugin extends AutoPlugin {
     config / smithy4sResourceDir := (config / resourceManaged).value,
     config / smithy4sCodegen := cachedSmithyCodegen(config).value,
     config / smithy4sSmithyLibrary := true,
-    config / smithy4sLocalJars := (config / internalDependencyAsJars).value
-      .map(_.data),
+    config / smithy4sExternalDependenciesAsJars := {
+      val updateReport =
+        (config / update).value +: (config / transitiveUpdate).value
+      findCodeGenDependencies(updateReport)
+    },
+    config / smithy4sInternalDependenciesAsJars := {
+      (config / internalDependencyAsJars).value.map(_.data)
+    },
+    config / smithy4sExternalCodegenDependenciesAsJars := {
+      smithy4sFetchUpstreamCodegenDependencies(config).value
+    },
+    config / smithy4sAllDependenciesAsJars := {
+      (config / smithy4sExternalDependenciesAsJars).value ++
+        (config / smithy4sExternalCodegenDependenciesAsJars).value ++
+        (config / smithy4sInternalDependenciesAsJars).value
+    },
     config / sourceGenerators += (config / smithy4sCodegen).map(
       _.filter(_.ext == "scala")
     ),
@@ -113,7 +161,24 @@ object Smithy4sCodegenPlugin extends AutoPlugin {
     ),
     config / cleanFiles += (config / smithy4sOutputDir).value,
     config / cleanFiles += (config / smithy4sResourceDir).value,
-    config / smithy4sModelTransformers := List.empty
+    config / smithy4sModelTransformers := List.empty,
+    config / packageBin / packageOptions += {
+      // This piece of logic aims at tracking the dependencies that Smithy4s used to generate
+      // code at build time, in the manifest of the jar. This helps automatically pulling
+      // the corresponding jars and prevents the users from having to search
+      import java.util.jar.Manifest
+      val manifest = new Manifest
+      val scalaBin = scalaBinaryVersion.?.value
+      val deps = libraryDependencies.value
+        .filter(_.configurations.exists(_.contains(Smithy4s.name)))
+        .flatMap(moduleIdEncode(_, scalaBin))
+        .mkString(",")
+
+      manifest
+        .getMainAttributes()
+        .put(new java.util.jar.Attributes.Name(SMITHY4S_DEPENDENCIES), deps)
+      Package.JarManifest(manifest)
+    }
   )
 
   override lazy val projectSettings =
@@ -125,7 +190,7 @@ object Smithy4sCodegenPlugin extends AutoPlugin {
 
   private def findCodeGenDependencies(
       updateReports: Seq[UpdateReport]
-  ): List[os.Path] =
+  ): List[File] =
     for {
       markerConfig <- List(Smithy4s)
       updateReport <- updateReports.toList
@@ -134,9 +199,78 @@ object Smithy4sCodegenPlugin extends AutoPlugin {
       artifactFile <- module.artifacts
     } yield {
       val (_, file) = artifactFile
-      os.Path(file)
+      file
+    }
+
+  private def moduleIdEncode(
+      moduleId: ModuleID,
+      scalaBinaryVersion: Option[String]
+  ): List[String] = {
+    (moduleId.crossVersion, scalaBinaryVersion) match {
+      case (Disabled, _) =>
+        List(s"${moduleId.organization}:${moduleId.name}:${moduleId.revision}")
+      case (_: Binary, Some(sbv)) =>
+        List(
+          s"${moduleId.organization}:${moduleId.name}_${sbv}:${moduleId.revision}"
+        )
+      case (_, _) => Nil
+    }
+  }
+
+  /**
+   * Retrieves the smithy4sDependencies that compile-dependencies may have listed
+   * in their jar manifests when they were packaged.
+   */
+  private def smithy4sFetchUpstreamCodegenDependencies(
+      config: Configuration
+  ): Def.Initialize[Task[Seq[File]]] =
+    Def.task {
+      val smithy4sDependencies =
+        (config / externalDependencyClasspath).value
+          .map(_.data)
+          .flatMap(extract)
+      def getJars(ids: Seq[ModuleID]): Seq[File] = {
+        val syntheticModule =
+          organization.value % (name.value + "-smithy4s-resolution") % version.value
+        val depRes = (update / dependencyResolution).value
+        val updc = (update / updateConfiguration).value
+        val uwconfig = (update / unresolvedWarningConfiguration).value
+        val modDescr = depRes.moduleDescriptor(
+          syntheticModule,
+          ids.toVector,
+          None
+        )
+
+        depRes
+          .update(
+            modDescr,
+            updc,
+            uwconfig,
+            streams.value.log
+          )
+          .map(_.allFiles)
+          .fold(uw => throw uw.resolveException, identity)
+      }
+      getJars(smithy4sDependencies)
     }
 
+  private lazy val simple = raw"([^:]*):([^:]*):([^:]*)".r
+  private lazy val cross = raw"([^:]*)::([^:]*):([^:]*)".r
+  private def extract(jarFile: java.io.File): Seq[ModuleID] = {
+    val jar = new JarFile(jarFile)
+    Option(
+      jar.getManifest().getMainAttributes().getValue("smithy4sDependencies")
+    ).toList.flatMap { listString =>
+      listString
+        .split(",")
+        .collect {
+          case cross(org, art, version)  => org %% art % version
+          case simple(org, art, version) => org % art % version
+        }
+        .toList
+    }
+  }
+
   def cachedSmithyCodegen(conf: Configuration) = Def.task {
     val inputFiles =
       Option((conf / smithy4sInputDirs).value).toSeq.flatten
@@ -148,15 +282,8 @@ object Smithy4sCodegenPlugin extends AutoPlugin {
       (conf / smithy4sAllowedNamespaces).?.value.map(_.toSet)
     val excludedNamespaces =
       (conf / smithy4sExcludedNamespaces).?.value.map(_.toSet)
-    val updateReport = {
-      (conf / update).value +: (conf / transitiveUpdate).value
-    }
-
-    val localDependencyJars =
-      (conf / smithy4sLocalJars).value.map(os.Path(_)).toList
-
-    val externalDependencyJars = findCodeGenDependencies(updateReport)
-    val localJars = localDependencyJars ++ externalDependencyJars
+    val localJars =
+      (conf / smithy4sAllDependenciesAsJars).value.map(os.Path(_)).toList
     val res =
       (conf / resolvers).value.toList.collect { case m: MavenRepository =>
         m.root
@@ -174,7 +301,7 @@ object Smithy4sCodegenPlugin extends AutoPlugin {
       output = os.Path(outputPath),
       resourceOutput = os.Path(resourceOutputPath),
       skip = skipSet,
-      discoverModels = true, // we need protocol here
+      discoverModels = false, // we need protocol here
       allowedNS = allowedNamespaces,
       excludedNS = excludedNamespaces,
       repositories = res,