Add a dependency tracking mechanism

modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/build.sbt

@@ -15,6 +15,9 @@ lazy val foo = (project in file("foo"))
 lazy val bar = (project in file("bar"))
   .enablePlugins(Smithy4sCodegenPlugin)
   .settings(
+    // Bar refers to foo explicitly in its ivy deps, and upon publishing,
+    // this information is stored in the manifest of bar's jar, for downstream
+    // consumption
     libraryDependencies ++= Seq(
       "foobar" %% "foo" % version.value % Smithy4sCompile
     )
@@ -23,6 +26,9 @@ lazy val bar = (project in file("bar"))
 lazy val baz = (project in file("baz"))
   .enablePlugins(Smithy4sCodegenPlugin)
   .settings(
+    // baz depend on bar, and an assumption is made that baz may depend on the same smithy models
+    // that bar depended on for its own codegen. Therefore, these are retrieved from bar's manifest,
+    // resolved and added to the list of jars to seek smithy models from during code generation
     libraryDependencies ++= Seq(
       "foobar" %% "bar" % version.value
     )

modules/codegen-plugin/src/smithy4s/codegen/Smithy4sCodegenPlugin.scala

@@ -20,6 +20,7 @@ import sbt.Keys._
 import java.util.jar.JarFile
 import sbt.util.CacheImplicits._
 import sbt.{fileJsonFormatter => _, _}
+import smithy4s.codegen.SMITHY4S_DEPENDENCIES
 import JsonConverters._
 
 object Smithy4sCodegenPlugin extends AutoPlugin {
@@ -125,8 +126,6 @@ object Smithy4sCodegenPlugin extends AutoPlugin {
     smithy4sVersion := BuildInfo.version
   )
 
-  private val SMITHY4S_DEPENDENCIES = "smithy4sDependencies"
-
   override def projectConfigurations: Seq[Configuration] = Seq(Smithy4s)
 
   // Use this with any configuration to enable the codegen in it.
@@ -259,7 +258,7 @@ object Smithy4sCodegenPlugin extends AutoPlugin {
   private def extract(jarFile: java.io.File): Seq[ModuleID] = {
     val jar = new JarFile(jarFile)
     Option(
-      jar.getManifest().getMainAttributes().getValue("smithy4sDependencies")
+      jar.getManifest().getMainAttributes().getValue(SMITHY4S_DEPENDENCIES)
     ).toList.flatMap { listString =>
       listString
         .split(",")

modules/codegen/src/smithy4s/codegen/package.scala

@@ -37,6 +37,8 @@ import scala.jdk.CollectionConverters._
 
 package object codegen {
 
+  val SMITHY4S_DEPENDENCIES = "smithy4sDependencies"
+
   val uuidShapeId = ShapeId.from("alloy#UUID")
 
   private[codegen] type LinesWithValue = ToLinesWithValue[_]

modules/mill-codegen-plugin/src/smithy4s/codegen/mill/Smithy4sModule.scala

@@ -23,6 +23,12 @@ import mill.define.Sources
 import mill.scalalib._
 import smithy4s.codegen.{CodegenArgs, Codegen => Smithy4s, FileType}
 import smithy4s.codegen.BuildInfo
+import smithy4s.codegen.SMITHY4S_DEPENDENCIES
+import mill.modules.Jvm
+import mill.scalalib.CrossVersion.Binary
+import mill.scalalib.CrossVersion.Constant
+import mill.scalalib.CrossVersion.Full
+import java.util.jar.JarFile
 
 trait Smithy4sModule extends ScalaModule {
 
@@ -54,6 +60,22 @@ trait Smithy4sModule extends ScalaModule {
     smithy4sDefaultIvyDeps() ++ smithy4sIvyDeps()
   }
 
+  override def manifest: T[Jvm.JarManifest] = T {
+    val m = super.manifest()
+    val deps = smithy4sIvyDeps().iterator.toList.flatMap { d =>
+      val mod = d.dep.module
+      val org = mod.organization.value
+      val name = mod.name.value
+      val version = d.dep.version
+      d.cross match {
+        case Binary(_)      => List(s"$org::$name:$version")
+        case Constant(_, _) => List(s"$org:$name:$version")
+        case Full(_)        => Nil
+      }
+    }
+    m.add(SMITHY4S_DEPENDENCIES -> deps.mkString(","))
+  }
+
   def smithy4sInternalDependenciesAsJars: T[List[PathRef]] = T {
     T.traverse(moduleDeps)(_.jar)
       .map(_.toList.map(_.path).map(PathRef(_)))
@@ -79,8 +101,29 @@ trait Smithy4sModule extends ScalaModule {
       .flatten
   }
 
-  def smithy4sResolvedIvyDeps: T[Agg[PathRef]] = T {
-    resolveDeps(T.task { smithy4sTransitiveIvyDeps() })()
+  def smithy4sResolvedIvyDeps: T[Agg[PathRef]] =
+    resolveDeps(smithy4sTransitiveIvyDeps)
+
+  def smithy4sExternalCodegenIvyDeps: T[Agg[Dep]] = T {
+    resolveDeps(transitiveIvyDeps)().flatMap { pathRef =>
+      val jarFile = new JarFile(pathRef.path.toIO)
+      val deps = Option(
+        jarFile
+          .getManifest()
+          .getMainAttributes()
+          .getValue(SMITHY4S_DEPENDENCIES)
+      ).toList.flatMap { listString =>
+        listString.split(",").toList.map(dep => ivy"$dep")
+      }
+      Agg.from(deps)
+    }
+  }
+
+  def smithy4sResolvedExternalCodegenIvyDeps: T[Agg[PathRef]] =
+    resolveDeps(smithy4sExternalCodegenIvyDeps)
+
+  def smithy4sAllDependenciesAsJars: T[Agg[PathRef]] = T {
+    smithy4sInternalDependenciesAsJars() ++ smithy4sResolvedIvyDeps() ++ smithy4sResolvedExternalCodegenIvyDeps()
   }
 
   def smithy4sCodegen: T[(PathRef, PathRef)] = T {
@@ -100,10 +143,8 @@ trait Smithy4sModule extends ScalaModule {
 
     val skipSet = skipResources ++ skipOpenApi
 
-    val resolvedDeps = smithy4sResolvedIvyDeps().iterator.map(_.path).toList
-
-    val localJars = smithy4sAllDependenciesAsJars().map(_.path)
-    val allLocalJars = localJars ++ resolvedDeps
+    val allLocalJars =
+      smithy4sAllDependenciesAsJars().map(_.path).iterator.to(List)
 
     val args = CodegenArgs(
       specs = specFiles.toList,

modules/mill-codegen-plugin/test/resources/multimodule-staged/bar/smithy/bar.smithy

@@ -0,0 +1,10 @@
+$version: "2.0"
+
+namespace bar
+
+use foo#Foo
+
+// Checking that Foo can be found by virtue of the bar project depending on the foo project
+structure Bar {
+  foo: Foo
+}

modules/mill-codegen-plugin/test/resources/multimodule-staged/bar/src/main/scala/Test.scala

@@ -0,0 +1,27 @@
+/*
+ *  Copyright 2021-2022 Disney Streaming
+ *
+ *  Licensed under the Tomorrow Open Source Technology License, Version 1.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *     https://disneystreaming.github.io/TOST-1.0.txt
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package bar
+
+import foo._
+
+object BarTest {
+
+  def main(args: Array[String]): Unit = {
+    println(Bar(Some(Foo(Some(1)))))
+  }
+
+}

modules/mill-codegen-plugin/test/resources/multimodule-staged/baz/smithy/bar.smithy

@@ -0,0 +1,12 @@
+$version: "2.0"
+
+namespace baz
+
+use foo#Foo
+
+// Checking that Foo can be found by virtue of the upstream `bar` project
+// defined as a compile-scope library dependency was published with an indication
+// in the manifest that it used the `foo` project for code generation.
+structure Baz {
+  foo: Foo
+}

modules/mill-codegen-plugin/test/resources/multimodule-staged/baz/src/main/scala/Test.scala

@@ -0,0 +1,28 @@
+/*
+ *  Copyright 2021-2022 Disney Streaming
+ *
+ *  Licensed under the Tomorrow Open Source Technology License, Version 1.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *     https://disneystreaming.github.io/TOST-1.0.txt
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package baz
+
+import foo._
+import bar._
+
+object BarTest {
+
+  def main(args: Array[String]): Unit = {
+    println(Baz(Some(Foo(Some(1)))))
+  }
+
+}

modules/mill-codegen-plugin/test/resources/multimodule-staged/foo/smithy/foo.smithy

@@ -0,0 +1,12 @@
+$version: "2.0"
+
+namespace foo
+
+use alloy#uuidFormat
+
+structure Foo {
+  a: Integer
+}
+
+@uuidFormat
+string MyUUID

modules/mill-codegen-plugin/test/src/smithy4s/codegen/mill/Smithy4sModuleSpec.scala

@@ -22,6 +22,8 @@ import mill._
 import munit.Location
 import sourcecode.FullName
 import java.nio.file.Paths
+import mill.scalalib.publish.PomSettings
+import mill.scalalib.publish.VersionControl
 
 class Smithy4sModuleSpec extends munit.FunSuite {
   private val resourcePath =
@@ -207,14 +209,101 @@ class Smithy4sModuleSpec extends munit.FunSuite {
     taskWorks(bar.smithy4sCodegen, barEv)
   }
 
+  test("multi-module staged codegen works".only) {
+
+    trait Base
+        extends testKit.BaseModule
+        with SbtModule
+        with Smithy4sModule
+        with PublishModule {
+      override def scalaVersion = "2.13.10"
+      def pomSettings: T[PomSettings] = PomSettings(
+        "foo",
+        "foobar",
+        "http://foobar",
+        Seq.empty,
+        VersionControl(),
+        Seq.empty
+      )
+      def publishVersion: T[String] = "0.0.1-SNAPSHOT"
+
+    }
+
+    object foo extends Base {
+      override def scalaVersion = "2.13.10"
+      override def ivyDeps = Agg(coreDep)
+      override def millSourcePath = resourcePath / "multimodule-staged" / "foo"
+    }
+
+    object bar extends Base {
+      override def scalaVersion = "2.13.10"
+      // Bar refers to foo explicitly in its ivy deps, and upon publishing,
+      // this information is stored in the manifest of bar's jar, for downstream
+      // consumption
+      override def smithy4sIvyDeps =
+        Agg(ivy"${pomSettings().organization}::foo:${publishVersion()}")
+      override def ivyDeps = T(smithy4sIvyDeps())
+      override def millSourcePath = resourcePath / "multimodule-staged" / "bar"
+    }
+
+    object baz extends Base {
+      override def scalaVersion = "2.13.10"
+      // baz depend on bar, and an assumption is made that baz may depend on the same smithy models
+      // that bar depended on for its own codegen. Therefore, these are retrieved from bar's manifest,
+      // resolved and added to the list of jars to seek smithy models from during code generation
+      override def ivyDeps =
+        Agg(ivy"${pomSettings().organization}::bar:${publishVersion()}")
+      override def millSourcePath = resourcePath / "multimodule-staged" / "baz"
+    }
+
+    val fooEv =
+      testKit.staticTestEvaluator(foo)(FullName("multi-module-staged-foo"))
+    val barEv =
+      testKit.staticTestEvaluator(bar)(FullName("multi-module-staged-bar"))
+    val bazEv =
+      testKit.staticTestEvaluator(baz)(FullName("multi-module-staged-bar"))
+
+    taskWorks(foo.publishLocal(), fooEv)
+    taskWorks(bar.compile, barEv)
+
+    checkFileExist(
+      barEv.outPath / "smithy4sOutputDir.dest" / "scala" / "bar" / "Bar.scala",
+      shouldExist = true
+    )
+    checkFileExist(
+      barEv.outPath / "smithy4sOutputDir.dest" / "scala" / "foo" / "Foo.scala",
+      shouldExist = false
+    )
+
+    taskWorks(bar.publishLocal(), barEv)
+    taskWorks(baz.compile, bazEv)
+
+    checkFileExist(
+      bazEv.outPath / "smithy4sOutputDir.dest" / "scala" / "baz" / "Baz.scala",
+      shouldExist = true
+    )
+    checkFileExist(
+      bazEv.outPath / "smithy4sOutputDir.dest" / "scala" / "foo" / "Baz.scala",
+      shouldExist = false
+    )
+    checkFileExist(
+      bazEv.outPath / "smithy4sOutputDir.dest" / "scala" / "bar" / "Bar.scala",
+      shouldExist = false
+    )
+
+    taskWorks(bar.run(), barEv)
+    taskWorks(baz.run(), bazEv)
+
+  }
+
   private def compileWorks(
       sm: ScalaModule,
       testEvaluator: testKit.TestEvaluator
   )(implicit loc: Location) =
     taskWorks(sm.compile, testEvaluator)
 
   private def taskWorks[A](
-      task: T[A],
+      task: mill.define.Task[A],
       testEvaluator: testKit.TestEvaluator
   )(implicit loc: Location) = {
     val result = testEvaluator(task).map(_._1)