From b10343d07cb747783c2d0aad9b6a0e6859e169e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20M=C3=A9lois?= Date: Wed, 16 Nov 2022 12:37:51 +0100 Subject: [PATCH 1/9] Add a dependency tracking mechanism This facilitates the tracking of dependencies without imposing additional burden on the user to know what dependencies were used to generate code in upstream libraries/projects --- .../multimodule-no-compile/build.sbt | 2 +- .../bar/src/main/scala/Test.scala | 4 +- .../baz/src/main/scala/Test.scala | 28 +++ .../baz/src/main/smithy/bar.smithy | 12 ++ .../multimodule-staged/build.sbt | 10 ++ .../codegen-plugin/multimodule-staged/test | 6 + .../codegen/Smithy4sCodegenPlugin.scala | 167 +++++++++++++++--- .../markdown/01-overview/05-sharing-specs.md | 54 +++++- .../codegen/mill/Smithy4sModule.scala | 4 +- .../codegen/mill/Smithy4sModuleSpec.scala | 2 +- project/build.properties | 2 +- 11 files changed, 260 insertions(+), 31 deletions(-) create mode 100644 modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/baz/src/main/scala/Test.scala create mode 100644 modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/baz/src/main/smithy/bar.smithy diff --git a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-no-compile/build.sbt b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-no-compile/build.sbt index a04d45ec1..e14d589fe 100644 --- a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-no-compile/build.sbt +++ b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-no-compile/build.sbt @@ -4,5 +4,5 @@ lazy val foo = (project in file("foo")) lazy val bar = (project in file("bar")) .enablePlugins(Smithy4sCodegenPlugin) - .settings(Compile / smithy4sLocalJars := Nil) + .settings(Compile / smithy4sInternalDependenciesAsJars := Nil) .dependsOn(foo) diff --git a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/bar/src/main/scala/Test.scala b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/bar/src/main/scala/Test.scala index 60009722a..c61d7291c 100644 --- a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/bar/src/main/scala/Test.scala +++ b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/bar/src/main/scala/Test.scala @@ -20,6 +20,8 @@ import foo._ object BarTest { - def main(args: Array[String]): Unit = println(Bar(Some(Foo(Some(1))))) + def main(args: Array[String]): Unit = { + println(Bar(Some(Foo(Some(1))))) + } } diff --git a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/baz/src/main/scala/Test.scala b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/baz/src/main/scala/Test.scala new file mode 100644 index 000000000..1ee5acbc7 --- /dev/null +++ b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/baz/src/main/scala/Test.scala @@ -0,0 +1,28 @@ +/* + * Copyright 2021-2022 Disney Streaming + * + * Licensed under the Tomorrow Open Source Technology License, Version 1.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://disneystreaming.github.io/TOST-1.0.txt + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package baz + +import foo._ +import bar._ + +object BarTest { + + def main(args: Array[String]): Unit = { + println(Baz(Some(Foo(Some(1))))) + } + +} diff --git a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/baz/src/main/smithy/bar.smithy b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/baz/src/main/smithy/bar.smithy new file mode 100644 index 000000000..f447cb167 --- /dev/null +++ b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/baz/src/main/smithy/bar.smithy @@ -0,0 +1,12 @@ +$version: "2.0" + +namespace baz + +use foo#Foo + +// Checking that Foo can be found by virtue of the upstream `bar` project +// defined as a compile-scope library dependency was published with an indication +// in the manifest that it used the `foo` project for code generation. +structure Baz { + foo: Foo +} diff --git a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/build.sbt b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/build.sbt index f7c712c42..d30fae71c 100644 --- a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/build.sbt +++ b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/build.sbt @@ -1,3 +1,5 @@ +import smithy4s.codegen.BuildInfo.smithyVersion + ThisBuild / scalaVersion := "2.13.10" ThisBuild / version := "0.0.1-SNAPSHOT" ThisBuild / organization := "foobar" @@ -17,3 +19,11 @@ lazy val bar = (project in file("bar")) "foobar" %% "foo" % version.value % Smithy4sCompile ) ) + +lazy val baz = (project in file("baz")) + .enablePlugins(Smithy4sCodegenPlugin) + .settings( + libraryDependencies ++= Seq( + "foobar" %% "bar" % version.value + ) + ) diff --git a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/test b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/test index 9b6f61e5d..f5a813173 100644 --- a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/test +++ b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/test @@ -3,6 +3,12 @@ > bar/compile $ exists bar/target/scala-2.13/src_managed/main/bar/Bar.scala $ absent bar/target/scala-2.13/src_managed/main/foo/Foo.scala +> bar/publishLocal +> baz/compile +$ exists baz/target/scala-2.13/src_managed/main/baz/Baz.scala +$ absent baz/target/scala-2.13/src_managed/main/foo/Foo.scala +$ absent baz/target/scala-2.13/src_managed/main/bar/Bar.scala # check if code can run, this can reveal runtime issues# such as initialization errors > bar/run +> baz/run diff --git a/modules/codegen-plugin/src/smithy4s/codegen/Smithy4sCodegenPlugin.scala b/modules/codegen-plugin/src/smithy4s/codegen/Smithy4sCodegenPlugin.scala index 9102af081..be0803c06 100644 --- a/modules/codegen-plugin/src/smithy4s/codegen/Smithy4sCodegenPlugin.scala +++ b/modules/codegen-plugin/src/smithy4s/codegen/Smithy4sCodegenPlugin.scala @@ -17,6 +17,7 @@ package smithy4s.codegen import sbt.Keys._ +import java.util.jar.JarFile import sbt.util.CacheImplicits._ import sbt.{fileJsonFormatter => _, _} import JsonConverters._ @@ -62,13 +63,44 @@ object Smithy4sCodegenPlugin extends AutoPlugin { "Sets whether this project should be used as a Smithy library by packaging the Smithy specs in the resulting jar" ) - val smithy4sLocalJars = + val smithy4sExternalDependenciesAsJars = taskKey[Seq[File]]( List( - "List of jars for local dependencies that should be used as sources of Smithy specs.", - "Namespaces that were used for code generation in the upstream dependencies will be excluded from code generation in this project.", - "By default, this includes the jars produced by packaging your project's build dependencies, so they'll need to be compiled for the codegen task to run.", - "You can clear this (set to an empty list) if your Smithy specs don't have dependencies on other module." + "List of jars for external dependencies that should be added to the classpath used by Smithy4s during code-generation", + "The smithy files and smithy validators contained by these jars are included in the Smithy4s code-generation process", + "Namespaces that were used for code generation in these dependencies will be excluded from code generation in this project.", + "By default, this includes the jars resulting from the resolution of library dependencies annotated with the `Smithy4s` configuration" + ).mkString(" ") + ) + + val smithy4sExternalCodegenDependenciesAsJars = + taskKey[Seq[File]]( + List( + "List of jars that external dependencies indicate having used during their own code-generation process", + "If a project using Smithy4s depends on a library that contains Smithy4s generated code, local Smithy files might need the jars", + "that were used by Smithy4s when the library in question was built. By default, Smithy4s adds a line in the jar manifests of the", + "projects it is enabled on to inform downstream projects of the jars they might want to pull during their own code-generation.", + "This is different from a transitive compile dependency, as the jars used during code-generation might not necessarily end up", + "on the compile class-path of a project" + ).mkString(" ") + ) + + val smithy4sInternalDependenciesAsJars = + taskKey[Seq[File]]( + List( + "List of jars of internal dependencies that should be added to the classpath used by Smithy4s during code-generation", + "The smithy files and smithy validators contained by these jars are included in the Smithy4s code-generation process", + "Namespaces that were used for code generation in these dependencies will be excluded from code generation in this project.", + "By default, this includes the jars produced by packaging this project's local dependencies, which implies these should compile for the codegen task to run", + "This can be set to an empty list to prevent the inclusion of local dependencies during the code-gen process" + ).mkString(" ") + ) + + val smithy4sAllDependenciesAsJars = + taskKey[Seq[File]]( + List( + "List of all jars for internal and external dependencies that should be used as sources of Smithy specs.", + "Namespaces that were used for code generation in these upstream dependencies will be excluded from code generation in this project." ).mkString(" ") ) @@ -93,6 +125,8 @@ object Smithy4sCodegenPlugin extends AutoPlugin { smithy4sVersion := BuildInfo.version ) + private val SMITHY4S_DEPENDENCIES = "smithy4sDependencies" + override def projectConfigurations: Seq[Configuration] = Seq(Smithy4s) // Use this with any configuration to enable the codegen in it. @@ -103,8 +137,22 @@ object Smithy4sCodegenPlugin extends AutoPlugin { config / smithy4sResourceDir := (config / resourceManaged).value, config / smithy4sCodegen := cachedSmithyCodegen(config).value, config / smithy4sSmithyLibrary := true, - config / smithy4sLocalJars := (config / internalDependencyAsJars).value - .map(_.data), + config / smithy4sExternalDependenciesAsJars := { + val updateReport = + (config / update).value +: (config / transitiveUpdate).value + findCodeGenDependencies(updateReport) + }, + config / smithy4sInternalDependenciesAsJars := { + (config / internalDependencyAsJars).value.map(_.data) + }, + config / smithy4sExternalCodegenDependenciesAsJars := { + smithy4sFetchUpstreamCodegenDependencies(config).value + }, + config / smithy4sAllDependenciesAsJars := { + (config / smithy4sExternalDependenciesAsJars).value ++ + (config / smithy4sExternalCodegenDependenciesAsJars).value ++ + (config / smithy4sInternalDependenciesAsJars).value + }, config / sourceGenerators += (config / smithy4sCodegen).map( _.filter(_.ext == "scala") ), @@ -113,7 +161,24 @@ object Smithy4sCodegenPlugin extends AutoPlugin { ), config / cleanFiles += (config / smithy4sOutputDir).value, config / cleanFiles += (config / smithy4sResourceDir).value, - config / smithy4sModelTransformers := List.empty + config / smithy4sModelTransformers := List.empty, + config / packageBin / packageOptions += { + // This piece of logic aims at tracking the dependencies that Smithy4s used to generate + // code at build time, in the manifest of the jar. This helps automatically pulling + // the corresponding jars and prevents the users from having to search + import java.util.jar.Manifest + val manifest = new Manifest + val scalaBin = scalaBinaryVersion.?.value + val deps = libraryDependencies.value + .filter(_.configurations.exists(_.contains(Smithy4s.name))) + .flatMap(moduleIdEncode(_, scalaBin)) + .mkString(",") + + manifest + .getMainAttributes() + .put(new java.util.jar.Attributes.Name(SMITHY4S_DEPENDENCIES), deps) + Package.JarManifest(manifest) + } ) override lazy val projectSettings = @@ -125,7 +190,7 @@ object Smithy4sCodegenPlugin extends AutoPlugin { private def findCodeGenDependencies( updateReports: Seq[UpdateReport] - ): List[os.Path] = + ): List[File] = for { markerConfig <- List(Smithy4s) updateReport <- updateReports.toList @@ -134,9 +199,78 @@ object Smithy4sCodegenPlugin extends AutoPlugin { artifactFile <- module.artifacts } yield { val (_, file) = artifactFile - os.Path(file) + file + } + + private def moduleIdEncode( + moduleId: ModuleID, + scalaBinaryVersion: Option[String] + ): List[String] = { + (moduleId.crossVersion, scalaBinaryVersion) match { + case (Disabled, _) => + List(s"${moduleId.organization}:${moduleId.name}:${moduleId.revision}") + case (_: Binary, Some(sbv)) => + List( + s"${moduleId.organization}:${moduleId.name}_${sbv}:${moduleId.revision}" + ) + case (_, _) => Nil + } + } + + /** + * Retrieves the smithy4sDependencies that compile-dependencies may have listed + * in their jar manifests when they were packaged. + */ + private def smithy4sFetchUpstreamCodegenDependencies( + config: Configuration + ): Def.Initialize[Task[Seq[File]]] = + Def.task { + val smithy4sDependencies = + (config / externalDependencyClasspath).value + .map(_.data) + .flatMap(extract) + def getJars(ids: Seq[ModuleID]): Seq[File] = { + val syntheticModule = + organization.value % (name.value + "-smithy4s-resolution") % version.value + val depRes = (update / dependencyResolution).value + val updc = (update / updateConfiguration).value + val uwconfig = (update / unresolvedWarningConfiguration).value + val modDescr = depRes.moduleDescriptor( + syntheticModule, + ids.toVector, + None + ) + + depRes + .update( + modDescr, + updc, + uwconfig, + streams.value.log + ) + .map(_.allFiles) + .fold(uw => throw uw.resolveException, identity) + } + getJars(smithy4sDependencies) } + private lazy val simple = raw"([^:]*):([^:]*):([^:]*)".r + private lazy val cross = raw"([^:]*)::([^:]*):([^:]*)".r + private def extract(jarFile: java.io.File): Seq[ModuleID] = { + val jar = new JarFile(jarFile) + Option( + jar.getManifest().getMainAttributes().getValue("smithy4sDependencies") + ).toList.flatMap { listString => + listString + .split(",") + .collect { + case cross(org, art, version) => org %% art % version + case simple(org, art, version) => org % art % version + } + .toList + } + } + def cachedSmithyCodegen(conf: Configuration) = Def.task { val inputFiles = Option((conf / smithy4sInputDirs).value).toSeq.flatten @@ -148,15 +282,8 @@ object Smithy4sCodegenPlugin extends AutoPlugin { (conf / smithy4sAllowedNamespaces).?.value.map(_.toSet) val excludedNamespaces = (conf / smithy4sExcludedNamespaces).?.value.map(_.toSet) - val updateReport = { - (conf / update).value +: (conf / transitiveUpdate).value - } - - val localDependencyJars = - (conf / smithy4sLocalJars).value.map(os.Path(_)).toList - - val externalDependencyJars = findCodeGenDependencies(updateReport) - val localJars = localDependencyJars ++ externalDependencyJars + val localJars = + (conf / smithy4sAllDependenciesAsJars).value.map(os.Path(_)).toList val res = (conf / resolvers).value.toList.collect { case m: MavenRepository => m.root @@ -174,7 +301,7 @@ object Smithy4sCodegenPlugin extends AutoPlugin { output = os.Path(outputPath), resourceOutput = os.Path(resourceOutputPath), skip = skipSet, - discoverModels = true, // we need protocol here + discoverModels = false, // we need protocol here allowedNS = allowedNamespaces, excludedNS = excludedNamespaces, repositories = res, diff --git a/modules/docs/markdown/01-overview/05-sharing-specs.md b/modules/docs/markdown/01-overview/05-sharing-specs.md index 35ba1e80f..af5d00bab 100644 --- a/modules/docs/markdown/01-overview/05-sharing-specs.md +++ b/modules/docs/markdown/01-overview/05-sharing-specs.md @@ -76,7 +76,7 @@ You can opt out of this behavior: ```scala val b = project.settings( - Compile / smithy4sLocalJars := Nil + Compile / smithy4sInternalDependenciesAsJars := Nil )//... ``` @@ -85,13 +85,13 @@ val b = project.settings( ```scala object b extends Smithy4sModule { //... - override def smithy4sLocalJars = List.empty[PathRef] + override def smithy4sInternalDependenciesAsJars = List.empty[PathRef] } ``` This will not only remove the need for compilation (for the purposes of codegen), but also remove any visibility of the Smithy files in the **local** dependencies of your project (**local** meaning they're defined in the same build). -You can use the same setting, `smithy4sLocalJars`, to add additional JARs containing Smithy specs - just keep in mind that remote dependencies (`libraryDependencies`) are added automatically! +You can use the same setting, `smithy4sInternalDependenciesAsJars`, to add additional JARs containing Smithy specs - just keep in mind that remote dependencies (`libraryDependencies`) are added automatically! ### A word of warning @@ -151,17 +151,61 @@ libraryDependencies += "organisation" % "artifact" % "version" % "smithy4s,compi ### Mill ```scala - def compileAndCodegenDeps = T(Agg(ivy"organisation:artifact:version")) def ivyDeps = T(super.ivyDeps() ++ compileAndCodegenDeps()) def smithy4sIvyDeps = T(super.smithy4sIvyDeps() ++ compileAndCodegenDeps()) - ``` ### Consequence Because the upstream usage of Smithy4s will have resulted in the creation of metadata tracking the namespaces that were already generated, the "local" Smithy4s code-generation will automatically skip the generation of code that should not be generated again. +## Artifacts containing Smithy4s generated code : dependency tracking + +When packaging a project/module via SBT or Mill, Smithy4s adds a line to the Jar manifest of the project, informing downstream projects of library dependencies that may have been used during the code-generation of this project/module. + +This information is used automatically by downstream usage of Smithy4s, which automatically pulls additional jars that would be specified +in this bit of metadata. + +So, for instance, if you have + +```scala +lazy val upstream = (project in file("foo")) + .enablePlugins(Smithy4sCodegenPlugin) + .settings( + organization := "foobar", + version := "0.0.1", + libraryDependencies ++= Seq( + "software.amazon.smithy" % "smithy-aws-iam-traits" % "1.14.1" % Smithy4s + ) + ) +``` + +and publish this project to an artifact repository, the Jar manifest will contain the following line : + +``` +smithy4sDependencies: software.amazon.smithy:smithy-aws-iam-traits:1.14.1 +``` + +Using this artifact in a downstream project, for instance with : + +```scala +lazy val downstream = (project in file("foo")) + .enablePlugins(Smithy4sCodegenPlugin) + .settings( + libraryDependencies ++= Seq( + // compile/runtime dependency that contains Smithy4s-generated code but doesn't contain smithy files + "foobar" % "upstream" % "0.0.1" + ) + ) +``` + +will result in the `"software.amazon.smithy" % "smithy-aws-iam-traits" % "1.14.1"` dependency being automatically fetched +and used for the smithy-level classpath of the smithy files contained by `downstream`. This effectively means that smithy files +in `downstream` can use the Smithy shapes present in the `smithy-aws-iam-traits` artifact. + + + ### Manually skipping (or including) namespaces during code-generation. diff --git a/modules/mill-codegen-plugin/src/smithy4s/codegen/mill/Smithy4sModule.scala b/modules/mill-codegen-plugin/src/smithy4s/codegen/mill/Smithy4sModule.scala index f8f71e03b..220419255 100644 --- a/modules/mill-codegen-plugin/src/smithy4s/codegen/mill/Smithy4sModule.scala +++ b/modules/mill-codegen-plugin/src/smithy4s/codegen/mill/Smithy4sModule.scala @@ -54,7 +54,7 @@ trait Smithy4sModule extends ScalaModule { smithy4sDefaultIvyDeps() ++ smithy4sIvyDeps() } - def smithy4sLocalJars: T[List[PathRef]] = T { + def smithy4sInternalDependenciesAsJars: T[List[PathRef]] = T { T.traverse(moduleDeps)(_.jar) .map(_.toList.map(_.path).map(PathRef(_))) } @@ -102,7 +102,7 @@ trait Smithy4sModule extends ScalaModule { val resolvedDeps = smithy4sResolvedIvyDeps().iterator.map(_.path).toList - val localJars = smithy4sLocalJars().map(_.path) + val localJars = smithy4sAllDependenciesAsJars().map(_.path) val allLocalJars = localJars ++ resolvedDeps val args = CodegenArgs( diff --git a/modules/mill-codegen-plugin/test/src/smithy4s/codegen/mill/Smithy4sModuleSpec.scala b/modules/mill-codegen-plugin/test/src/smithy4s/codegen/mill/Smithy4sModuleSpec.scala index 96d4d66ff..6b25362e0 100644 --- a/modules/mill-codegen-plugin/test/src/smithy4s/codegen/mill/Smithy4sModuleSpec.scala +++ b/modules/mill-codegen-plugin/test/src/smithy4s/codegen/mill/Smithy4sModuleSpec.scala @@ -199,7 +199,7 @@ class Smithy4sModuleSpec extends munit.FunSuite { override def millSourcePath = resourcePath / "multi-module-no-compile" / "bar" - override def smithy4sLocalJars = List.empty[PathRef] + override def smithy4sInternalDependenciesAsJars = List.empty[PathRef] } val barEv = testKit.staticTestEvaluator(bar)(FullName("multi-module-bar")) diff --git a/project/build.properties b/project/build.properties index 6a9f03889..8b9a0b0ab 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.7.3 +sbt.version=1.8.0 From 644f3e492b95bd1359b5689418b5e68594862e91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20M=C3=A9lois?= Date: Wed, 16 Nov 2022 14:02:16 +0100 Subject: [PATCH 2/9] Fix mill module compilation --- .../src/smithy4s/codegen/mill/Smithy4sModule.scala | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/mill-codegen-plugin/src/smithy4s/codegen/mill/Smithy4sModule.scala b/modules/mill-codegen-plugin/src/smithy4s/codegen/mill/Smithy4sModule.scala index 220419255..3c30cfa8d 100644 --- a/modules/mill-codegen-plugin/src/smithy4s/codegen/mill/Smithy4sModule.scala +++ b/modules/mill-codegen-plugin/src/smithy4s/codegen/mill/Smithy4sModule.scala @@ -83,6 +83,10 @@ trait Smithy4sModule extends ScalaModule { resolveDeps(T.task { smithy4sTransitiveIvyDeps() })() } + def smithy4sAllDependenciesAsJars: T[Agg[PathRef]] = T { + smithy4sInternalDependenciesAsJars() ++ smithy4sResolvedIvyDeps() + } + def smithy4sCodegen: T[(PathRef, PathRef)] = T { val specFiles = smithy4sInputDirs().map(_.path).filter(os.exists(_)) @@ -100,10 +104,8 @@ trait Smithy4sModule extends ScalaModule { val skipSet = skipResources ++ skipOpenApi - val resolvedDeps = smithy4sResolvedIvyDeps().iterator.map(_.path).toList - - val localJars = smithy4sAllDependenciesAsJars().map(_.path) - val allLocalJars = localJars ++ resolvedDeps + val allLocalJars = + smithy4sAllDependenciesAsJars().map(_.path).iterator.to(List) val args = CodegenArgs( specs = specFiles.toList, From 9a1e766b4eca1c1d93bc2b49b83cb429e8bd3dc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20M=C3=A9lois?= Date: Thu, 17 Nov 2022 15:34:40 +0100 Subject: [PATCH 3/9] Add dependency-tracking logic to the Smithy4sModule (mill) --- .../multimodule-staged/build.sbt | 6 ++ .../codegen/Smithy4sCodegenPlugin.scala | 5 +- .../src/smithy4s/codegen/package.scala | 2 + .../codegen/mill/Smithy4sModule.scala | 45 ++++++++- .../multimodule-staged/bar/smithy/bar.smithy | 10 ++ .../bar/src/main/scala/Test.scala | 27 ++++++ .../multimodule-staged/baz/smithy/bar.smithy | 12 +++ .../baz/src/main/scala/Test.scala | 28 ++++++ .../multimodule-staged/foo/smithy/foo.smithy | 12 +++ .../codegen/mill/Smithy4sModuleSpec.scala | 91 ++++++++++++++++++- 10 files changed, 231 insertions(+), 7 deletions(-) create mode 100644 modules/mill-codegen-plugin/test/resources/multimodule-staged/bar/smithy/bar.smithy create mode 100644 modules/mill-codegen-plugin/test/resources/multimodule-staged/bar/src/main/scala/Test.scala create mode 100644 modules/mill-codegen-plugin/test/resources/multimodule-staged/baz/smithy/bar.smithy create mode 100644 modules/mill-codegen-plugin/test/resources/multimodule-staged/baz/src/main/scala/Test.scala create mode 100644 modules/mill-codegen-plugin/test/resources/multimodule-staged/foo/smithy/foo.smithy diff --git a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/build.sbt b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/build.sbt index d30fae71c..707eb5f39 100644 --- a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/build.sbt +++ b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/build.sbt @@ -15,6 +15,9 @@ lazy val foo = (project in file("foo")) lazy val bar = (project in file("bar")) .enablePlugins(Smithy4sCodegenPlugin) .settings( + // Bar refers to foo explicitly in its ivy deps, and upon publishing, + // this information is stored in the manifest of bar's jar, for downstream + // consumption libraryDependencies ++= Seq( "foobar" %% "foo" % version.value % Smithy4sCompile ) @@ -23,6 +26,9 @@ lazy val bar = (project in file("bar")) lazy val baz = (project in file("baz")) .enablePlugins(Smithy4sCodegenPlugin) .settings( + // baz depend on bar, and an assumption is made that baz may depend on the same smithy models + // that bar depended on for its own codegen. Therefore, these are retrieved from bar's manifest, + // resolved and added to the list of jars to seek smithy models from during code generation libraryDependencies ++= Seq( "foobar" %% "bar" % version.value ) diff --git a/modules/codegen-plugin/src/smithy4s/codegen/Smithy4sCodegenPlugin.scala b/modules/codegen-plugin/src/smithy4s/codegen/Smithy4sCodegenPlugin.scala index be0803c06..ae8243fb9 100644 --- a/modules/codegen-plugin/src/smithy4s/codegen/Smithy4sCodegenPlugin.scala +++ b/modules/codegen-plugin/src/smithy4s/codegen/Smithy4sCodegenPlugin.scala @@ -20,6 +20,7 @@ import sbt.Keys._ import java.util.jar.JarFile import sbt.util.CacheImplicits._ import sbt.{fileJsonFormatter => _, _} +import smithy4s.codegen.SMITHY4S_DEPENDENCIES import JsonConverters._ object Smithy4sCodegenPlugin extends AutoPlugin { @@ -125,8 +126,6 @@ object Smithy4sCodegenPlugin extends AutoPlugin { smithy4sVersion := BuildInfo.version ) - private val SMITHY4S_DEPENDENCIES = "smithy4sDependencies" - override def projectConfigurations: Seq[Configuration] = Seq(Smithy4s) // Use this with any configuration to enable the codegen in it. @@ -259,7 +258,7 @@ object Smithy4sCodegenPlugin extends AutoPlugin { private def extract(jarFile: java.io.File): Seq[ModuleID] = { val jar = new JarFile(jarFile) Option( - jar.getManifest().getMainAttributes().getValue("smithy4sDependencies") + jar.getManifest().getMainAttributes().getValue(SMITHY4S_DEPENDENCIES) ).toList.flatMap { listString => listString .split(",") diff --git a/modules/codegen/src/smithy4s/codegen/package.scala b/modules/codegen/src/smithy4s/codegen/package.scala index e0309792a..13d8a1020 100644 --- a/modules/codegen/src/smithy4s/codegen/package.scala +++ b/modules/codegen/src/smithy4s/codegen/package.scala @@ -37,6 +37,8 @@ import scala.jdk.CollectionConverters._ package object codegen { + val SMITHY4S_DEPENDENCIES = "smithy4sDependencies" + val uuidShapeId = ShapeId.from("alloy#UUID") private[codegen] type LinesWithValue = ToLinesWithValue[_] diff --git a/modules/mill-codegen-plugin/src/smithy4s/codegen/mill/Smithy4sModule.scala b/modules/mill-codegen-plugin/src/smithy4s/codegen/mill/Smithy4sModule.scala index 3c30cfa8d..5bd1bbd4c 100644 --- a/modules/mill-codegen-plugin/src/smithy4s/codegen/mill/Smithy4sModule.scala +++ b/modules/mill-codegen-plugin/src/smithy4s/codegen/mill/Smithy4sModule.scala @@ -23,6 +23,12 @@ import mill.define.Sources import mill.scalalib._ import smithy4s.codegen.{CodegenArgs, Codegen => Smithy4s, FileType} import smithy4s.codegen.BuildInfo +import smithy4s.codegen.SMITHY4S_DEPENDENCIES +import mill.modules.Jvm +import mill.scalalib.CrossVersion.Binary +import mill.scalalib.CrossVersion.Constant +import mill.scalalib.CrossVersion.Full +import java.util.jar.JarFile trait Smithy4sModule extends ScalaModule { @@ -54,6 +60,22 @@ trait Smithy4sModule extends ScalaModule { smithy4sDefaultIvyDeps() ++ smithy4sIvyDeps() } + override def manifest: T[Jvm.JarManifest] = T { + val m = super.manifest() + val deps = smithy4sIvyDeps().iterator.toList.flatMap { d => + val mod = d.dep.module + val org = mod.organization.value + val name = mod.name.value + val version = d.dep.version + d.cross match { + case Binary(_) => List(s"$org::$name:$version") + case Constant(_, _) => List(s"$org:$name:$version") + case Full(_) => Nil + } + } + m.add(SMITHY4S_DEPENDENCIES -> deps.mkString(",")) + } + def smithy4sInternalDependenciesAsJars: T[List[PathRef]] = T { T.traverse(moduleDeps)(_.jar) .map(_.toList.map(_.path).map(PathRef(_))) @@ -79,12 +101,29 @@ trait Smithy4sModule extends ScalaModule { .flatten } - def smithy4sResolvedIvyDeps: T[Agg[PathRef]] = T { - resolveDeps(T.task { smithy4sTransitiveIvyDeps() })() + def smithy4sResolvedIvyDeps: T[Agg[PathRef]] = + resolveDeps(smithy4sTransitiveIvyDeps) + + def smithy4sExternalCodegenIvyDeps: T[Agg[Dep]] = T { + resolveDeps(transitiveIvyDeps)().flatMap { pathRef => + val jarFile = new JarFile(pathRef.path.toIO) + val deps = Option( + jarFile + .getManifest() + .getMainAttributes() + .getValue(SMITHY4S_DEPENDENCIES) + ).toList.flatMap { listString => + listString.split(",").toList.map(dep => ivy"$dep") + } + Agg.from(deps) + } } + def smithy4sResolvedExternalCodegenIvyDeps: T[Agg[PathRef]] = + resolveDeps(smithy4sExternalCodegenIvyDeps) + def smithy4sAllDependenciesAsJars: T[Agg[PathRef]] = T { - smithy4sInternalDependenciesAsJars() ++ smithy4sResolvedIvyDeps() + smithy4sInternalDependenciesAsJars() ++ smithy4sResolvedIvyDeps() ++ smithy4sResolvedExternalCodegenIvyDeps() } def smithy4sCodegen: T[(PathRef, PathRef)] = T { diff --git a/modules/mill-codegen-plugin/test/resources/multimodule-staged/bar/smithy/bar.smithy b/modules/mill-codegen-plugin/test/resources/multimodule-staged/bar/smithy/bar.smithy new file mode 100644 index 000000000..19b8d4847 --- /dev/null +++ b/modules/mill-codegen-plugin/test/resources/multimodule-staged/bar/smithy/bar.smithy @@ -0,0 +1,10 @@ +$version: "2.0" + +namespace bar + +use foo#Foo + +// Checking that Foo can be found by virtue of the bar project depending on the foo project +structure Bar { + foo: Foo +} diff --git a/modules/mill-codegen-plugin/test/resources/multimodule-staged/bar/src/main/scala/Test.scala b/modules/mill-codegen-plugin/test/resources/multimodule-staged/bar/src/main/scala/Test.scala new file mode 100644 index 000000000..c61d7291c --- /dev/null +++ b/modules/mill-codegen-plugin/test/resources/multimodule-staged/bar/src/main/scala/Test.scala @@ -0,0 +1,27 @@ +/* + * Copyright 2021-2022 Disney Streaming + * + * Licensed under the Tomorrow Open Source Technology License, Version 1.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://disneystreaming.github.io/TOST-1.0.txt + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package bar + +import foo._ + +object BarTest { + + def main(args: Array[String]): Unit = { + println(Bar(Some(Foo(Some(1))))) + } + +} diff --git a/modules/mill-codegen-plugin/test/resources/multimodule-staged/baz/smithy/bar.smithy b/modules/mill-codegen-plugin/test/resources/multimodule-staged/baz/smithy/bar.smithy new file mode 100644 index 000000000..f447cb167 --- /dev/null +++ b/modules/mill-codegen-plugin/test/resources/multimodule-staged/baz/smithy/bar.smithy @@ -0,0 +1,12 @@ +$version: "2.0" + +namespace baz + +use foo#Foo + +// Checking that Foo can be found by virtue of the upstream `bar` project +// defined as a compile-scope library dependency was published with an indication +// in the manifest that it used the `foo` project for code generation. +structure Baz { + foo: Foo +} diff --git a/modules/mill-codegen-plugin/test/resources/multimodule-staged/baz/src/main/scala/Test.scala b/modules/mill-codegen-plugin/test/resources/multimodule-staged/baz/src/main/scala/Test.scala new file mode 100644 index 000000000..1ee5acbc7 --- /dev/null +++ b/modules/mill-codegen-plugin/test/resources/multimodule-staged/baz/src/main/scala/Test.scala @@ -0,0 +1,28 @@ +/* + * Copyright 2021-2022 Disney Streaming + * + * Licensed under the Tomorrow Open Source Technology License, Version 1.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://disneystreaming.github.io/TOST-1.0.txt + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package baz + +import foo._ +import bar._ + +object BarTest { + + def main(args: Array[String]): Unit = { + println(Baz(Some(Foo(Some(1))))) + } + +} diff --git a/modules/mill-codegen-plugin/test/resources/multimodule-staged/foo/smithy/foo.smithy b/modules/mill-codegen-plugin/test/resources/multimodule-staged/foo/smithy/foo.smithy new file mode 100644 index 000000000..28ed79088 --- /dev/null +++ b/modules/mill-codegen-plugin/test/resources/multimodule-staged/foo/smithy/foo.smithy @@ -0,0 +1,12 @@ +$version: "2.0" + +namespace foo + +use alloy#uuidFormat + +structure Foo { + a: Integer +} + +@uuidFormat +string MyUUID diff --git a/modules/mill-codegen-plugin/test/src/smithy4s/codegen/mill/Smithy4sModuleSpec.scala b/modules/mill-codegen-plugin/test/src/smithy4s/codegen/mill/Smithy4sModuleSpec.scala index 6b25362e0..0d3d33da0 100644 --- a/modules/mill-codegen-plugin/test/src/smithy4s/codegen/mill/Smithy4sModuleSpec.scala +++ b/modules/mill-codegen-plugin/test/src/smithy4s/codegen/mill/Smithy4sModuleSpec.scala @@ -22,6 +22,8 @@ import mill._ import munit.Location import sourcecode.FullName import java.nio.file.Paths +import mill.scalalib.publish.PomSettings +import mill.scalalib.publish.VersionControl class Smithy4sModuleSpec extends munit.FunSuite { private val resourcePath = @@ -207,6 +209,93 @@ class Smithy4sModuleSpec extends munit.FunSuite { taskWorks(bar.smithy4sCodegen, barEv) } + test("multi-module staged codegen works".only) { + + trait Base + extends testKit.BaseModule + with SbtModule + with Smithy4sModule + with PublishModule { + override def scalaVersion = "2.13.10" + def pomSettings: T[PomSettings] = PomSettings( + "foo", + "foobar", + "http://foobar", + Seq.empty, + VersionControl(), + Seq.empty + ) + def publishVersion: T[String] = "0.0.1-SNAPSHOT" + + } + + object foo extends Base { + override def scalaVersion = "2.13.10" + override def ivyDeps = Agg(coreDep) + override def millSourcePath = resourcePath / "multimodule-staged" / "foo" + } + + object bar extends Base { + override def scalaVersion = "2.13.10" + // Bar refers to foo explicitly in its ivy deps, and upon publishing, + // this information is stored in the manifest of bar's jar, for downstream + // consumption + override def smithy4sIvyDeps = + Agg(ivy"${pomSettings().organization}::foo:${publishVersion()}") + override def ivyDeps = T(smithy4sIvyDeps()) + override def millSourcePath = resourcePath / "multimodule-staged" / "bar" + } + + object baz extends Base { + override def scalaVersion = "2.13.10" + // baz depend on bar, and an assumption is made that baz may depend on the same smithy models + // that bar depended on for its own codegen. Therefore, these are retrieved from bar's manifest, + // resolved and added to the list of jars to seek smithy models from during code generation + override def ivyDeps = + Agg(ivy"${pomSettings().organization}::bar:${publishVersion()}") + override def millSourcePath = resourcePath / "multimodule-staged" / "baz" + } + + val fooEv = + testKit.staticTestEvaluator(foo)(FullName("multi-module-staged-foo")) + val barEv = + testKit.staticTestEvaluator(bar)(FullName("multi-module-staged-bar")) + val bazEv = + testKit.staticTestEvaluator(baz)(FullName("multi-module-staged-bar")) + + taskWorks(foo.publishLocal(), fooEv) + taskWorks(bar.compile, barEv) + + checkFileExist( + barEv.outPath / "smithy4sOutputDir.dest" / "scala" / "bar" / "Bar.scala", + shouldExist = true + ) + checkFileExist( + barEv.outPath / "smithy4sOutputDir.dest" / "scala" / "foo" / "Foo.scala", + shouldExist = false + ) + + taskWorks(bar.publishLocal(), barEv) + taskWorks(baz.compile, bazEv) + + checkFileExist( + bazEv.outPath / "smithy4sOutputDir.dest" / "scala" / "baz" / "Baz.scala", + shouldExist = true + ) + checkFileExist( + bazEv.outPath / "smithy4sOutputDir.dest" / "scala" / "foo" / "Baz.scala", + shouldExist = false + ) + checkFileExist( + bazEv.outPath / "smithy4sOutputDir.dest" / "scala" / "bar" / "Bar.scala", + shouldExist = false + ) + + taskWorks(bar.run(), barEv) + taskWorks(baz.run(), bazEv) + + } + private def compileWorks( sm: ScalaModule, testEvaluator: testKit.TestEvaluator @@ -214,7 +303,7 @@ class Smithy4sModuleSpec extends munit.FunSuite { taskWorks(sm.compile, testEvaluator) private def taskWorks[A]( - task: T[A], + task: mill.define.Task[A], testEvaluator: testKit.TestEvaluator )(implicit loc: Location) = { val result = testEvaluator(task).map(_._1) From 68437eced9ae22ba8b998cf8e8c703754cc67cc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20M=C3=A9lois?= Date: Mon, 21 Nov 2022 18:42:48 +0100 Subject: [PATCH 4/9] Dogfooding : set-up our own libraries with manifest metadata The same smithy4sDependencies metadata is now being populated for the artifacts published in the smithy4s repository --- build.sbt | 158 +----------------- .../codegen/Smithy4sCodegenPlugin.scala | 1 - project/Dependencies.scala | 128 ++++++++++++++ project/Smithy4sBuildPlugin.scala | 53 +++++- 4 files changed, 189 insertions(+), 151 deletions(-) create mode 100644 project/Dependencies.scala diff --git a/build.sbt b/build.sbt index ca93bd723..d1754da46 100644 --- a/build.sbt +++ b/build.sbt @@ -111,7 +111,7 @@ lazy val docs = Dependencies.Http4s.emberServer.value, Dependencies.Decline.effect.value ), - Compile / genSmithyDependencies ++= Seq(Dependencies.Smithy.testTraits), + Compile / smithy4sDependencies ++= Seq(Dependencies.Smithy.testTraits), Compile / sourceGenerators := Seq(genSmithyScala(Compile).taskValue), Compile / smithySpecs := Seq( (Compile / sourceDirectory).value / "smithy", @@ -160,7 +160,7 @@ lazy val core = projectMatrix "smithy.waiters", "alloy" ), - genSmithyDependencies ++= Seq( + smithy4sDependencies ++= Seq( Dependencies.Smithy.waiters ), Compile / sourceGenerators := Seq(genSmithyScala(Compile).taskValue), @@ -261,7 +261,7 @@ lazy val `aws-kernel` = projectMatrix Dependencies.Weaver.cats.value % Test, Dependencies.Weaver.scalacheck.value % Test ), - genSmithyDependencies ++= Seq(Dependencies.Smithy.awsTraits), + smithy4sDependencies ++= Seq(Dependencies.Smithy.awsTraits), Compile / allowedNamespaces := Seq( "aws.api", "aws.auth", @@ -322,7 +322,7 @@ lazy val `aws-http4s` = projectMatrix Dependencies.Http4s.emberClient.value % Test ) }, - Test / genSmithyDependencies ++= Seq( + Test / smithy4sDependencies ++= Seq( Dependencies.Smithy.waiters, Dependencies.Smithy.awsTraits ), @@ -704,7 +704,7 @@ lazy val complianceTests = projectMatrix .settings( name := "compliance-tests", Compile / allowedNamespaces := Seq("smithy.test", "smithy4s.example.test"), - Compile / genSmithyDependencies ++= Seq(Dependencies.Smithy.testTraits), + Compile / smithy4sDependencies ++= Seq(Dependencies.Smithy.testTraits), Compile / sourceGenerators := Seq(genSmithyScala(Compile).taskValue), isCE3 := virtualAxes.value.contains(CatsEffect3Axis), libraryDependencies ++= { @@ -814,146 +814,6 @@ lazy val benchmark = projectMatrix .jvmPlatform(List(Scala213), jvmDimSettings) .settings(Smithy4sBuildPlugin.doNotPublishArtifact) -val isCE3 = settingKey[Boolean]("Is the current build using CE3?") - -lazy val Dependencies = new { - - val collectionsCompat = - Def.setting( - "org.scala-lang.modules" %%% "scala-collection-compat" % "2.8.1" - ) - - val Jsoniter = - Def.setting( - "com.github.plokhotnyuk.jsoniter-scala" %%% "jsoniter-scala-core" % "2.17.9" - ) - - val Smithy = new { - val org = "software.amazon.smithy" - val version = "1.26.0" - val model = org % "smithy-model" % version - val testTraits = org % "smithy-protocol-test-traits" % version - val build = org % "smithy-build" % version - val awsTraits = org % "smithy-aws-traits" % version - val waiters = org % "smithy-waiters" % version - } - - val Alloy = new { - val org = "com.disneystreaming.alloy" - val version = "0.1.0" - val core = org % "alloy-core" % version - val openapi = org %% "alloy-openapi" % version - } - - val Cats = new { - val core: Def.Initialize[ModuleID] = - Def.setting("org.typelevel" %%% "cats-core" % "2.8.0") - } - - object Decline { - val declineVersion = "2.3.1" - - val core = Def.setting("com.monovore" %%% "decline" % declineVersion) - val effect = - Def.setting("com.monovore" %%% "decline-effect" % declineVersion) - } - object Fs2 { - val core: Def.Initialize[ModuleID] = - Def.setting("co.fs2" %%% "fs2-core" % "3.3.0") - } - - object Mill { - val millVersion = "0.10.7" - - val scalalib = "com.lihaoyi" %% "mill-scalalib" % millVersion - val main = "com.lihaoyi" %% "mill-main" % millVersion - val mainApi = "com.lihaoyi" %% "mill-main-api" % millVersion - val mainTestkit = "com.lihaoyi" %% "mill-main-testkit" % millVersion % Test - } - - val Circe = new { - val generic: Def.Initialize[ModuleID] = - Def.setting("io.circe" %%% "circe-generic" % "0.14.3") - } - - /* - * we override the version to use the fix included in - * https://github.com/typelevel/cats-effect/pull/2945 - * it allows us to use UUIDGen instead of calling - * UUID.randomUUID manually - * - * we also provide a 2.12 shim under: - * modules/tests/src-ce2/UUIDGen.scala - */ - val CatsEffect3: Def.Initialize[ModuleID] = - Def.setting("org.typelevel" %%% "cats-effect" % "3.3.14") - - object Http4s { - val http4sVersion = Def.setting(if (isCE3.value) "0.23.16" else "0.22.14") - - val emberServer: Def.Initialize[ModuleID] = - Def.setting("org.http4s" %%% "http4s-ember-server" % http4sVersion.value) - val emberClient: Def.Initialize[ModuleID] = - Def.setting("org.http4s" %%% "http4s-ember-client" % http4sVersion.value) - val circe: Def.Initialize[ModuleID] = - Def.setting("org.http4s" %%% "http4s-circe" % http4sVersion.value) - val core: Def.Initialize[ModuleID] = - Def.setting("org.http4s" %%% "http4s-core" % http4sVersion.value) - val dsl: Def.Initialize[ModuleID] = - Def.setting("org.http4s" %%% "http4s-dsl" % http4sVersion.value) - val client: Def.Initialize[ModuleID] = - Def.setting("org.http4s" %%% "http4s-client" % http4sVersion.value) - } - - object Weaver { - - val weaverVersion = Def.setting(if (isCE3.value) "0.8.0" else "0.6.15") - - val cats: Def.Initialize[ModuleID] = - Def.setting("com.disneystreaming" %%% "weaver-cats" % weaverVersion.value) - - val scalacheck: Def.Initialize[ModuleID] = - Def.setting( - "com.disneystreaming" %%% "weaver-scalacheck" % weaverVersion.value - ) - } - - class MunitCross(munitVersion: String) { - val core: Def.Initialize[ModuleID] = - Def.setting("org.scalameta" %%% "munit" % munitVersion) - val scalacheck: Def.Initialize[ModuleID] = - Def.setting("org.scalameta" %%% "munit-scalacheck" % munitVersion) - } - object Munit extends MunitCross("0.7.29") - object MunitMilestone extends MunitCross("1.0.0-M6") - - val Scalacheck = new { - val version = "1.16.0" - val scalacheck = - Def.setting("org.scalacheck" %%% "scalacheck" % version) - } - - object Webjars { - val swaggerUi: ModuleID = "org.webjars.npm" % "swagger-ui-dist" % "4.15.2" - - val webjarsLocator: ModuleID = "org.webjars" % "webjars-locator" % "0.42" - } - -} - -lazy val smithySpecs = SettingKey[Seq[File]]("smithySpecs") -lazy val genSmithyOutput = SettingKey[File]("genSmithyOutput") -lazy val genSmithyResourcesOutput = SettingKey[File]("genSmithyResourcesOutput") -lazy val allowedNamespaces = SettingKey[Seq[String]]("allowedNamespaces") -lazy val genSmithyDependencies = - SettingKey[Seq[ModuleID]]("genSmithyDependencies") - -(ThisBuild / genSmithyDependencies) := Seq(Dependencies.Alloy.core) - -lazy val smithy4sSkip = SettingKey[Seq[String]]("smithy4sSkip") - -(ThisBuild / smithySpecs) := Seq.empty - def genSmithyScala(config: Configuration) = genSmithyImpl(config).map(_._1) def genSmithyResources(config: Configuration) = genSmithyImpl(config).map(_._2) @@ -972,8 +832,8 @@ def genSmithyImpl(config: Configuration) = Def.task { .getAbsolutePath() val allowedNS = (config / allowedNamespaces).?.value.filterNot(_.isEmpty) val skip = (config / smithy4sSkip).?.value.getOrElse(Seq.empty) - val smithy4sDependencies = - (config / genSmithyDependencies).?.value.getOrElse(Seq.empty).map { + val smithy4sDeps = + (config / smithy4sDependencies).?.value.getOrElse(Seq.empty).map { moduleId => s"${moduleId.organization}:${moduleId.name}:${moduleId.revision}" } @@ -1052,8 +912,8 @@ def genSmithyImpl(config: Configuration) = Def.task { else Nil val skipOpt = skip.flatMap(s => List("--skip", s)) val dependenciesOpt = - if (smithy4sDependencies.nonEmpty) - List("--dependencies", smithy4sDependencies.mkString(",")) + if (smithy4sDeps.nonEmpty) + List("--dependencies", smithy4sDeps.mkString(",")) else Nil val args = outputOpt ++ resourceOutputOpt ++ diff --git a/modules/codegen-plugin/src/smithy4s/codegen/Smithy4sCodegenPlugin.scala b/modules/codegen-plugin/src/smithy4s/codegen/Smithy4sCodegenPlugin.scala index ae8243fb9..4c73d6752 100644 --- a/modules/codegen-plugin/src/smithy4s/codegen/Smithy4sCodegenPlugin.scala +++ b/modules/codegen-plugin/src/smithy4s/codegen/Smithy4sCodegenPlugin.scala @@ -20,7 +20,6 @@ import sbt.Keys._ import java.util.jar.JarFile import sbt.util.CacheImplicits._ import sbt.{fileJsonFormatter => _, _} -import smithy4s.codegen.SMITHY4S_DEPENDENCIES import JsonConverters._ object Smithy4sCodegenPlugin extends AutoPlugin { diff --git a/project/Dependencies.scala b/project/Dependencies.scala new file mode 100644 index 000000000..fe7fd6522 --- /dev/null +++ b/project/Dependencies.scala @@ -0,0 +1,128 @@ +import sbt._ +import org.portablescala.sbtplatformdeps.PlatformDepsPlugin.autoImport._ +import Smithy4sBuildPlugin.autoImport.isCE3 + +object Dependencies { + + val collectionsCompat = + Def.setting( + "org.scala-lang.modules" %%% "scala-collection-compat" % "2.8.1" + ) + + val Jsoniter = + Def.setting( + "com.github.plokhotnyuk.jsoniter-scala" %%% "jsoniter-scala-core" % "2.17.9" + ) + + val Smithy = new { + val org = "software.amazon.smithy" + val version = "1.26.0" + val model = org % "smithy-model" % version + val testTraits = org % "smithy-protocol-test-traits" % version + val build = org % "smithy-build" % version + val awsTraits = org % "smithy-aws-traits" % version + val waiters = org % "smithy-waiters" % version + } + + val Alloy = new { + val org = "com.disneystreaming.alloy" + val version = "0.1.0" + val core = org % "alloy-core" % version + val openapi = org %% "alloy-openapi" % version + } + + val Cats = new { + val core: Def.Initialize[ModuleID] = + Def.setting("org.typelevel" %%% "cats-core" % "2.8.0") + } + + object Decline { + val declineVersion = "2.3.1" + + val core = Def.setting("com.monovore" %%% "decline" % declineVersion) + val effect = + Def.setting("com.monovore" %%% "decline-effect" % declineVersion) + } + object Fs2 { + val core: Def.Initialize[ModuleID] = + Def.setting("co.fs2" %%% "fs2-core" % "3.3.0") + } + + object Mill { + val millVersion = "0.10.7" + + val scalalib = "com.lihaoyi" %% "mill-scalalib" % millVersion + val main = "com.lihaoyi" %% "mill-main" % millVersion + val mainApi = "com.lihaoyi" %% "mill-main-api" % millVersion + val mainTestkit = "com.lihaoyi" %% "mill-main-testkit" % millVersion % Test + } + + val Circe = new { + val generic: Def.Initialize[ModuleID] = + Def.setting("io.circe" %%% "circe-generic" % "0.14.3") + } + + /* + * we override the version to use the fix included in + * https://github.com/typelevel/cats-effect/pull/2945 + * it allows us to use UUIDGen instead of calling + * UUID.randomUUID manually + * + * we also provide a 2.12 shim under: + * modules/tests/src-ce2/UUIDGen.scala + */ + val CatsEffect3: Def.Initialize[ModuleID] = + Def.setting("org.typelevel" %%% "cats-effect" % "3.3.14") + + object Http4s { + val http4sVersion = Def.setting(if (isCE3.value) "0.23.16" else "0.22.14") + + val emberServer: Def.Initialize[ModuleID] = + Def.setting("org.http4s" %%% "http4s-ember-server" % http4sVersion.value) + val emberClient: Def.Initialize[ModuleID] = + Def.setting("org.http4s" %%% "http4s-ember-client" % http4sVersion.value) + val circe: Def.Initialize[ModuleID] = + Def.setting("org.http4s" %%% "http4s-circe" % http4sVersion.value) + val core: Def.Initialize[ModuleID] = + Def.setting("org.http4s" %%% "http4s-core" % http4sVersion.value) + val dsl: Def.Initialize[ModuleID] = + Def.setting("org.http4s" %%% "http4s-dsl" % http4sVersion.value) + val client: Def.Initialize[ModuleID] = + Def.setting("org.http4s" %%% "http4s-client" % http4sVersion.value) + } + + object Weaver { + + val weaverVersion = Def.setting(if (isCE3.value) "0.8.0" else "0.6.15") + + val cats: Def.Initialize[ModuleID] = + Def.setting("com.disneystreaming" %%% "weaver-cats" % weaverVersion.value) + + val scalacheck: Def.Initialize[ModuleID] = + Def.setting( + "com.disneystreaming" %%% "weaver-scalacheck" % weaverVersion.value + ) + } + + class MunitCross(munitVersion: String) { + val core: Def.Initialize[ModuleID] = + Def.setting("org.scalameta" %%% "munit" % munitVersion) + val scalacheck: Def.Initialize[ModuleID] = + Def.setting("org.scalameta" %%% "munit-scalacheck" % munitVersion) + } + object Munit extends MunitCross("0.7.29") + object MunitMilestone extends MunitCross("1.0.0-M6") + + val Scalacheck = new { + val version = "1.16.0" + val scalacheck = + Def.setting("org.scalacheck" %%% "scalacheck" % version) + } + + object Webjars { + val swaggerUi: ModuleID = "org.webjars.npm" % "swagger-ui-dist" % "4.15.2" + + val webjarsLocator: ModuleID = "org.webjars" % "webjars-locator" % "0.42" + } + +} diff --git a/project/Smithy4sBuildPlugin.scala b/project/Smithy4sBuildPlugin.scala index ac408789e..5e338bdeb 100644 --- a/project/Smithy4sBuildPlugin.scala +++ b/project/Smithy4sBuildPlugin.scala @@ -32,6 +32,19 @@ object Smithy4sBuildPlugin extends AutoPlugin { val Scala213 = "2.13.10" val Scala3 = "3.2.1" + object autoImport { + // format: off + val smithySpecs = SettingKey[Seq[File]]("smithySpecs") + val genSmithyOutput = SettingKey[File]("genSmithyOutput") + val genSmithyResourcesOutput = SettingKey[File]("genSmithyResourcesOutput") + val allowedNamespaces = SettingKey[Seq[String]]("allowedNamespaces") + val smithy4sDependencies = SettingKey[Seq[ModuleID]]("smithy4sDependencies") + val smithy4sSkip = SettingKey[Seq[String]]("smithy4sSkip") + val isCE3 = settingKey[Boolean]("Is the current build using CE3?") + // format: on + } + import autoImport._ + implicit class ProjectMatrixOps(val pm: ProjectMatrix) extends AnyVal { def http4sJvmPlatform( scalaVersions: Seq[String], @@ -76,6 +89,11 @@ object Smithy4sBuildPlugin extends AutoPlugin { override def requires = plugins.JvmPlugin override def trigger = allRequirements + override def buildSettings: Seq[Setting[_]] = Seq( + smithySpecs := Seq.empty, + smithy4sDependencies := Seq(Dependencies.Alloy.core) + ) + override val globalSettings = Seq( excludeLintKeys ++= Set( logManager, @@ -107,7 +125,25 @@ object Smithy4sBuildPlugin extends AutoPlugin { semanticdbVersion := scalafixSemanticdb.revision, testFrameworks += new TestFramework("weaver.framework.CatsEffect"), Test / fork := virtualAxes.?.value.forall(_.contains(VirtualAxis.jvm)), - Test / javaOptions += s"-Duser.dir=${sys.props("user.dir")}" + Test / javaOptions += s"-Duser.dir=${sys.props("user.dir")}", + Compile / packageBin / packageOptions += { + // This piece of logic aims at tracking the dependencies that Smithy4s used to generate + // code at build time, in the manifest of the jar. This helps automatically pulling + // the corresponding jars and prevents the users from having to search + import java.util.jar.Manifest + val manifest = new Manifest + val scalaBin = scalaBinaryVersion.?.value + val maybeDeps = smithy4sDependencies.?.value.map { + _.flatMap(moduleIdEncode(_, scalaBin)) + .mkString(",") + } + maybeDeps.foreach { deps => + manifest + .getMainAttributes() + .put(new java.util.jar.Attributes.Name("smithy4sDependencies"), deps) + } + Package.JarManifest(manifest) + } ) ++ publishSettings ++ loggingSettings ++ compilerPlugins ++ headerSettings lazy val compilerPlugins = Seq( @@ -526,4 +562,19 @@ object Smithy4sBuildPlugin extends AutoPlugin { case _ => sys.error("Unsupported mill platform.") } + private def moduleIdEncode( + moduleId: ModuleID, + scalaBinaryVersion: Option[String] + ): List[String] = { + (moduleId.crossVersion, scalaBinaryVersion) match { + case (Disabled, _) => + List(s"${moduleId.organization}:${moduleId.name}:${moduleId.revision}") + case (_: Binary, Some(sbv)) => + List( + s"${moduleId.organization}:${moduleId.name}_${sbv}:${moduleId.revision}" + ) + case (_, _) => Nil + } + } + } From ac1a41638b14fb4669fd71c80ee089708608476d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20M=C3=A9lois?= Date: Mon, 21 Nov 2022 18:48:03 +0100 Subject: [PATCH 5/9] Update documentation ro remove un-necessary dependency declarations --- .../src/sbt-test/codegen-plugin/multimodule-aws/build.sbt | 1 - modules/docs/markdown/03-protocols/03-aws/01-aws.md | 3 --- modules/docs/markdown/03-protocols/05-compliance-tests.md | 2 -- 3 files changed, 6 deletions(-) diff --git a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-aws/build.sbt b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-aws/build.sbt index 90a9d0e14..f5bc7434e 100644 --- a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-aws/build.sbt +++ b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-aws/build.sbt @@ -6,7 +6,6 @@ lazy val foo = (project in file("foo")) .enablePlugins(Smithy4sCodegenPlugin) .settings( libraryDependencies ++= Seq( - "software.amazon.smithy" % "smithy-aws-traits" % smithyVersion % Smithy4s, "com.disneystreaming.smithy4s" %% "smithy4s-aws-kernel" % smithy4sVersion.value ) ) diff --git a/modules/docs/markdown/03-protocols/03-aws/01-aws.md b/modules/docs/markdown/03-protocols/03-aws/01-aws.md index 7fda14d5a..3de837501 100644 --- a/modules/docs/markdown/03-protocols/03-aws/01-aws.md +++ b/modules/docs/markdown/03-protocols/03-aws/01-aws.md @@ -18,9 +18,6 @@ In `build.sbt` import smithy4s.codegen.BuildInfo._ libraryDependencies ++= Seq( - // contains traits used by specs of AWS services" - "software.amazon.smithy" % "smithy-aws-traits" % smithyVersion % Smithy4s, - "software.amazon.smithy" % "smithy-waiters" % smithyVersion % Smithy4s, // version sourced from the plugin "com.disneystreaming.smithy4s" %% "smithy4s-aws-http4s" % smithy4sVersion.value ) diff --git a/modules/docs/markdown/03-protocols/05-compliance-tests.md b/modules/docs/markdown/03-protocols/05-compliance-tests.md index 8516f78b5..76fc58a79 100644 --- a/modules/docs/markdown/03-protocols/05-compliance-tests.md +++ b/modules/docs/markdown/03-protocols/05-compliance-tests.md @@ -14,8 +14,6 @@ Smithy4s publishes a module that you can use to write compliance test if you're import smithy4s.codegen.BuildInfo._ libraryDependencies ++= Seq( - // Needed to access the smithy.test traits in your smithy models - "software.amazon.smithy" % "smithy-protocol-test-traits" % smithyVersion "com.disneystreaming.smithy4s" %% "smithy4s-compliance-tests" % smithy4sVersion.value ) ``` From 5f72f0da9d815537f6b33de35f28d37c17cedf75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Koz=C5=82owski?= Date: Tue, 22 Nov 2022 00:31:52 +0100 Subject: [PATCH 6/9] Remove .only --- .../test/src/smithy4s/codegen/mill/Smithy4sModuleSpec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/mill-codegen-plugin/test/src/smithy4s/codegen/mill/Smithy4sModuleSpec.scala b/modules/mill-codegen-plugin/test/src/smithy4s/codegen/mill/Smithy4sModuleSpec.scala index 0d3d33da0..83d20c8d0 100644 --- a/modules/mill-codegen-plugin/test/src/smithy4s/codegen/mill/Smithy4sModuleSpec.scala +++ b/modules/mill-codegen-plugin/test/src/smithy4s/codegen/mill/Smithy4sModuleSpec.scala @@ -209,7 +209,7 @@ class Smithy4sModuleSpec extends munit.FunSuite { taskWorks(bar.smithy4sCodegen, barEv) } - test("multi-module staged codegen works".only) { + test("multi-module staged codegen works") { trait Base extends testKit.BaseModule From 2b8e665b045b79244fb5ea6483b211d5657deedd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20M=C3=A9lois?= Date: Tue, 22 Nov 2022 09:57:57 +0100 Subject: [PATCH 7/9] Fix mill module --- .../codegen/mill/Smithy4sModule.scala | 4 +++- .../codegen/mill/Smithy4sModuleSpec.scala | 19 +++++++++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/modules/mill-codegen-plugin/src/smithy4s/codegen/mill/Smithy4sModule.scala b/modules/mill-codegen-plugin/src/smithy4s/codegen/mill/Smithy4sModule.scala index 5bd1bbd4c..b7035210e 100644 --- a/modules/mill-codegen-plugin/src/smithy4s/codegen/mill/Smithy4sModule.scala +++ b/modules/mill-codegen-plugin/src/smithy4s/codegen/mill/Smithy4sModule.scala @@ -73,7 +73,9 @@ trait Smithy4sModule extends ScalaModule { case Full(_) => Nil } } - m.add(SMITHY4S_DEPENDENCIES -> deps.mkString(",")) + if (deps.nonEmpty) { + m.add(SMITHY4S_DEPENDENCIES -> deps.mkString(",")) + } else m } def smithy4sInternalDependenciesAsJars: T[List[PathRef]] = T { diff --git a/modules/mill-codegen-plugin/test/src/smithy4s/codegen/mill/Smithy4sModuleSpec.scala b/modules/mill-codegen-plugin/test/src/smithy4s/codegen/mill/Smithy4sModuleSpec.scala index 83d20c8d0..552db89c8 100644 --- a/modules/mill-codegen-plugin/test/src/smithy4s/codegen/mill/Smithy4sModuleSpec.scala +++ b/modules/mill-codegen-plugin/test/src/smithy4s/codegen/mill/Smithy4sModuleSpec.scala @@ -24,6 +24,9 @@ import sourcecode.FullName import java.nio.file.Paths import mill.scalalib.publish.PomSettings import mill.scalalib.publish.VersionControl +import coursier.Repository +import coursier.ivy.IvyRepository +import mill.define.Task class Smithy4sModuleSpec extends munit.FunSuite { private val resourcePath = @@ -211,12 +214,21 @@ class Smithy4sModuleSpec extends munit.FunSuite { test("multi-module staged codegen works") { + val localIvyRepo = os.temp.dir() / ".ivy2" / "local" + trait Base extends testKit.BaseModule with SbtModule with Smithy4sModule with PublishModule { override def scalaVersion = "2.13.10" + override def repositoriesTask: Task[Seq[Repository]] = T.task { + val ivy2Local = IvyRepository.fromPattern( + (localIvyRepo.toNIO.toUri.toString + "/") +: coursier.ivy.Pattern.default, + dropInfoAttributes = true + ) + super.repositoriesTask() ++ Seq(ivy2Local) + } def pomSettings: T[PomSettings] = PomSettings( "foo", "foobar", @@ -230,12 +242,14 @@ class Smithy4sModuleSpec extends munit.FunSuite { } object foo extends Base { + override def artifactName: T[String] = "foo" override def scalaVersion = "2.13.10" override def ivyDeps = Agg(coreDep) override def millSourcePath = resourcePath / "multimodule-staged" / "foo" } object bar extends Base { + override def artifactName: T[String] = "bar" override def scalaVersion = "2.13.10" // Bar refers to foo explicitly in its ivy deps, and upon publishing, // this information is stored in the manifest of bar's jar, for downstream @@ -247,6 +261,7 @@ class Smithy4sModuleSpec extends munit.FunSuite { } object baz extends Base { + override def artifactName: T[String] = "baz" override def scalaVersion = "2.13.10" // baz depend on bar, and an assumption is made that baz may depend on the same smithy models // that bar depended on for its own codegen. Therefore, these are retrieved from bar's manifest, @@ -263,7 +278,7 @@ class Smithy4sModuleSpec extends munit.FunSuite { val bazEv = testKit.staticTestEvaluator(baz)(FullName("multi-module-staged-bar")) - taskWorks(foo.publishLocal(), fooEv) + taskWorks(foo.publishLocal(localIvyRepo.toString()), fooEv) taskWorks(bar.compile, barEv) checkFileExist( @@ -275,7 +290,7 @@ class Smithy4sModuleSpec extends munit.FunSuite { shouldExist = false ) - taskWorks(bar.publishLocal(), barEv) + taskWorks(bar.publishLocal(localIvyRepo.toString()), barEv) taskWorks(baz.compile, bazEv) checkFileExist( From a9e06125e541ff430a9ec984a92ba4751832fe68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20M=C3=A9lois?= Date: Tue, 22 Nov 2022 19:13:26 +0100 Subject: [PATCH 8/9] Change the build plugins so that libraryDependencies are taken into account for codegen Now, by default, any library declared as a "normal" dependency, by means of `ivyDeps` in mill or `libraryDependencies` in sbt are used during the code-generation process. This makes Smithy4s more consistent with itself, as the same behaviour is applied to internal dependencies. It is also argueably more intuitive to do it this way. --- .../dependencies-only/build.sbt | 2 +- .../bar/src/main/smithy/bar.smithy | 2 + .../baz/src/main/scala/Test.scala | 28 --------- .../baz/src/main/smithy/bar.smithy | 12 ---- .../multimodule-staged/build.sbt | 23 +++----- .../foo/src/main/smithy/foo.smithy | 2 + .../codegen-plugin/multimodule-staged/test | 6 -- .../codegen/Smithy4sCodegenPlugin.scala | 11 ++-- .../markdown/01-overview/05-sharing-specs.md | 23 +++----- .../codegen/mill/Smithy4sModule.scala | 9 ++- .../multimodule-staged/bar/smithy/bar.smithy | 2 + .../multimodule-staged/baz/smithy/bar.smithy | 12 ---- .../baz/src/main/scala/Test.scala | 28 --------- .../multimodule-staged/foo/smithy/foo.smithy | 2 + .../codegen/mill/Smithy4sModuleSpec.scala | 59 ++++++------------- 15 files changed, 55 insertions(+), 166 deletions(-) delete mode 100644 modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/baz/src/main/scala/Test.scala delete mode 100644 modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/baz/src/main/smithy/bar.smithy delete mode 100644 modules/mill-codegen-plugin/test/resources/multimodule-staged/baz/smithy/bar.smithy delete mode 100644 modules/mill-codegen-plugin/test/resources/multimodule-staged/baz/src/main/scala/Test.scala diff --git a/modules/codegen-plugin/src/sbt-test/codegen-plugin/dependencies-only/build.sbt b/modules/codegen-plugin/src/sbt-test/codegen-plugin/dependencies-only/build.sbt index 0db320de5..3daf30450 100644 --- a/modules/codegen-plugin/src/sbt-test/codegen-plugin/dependencies-only/build.sbt +++ b/modules/codegen-plugin/src/sbt-test/codegen-plugin/dependencies-only/build.sbt @@ -6,6 +6,6 @@ lazy val p1 = project "aws.iam" ), libraryDependencies += "com.disneystreaming.smithy4s" %% "smithy4s-core" % smithy4sVersion.value, - libraryDependencies += "software.amazon.smithy" % "smithy-aws-iam-traits" % smithy4s.codegen.BuildInfo.smithyVersion % Smithy4sCompile, + libraryDependencies += "software.amazon.smithy" % "smithy-aws-iam-traits" % smithy4s.codegen.BuildInfo.smithyVersion % Smithy4s, Compile / smithy4sOutputDir := baseDirectory.value / "smithy_output" ) diff --git a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/bar/src/main/smithy/bar.smithy b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/bar/src/main/smithy/bar.smithy index 19b8d4847..6a6fd9c6a 100644 --- a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/bar/src/main/smithy/bar.smithy +++ b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/bar/src/main/smithy/bar.smithy @@ -3,8 +3,10 @@ $version: "2.0" namespace bar use foo#Foo +use aws.api#data // Checking that Foo can be found by virtue of the bar project depending on the foo project +@data("tagging") structure Bar { foo: Foo } diff --git a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/baz/src/main/scala/Test.scala b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/baz/src/main/scala/Test.scala deleted file mode 100644 index 1ee5acbc7..000000000 --- a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/baz/src/main/scala/Test.scala +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2021-2022 Disney Streaming - * - * Licensed under the Tomorrow Open Source Technology License, Version 1.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://disneystreaming.github.io/TOST-1.0.txt - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package baz - -import foo._ -import bar._ - -object BarTest { - - def main(args: Array[String]): Unit = { - println(Baz(Some(Foo(Some(1))))) - } - -} diff --git a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/baz/src/main/smithy/bar.smithy b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/baz/src/main/smithy/bar.smithy deleted file mode 100644 index f447cb167..000000000 --- a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/baz/src/main/smithy/bar.smithy +++ /dev/null @@ -1,12 +0,0 @@ -$version: "2.0" - -namespace baz - -use foo#Foo - -// Checking that Foo can be found by virtue of the upstream `bar` project -// defined as a compile-scope library dependency was published with an indication -// in the manifest that it used the `foo` project for code generation. -structure Baz { - foo: Foo -} diff --git a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/build.sbt b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/build.sbt index 707eb5f39..d1063dd74 100644 --- a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/build.sbt +++ b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/build.sbt @@ -7,29 +7,22 @@ ThisBuild / organization := "foobar" lazy val foo = (project in file("foo")) .enablePlugins(Smithy4sCodegenPlugin) .settings( + // foo refers to smithy-aws-traits explicitly as a code-gen only dep, and upon publishing, + // this information is stored in the manifest of bar's jar, for downstream consumption + smithy4sAllowedNamespaces := List("aws.api", "foo"), libraryDependencies ++= Seq( - "com.disneystreaming.smithy4s" %% "smithy4s-core" % smithy4sVersion.value + "com.disneystreaming.smithy4s" %% "smithy4s-core" % smithy4sVersion.value, + "software.amazon.smithy" % "smithy-aws-traits" % smithy4s.codegen.BuildInfo.smithyVersion % Smithy4s ) ) lazy val bar = (project in file("bar")) .enablePlugins(Smithy4sCodegenPlugin) .settings( - // Bar refers to foo explicitly in its ivy deps, and upon publishing, - // this information is stored in the manifest of bar's jar, for downstream - // consumption - libraryDependencies ++= Seq( - "foobar" %% "foo" % version.value % Smithy4sCompile - ) - ) - -lazy val baz = (project in file("baz")) - .enablePlugins(Smithy4sCodegenPlugin) - .settings( - // baz depend on bar, and an assumption is made that baz may depend on the same smithy models - // that bar depended on for its own codegen. Therefore, these are retrieved from bar's manifest, + // bar depend on foo as a library, and an assumption is made that bar may depend on the same smithy models + // that foo depended on for its own codegen. Therefore, these are retrieved from foo's manifest, // resolved and added to the list of jars to seek smithy models from during code generation libraryDependencies ++= Seq( - "foobar" %% "bar" % version.value + "foobar" %% "foo" % version.value ) ) diff --git a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/foo/src/main/smithy/foo.smithy b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/foo/src/main/smithy/foo.smithy index 28ed79088..82cd4d2c9 100644 --- a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/foo/src/main/smithy/foo.smithy +++ b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/foo/src/main/smithy/foo.smithy @@ -3,10 +3,12 @@ $version: "2.0" namespace foo use alloy#uuidFormat +use aws.api#data structure Foo { a: Integer } +@data("tagging") @uuidFormat string MyUUID diff --git a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/test b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/test index f5a813173..9b6f61e5d 100644 --- a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/test +++ b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/test @@ -3,12 +3,6 @@ > bar/compile $ exists bar/target/scala-2.13/src_managed/main/bar/Bar.scala $ absent bar/target/scala-2.13/src_managed/main/foo/Foo.scala -> bar/publishLocal -> baz/compile -$ exists baz/target/scala-2.13/src_managed/main/baz/Baz.scala -$ absent baz/target/scala-2.13/src_managed/main/foo/Foo.scala -$ absent baz/target/scala-2.13/src_managed/main/bar/Bar.scala # check if code can run, this can reveal runtime issues# such as initialization errors > bar/run -> baz/run diff --git a/modules/codegen-plugin/src/smithy4s/codegen/Smithy4sCodegenPlugin.scala b/modules/codegen-plugin/src/smithy4s/codegen/Smithy4sCodegenPlugin.scala index 4c73d6752..65e9a06d7 100644 --- a/modules/codegen-plugin/src/smithy4s/codegen/Smithy4sCodegenPlugin.scala +++ b/modules/codegen-plugin/src/smithy4s/codegen/Smithy4sCodegenPlugin.scala @@ -69,7 +69,8 @@ object Smithy4sCodegenPlugin extends AutoPlugin { "List of jars for external dependencies that should be added to the classpath used by Smithy4s during code-generation", "The smithy files and smithy validators contained by these jars are included in the Smithy4s code-generation process", "Namespaces that were used for code generation in these dependencies will be excluded from code generation in this project.", - "By default, this includes the jars resulting from the resolution of library dependencies annotated with the `Smithy4s` configuration" + "By default, this includes the jars resulting from the resolution of library dependencies annotated with the `Smithy4s` configuration", + "as well as the `standard` library dependencies" ).mkString(" ") ) @@ -109,10 +110,6 @@ object Smithy4sCodegenPlugin extends AutoPlugin { "Dependencies containing Smithy code, used at codegen-time only." ) - // A shortcut to `"smithy4s,compile"` to use when defining dependencies that need to be consumed - // at odegen-time (for their smithy specs) AND at compile-time (for the Scala-code they contain) - val Smithy4sCompile = List(Smithy4s, Compile).map(_.name).mkString(",") - val smithy4sModelTransformers = settingKey[List[String]]( "List of transformer names that should be applied to the model prior to codegen" @@ -138,7 +135,9 @@ object Smithy4sCodegenPlugin extends AutoPlugin { config / smithy4sExternalDependenciesAsJars := { val updateReport = (config / update).value +: (config / transitiveUpdate).value - findCodeGenDependencies(updateReport) + val externalDependencyFiles = + (config / externalDependencyClasspath).value.map(_.data) + findCodeGenDependencies(updateReport) ++ externalDependencyFiles }, config / smithy4sInternalDependenciesAsJars := { (config / internalDependencyAsJars).value.map(_.data) diff --git a/modules/docs/markdown/01-overview/05-sharing-specs.md b/modules/docs/markdown/01-overview/05-sharing-specs.md index af5d00bab..d4c022139 100644 --- a/modules/docs/markdown/01-overview/05-sharing-specs.md +++ b/modules/docs/markdown/01-overview/05-sharing-specs.md @@ -134,26 +134,20 @@ Smithy specs (and validators) it may contain. ## Artifacts containing both Smithy files and Smithy4s generated code When using Smithy4s, you may want to depend on artifacts that may have been built using Smithy4s, containing both Smithy specifications -and generated Scala code (or rather, JVM bytecode resulting from the compilation of generated Scala code). In this case, you have to tell your build tool that a dependency should be used both by Smithy4s at codegen-time, and by the Scala compiler at compile time. This is achieved by doing the following +and generated Scala code (or rather, JVM bytecode resulting from the compilation of generated Scala code). In this case, you don't have to +do anything particular, the simple fact of declaring a library dependency will result in the smithy files contained by that dependency to be +used during the "compilation" of your smithy specs during the code-generation process. ### SBT ```scala -libraryDependencies += "organisation" % "artifact" % "version" % Smithy4sCompile -``` - -Which is merely a shortcut for: - -```scala -libraryDependencies += "organisation" % "artifact" % "version" % "smithy4s,compile" +libraryDependencies += "organisation" % "artifact" % "version" ``` ### Mill ```scala -def compileAndCodegenDeps = T(Agg(ivy"organisation:artifact:version")) -def ivyDeps = T(super.ivyDeps() ++ compileAndCodegenDeps()) -def smithy4sIvyDeps = T(super.smithy4sIvyDeps() ++ compileAndCodegenDeps()) +def ivyDeps = T(Agg(ivy"organisation:artifact:version")) ``` ### Consequence @@ -162,9 +156,10 @@ Because the upstream usage of Smithy4s will have resulted in the creation of met ## Artifacts containing Smithy4s generated code : dependency tracking -When packaging a project/module via SBT or Mill, Smithy4s adds a line to the Jar manifest of the project, informing downstream projects of library dependencies that may have been used during the code-generation of this project/module. +When packaging a project/module via SBT or Mill, Smithy4s adds a line to the Jar manifest of the project, informing downstream projects of library dependencies that may have been used during the code-generation of this project/module (ie, the dependencies annotated with `% Smithy4s` in SBT, and the ones provided by +`smithy4sIvyDeps` in mill). -This information is used automatically by downstream usage of Smithy4s, which automatically pulls additional jars that would be specified +This information is used automatically by downstream projects using Smithy4s, which automatically pulls additional jars that would be specified in this bit of metadata. So, for instance, if you have @@ -195,7 +190,7 @@ lazy val downstream = (project in file("foo")) .settings( libraryDependencies ++= Seq( // compile/runtime dependency that contains Smithy4s-generated code but doesn't contain smithy files - "foobar" % "upstream" % "0.0.1" + "foobar" %% "upstream" % "0.0.1" ) ) ``` diff --git a/modules/mill-codegen-plugin/src/smithy4s/codegen/mill/Smithy4sModule.scala b/modules/mill-codegen-plugin/src/smithy4s/codegen/mill/Smithy4sModule.scala index b7035210e..87f909da4 100644 --- a/modules/mill-codegen-plugin/src/smithy4s/codegen/mill/Smithy4sModule.scala +++ b/modules/mill-codegen-plugin/src/smithy4s/codegen/mill/Smithy4sModule.scala @@ -103,7 +103,7 @@ trait Smithy4sModule extends ScalaModule { .flatten } - def smithy4sResolvedIvyDeps: T[Agg[PathRef]] = + private def smithy4sResolvedIvyDeps: T[Agg[PathRef]] = resolveDeps(smithy4sTransitiveIvyDeps) def smithy4sExternalCodegenIvyDeps: T[Agg[Dep]] = T { @@ -121,11 +121,14 @@ trait Smithy4sModule extends ScalaModule { } } - def smithy4sResolvedExternalCodegenIvyDeps: T[Agg[PathRef]] = + private def smithy4sResolvedExternalCodegenIvyDeps: T[Agg[PathRef]] = resolveDeps(smithy4sExternalCodegenIvyDeps) def smithy4sAllDependenciesAsJars: T[Agg[PathRef]] = T { - smithy4sInternalDependenciesAsJars() ++ smithy4sResolvedIvyDeps() ++ smithy4sResolvedExternalCodegenIvyDeps() + resolvedIvyDeps() ++ + smithy4sInternalDependenciesAsJars() ++ + smithy4sResolvedIvyDeps() ++ + smithy4sResolvedExternalCodegenIvyDeps() } def smithy4sCodegen: T[(PathRef, PathRef)] = T { diff --git a/modules/mill-codegen-plugin/test/resources/multimodule-staged/bar/smithy/bar.smithy b/modules/mill-codegen-plugin/test/resources/multimodule-staged/bar/smithy/bar.smithy index 19b8d4847..6a6fd9c6a 100644 --- a/modules/mill-codegen-plugin/test/resources/multimodule-staged/bar/smithy/bar.smithy +++ b/modules/mill-codegen-plugin/test/resources/multimodule-staged/bar/smithy/bar.smithy @@ -3,8 +3,10 @@ $version: "2.0" namespace bar use foo#Foo +use aws.api#data // Checking that Foo can be found by virtue of the bar project depending on the foo project +@data("tagging") structure Bar { foo: Foo } diff --git a/modules/mill-codegen-plugin/test/resources/multimodule-staged/baz/smithy/bar.smithy b/modules/mill-codegen-plugin/test/resources/multimodule-staged/baz/smithy/bar.smithy deleted file mode 100644 index f447cb167..000000000 --- a/modules/mill-codegen-plugin/test/resources/multimodule-staged/baz/smithy/bar.smithy +++ /dev/null @@ -1,12 +0,0 @@ -$version: "2.0" - -namespace baz - -use foo#Foo - -// Checking that Foo can be found by virtue of the upstream `bar` project -// defined as a compile-scope library dependency was published with an indication -// in the manifest that it used the `foo` project for code generation. -structure Baz { - foo: Foo -} diff --git a/modules/mill-codegen-plugin/test/resources/multimodule-staged/baz/src/main/scala/Test.scala b/modules/mill-codegen-plugin/test/resources/multimodule-staged/baz/src/main/scala/Test.scala deleted file mode 100644 index 1ee5acbc7..000000000 --- a/modules/mill-codegen-plugin/test/resources/multimodule-staged/baz/src/main/scala/Test.scala +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2021-2022 Disney Streaming - * - * Licensed under the Tomorrow Open Source Technology License, Version 1.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://disneystreaming.github.io/TOST-1.0.txt - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package baz - -import foo._ -import bar._ - -object BarTest { - - def main(args: Array[String]): Unit = { - println(Baz(Some(Foo(Some(1))))) - } - -} diff --git a/modules/mill-codegen-plugin/test/resources/multimodule-staged/foo/smithy/foo.smithy b/modules/mill-codegen-plugin/test/resources/multimodule-staged/foo/smithy/foo.smithy index 28ed79088..82cd4d2c9 100644 --- a/modules/mill-codegen-plugin/test/resources/multimodule-staged/foo/smithy/foo.smithy +++ b/modules/mill-codegen-plugin/test/resources/multimodule-staged/foo/smithy/foo.smithy @@ -3,10 +3,12 @@ $version: "2.0" namespace foo use alloy#uuidFormat +use aws.api#data structure Foo { a: Integer } +@data("tagging") @uuidFormat string MyUUID diff --git a/modules/mill-codegen-plugin/test/src/smithy4s/codegen/mill/Smithy4sModuleSpec.scala b/modules/mill-codegen-plugin/test/src/smithy4s/codegen/mill/Smithy4sModuleSpec.scala index 552db89c8..bc849aea4 100644 --- a/modules/mill-codegen-plugin/test/src/smithy4s/codegen/mill/Smithy4sModuleSpec.scala +++ b/modules/mill-codegen-plugin/test/src/smithy4s/codegen/mill/Smithy4sModuleSpec.scala @@ -227,7 +227,7 @@ class Smithy4sModuleSpec extends munit.FunSuite { (localIvyRepo.toNIO.toUri.toString + "/") +: coursier.ivy.Pattern.default, dropInfoAttributes = true ) - super.repositoriesTask() ++ Seq(ivy2Local) + Seq(ivy2Local) ++ super.repositoriesTask() } def pomSettings: T[PomSettings] = PomSettings( "foo", @@ -242,41 +242,37 @@ class Smithy4sModuleSpec extends munit.FunSuite { } object foo extends Base { - override def artifactName: T[String] = "foo" + override def artifactName: T[String] = "foo-mill" override def scalaVersion = "2.13.10" override def ivyDeps = Agg(coreDep) + override def smithy4sAllowedNamespaces: T[Option[Set[String]]] = + Some(Set("aws.api", "foo")) override def millSourcePath = resourcePath / "multimodule-staged" / "foo" + // foo refers to smithy-aws-traits explicitly as a code-gen only dep, and upon publishing, + // this information is stored in the manifest of bar's jar, for downstream consumption + override def smithy4sIvyDeps = Agg( + ivy"software.amazon.smithy:smithy-aws-traits:${smithy4s.codegen.BuildInfo.smithyVersion}" + ) } object bar extends Base { - override def artifactName: T[String] = "bar" + override def artifactName: T[String] = "bar-mill" override def scalaVersion = "2.13.10" - // Bar refers to foo explicitly in its ivy deps, and upon publishing, - // this information is stored in the manifest of bar's jar, for downstream - // consumption - override def smithy4sIvyDeps = - Agg(ivy"${pomSettings().organization}::foo:${publishVersion()}") - override def ivyDeps = T(smithy4sIvyDeps()) - override def millSourcePath = resourcePath / "multimodule-staged" / "bar" - } - - object baz extends Base { - override def artifactName: T[String] = "baz" - override def scalaVersion = "2.13.10" - // baz depend on bar, and an assumption is made that baz may depend on the same smithy models - // that bar depended on for its own codegen. Therefore, these are retrieved from bar's manifest, + // bar depend on foo as a library, and an assumption is made that bar may depend on the same smithy models + // that foo depended on for its own codegen. Therefore, these are retrieved from foo's manifest, // resolved and added to the list of jars to seek smithy models from during code generation - override def ivyDeps = - Agg(ivy"${pomSettings().organization}::bar:${publishVersion()}") - override def millSourcePath = resourcePath / "multimodule-staged" / "baz" + override def ivyDeps = T { + super.ivyDeps() ++ Agg( + ivy"${pomSettings().organization}::foo-mill:${publishVersion()}" + ) + } + override def millSourcePath = resourcePath / "multimodule-staged" / "bar" } val fooEv = testKit.staticTestEvaluator(foo)(FullName("multi-module-staged-foo")) val barEv = testKit.staticTestEvaluator(bar)(FullName("multi-module-staged-bar")) - val bazEv = - testKit.staticTestEvaluator(baz)(FullName("multi-module-staged-bar")) taskWorks(foo.publishLocal(localIvyRepo.toString()), fooEv) taskWorks(bar.compile, barEv) @@ -290,25 +286,6 @@ class Smithy4sModuleSpec extends munit.FunSuite { shouldExist = false ) - taskWorks(bar.publishLocal(localIvyRepo.toString()), barEv) - taskWorks(baz.compile, bazEv) - - checkFileExist( - bazEv.outPath / "smithy4sOutputDir.dest" / "scala" / "baz" / "Baz.scala", - shouldExist = true - ) - checkFileExist( - bazEv.outPath / "smithy4sOutputDir.dest" / "scala" / "foo" / "Baz.scala", - shouldExist = false - ) - checkFileExist( - bazEv.outPath / "smithy4sOutputDir.dest" / "scala" / "bar" / "Bar.scala", - shouldExist = false - ) - - taskWorks(bar.run(), barEv) - taskWorks(baz.run(), bazEv) - } private def compileWorks( From 8d406a17d869338625d929b91c32e9aec7538756 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20M=C3=A9lois?= Date: Wed, 23 Nov 2022 10:21:07 +0100 Subject: [PATCH 9/9] Cleanup --- .../src/smithy4s/codegen/Smithy4sCodegenPlugin.scala | 2 +- modules/docs/markdown/01-overview/05-sharing-specs.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/codegen-plugin/src/smithy4s/codegen/Smithy4sCodegenPlugin.scala b/modules/codegen-plugin/src/smithy4s/codegen/Smithy4sCodegenPlugin.scala index 65e9a06d7..d839107cf 100644 --- a/modules/codegen-plugin/src/smithy4s/codegen/Smithy4sCodegenPlugin.scala +++ b/modules/codegen-plugin/src/smithy4s/codegen/Smithy4sCodegenPlugin.scala @@ -298,7 +298,7 @@ object Smithy4sCodegenPlugin extends AutoPlugin { output = os.Path(outputPath), resourceOutput = os.Path(resourceOutputPath), skip = skipSet, - discoverModels = false, // we need protocol here + discoverModels = false, allowedNS = allowedNamespaces, excludedNS = excludedNamespaces, repositories = res, diff --git a/modules/docs/markdown/01-overview/05-sharing-specs.md b/modules/docs/markdown/01-overview/05-sharing-specs.md index d4c022139..5c835e4ce 100644 --- a/modules/docs/markdown/01-overview/05-sharing-specs.md +++ b/modules/docs/markdown/01-overview/05-sharing-specs.md @@ -176,7 +176,8 @@ lazy val upstream = (project in file("foo")) ) ``` -and publish this project to an artifact repository, the Jar manifest will contain the following line : +and publish this project to an artifact repository, the Jar manifest will contain a line with the relevant +dependencies (comma separated if there are more than one) : ``` smithy4sDependencies: software.amazon.smithy:smithy-aws-iam-traits:1.14.1