From d35695e67151fbcd1cc4b137defd403ace3970ff Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Mon, 25 Nov 2024 12:33:09 +0100 Subject: [PATCH 01/18] Add BOM / dependency management support --- .../ROOT/pages/fundamentals/library-deps.adoc | 28 + .../ROOT/pages/javalib/dependencies.adoc | 5 + .../ROOT/pages/kotlinlib/dependencies.adoc | 6 + .../ROOT/pages/scalalib/dependencies.adoc | 5 + .../bom-1-external-bom/build.mill | 21 + .../bom-2-dependency-management/build.mill | 54 ++ example/package.mill | 1 + main/util/src/mill/util/CoursierSupport.scala | 37 +- .../src/mill/scalalib/CoursierModule.scala | 31 ++ scalalib/src/mill/scalalib/Dep.scala | 2 + scalalib/src/mill/scalalib/JavaModule.scala | 171 ++++++- .../src/mill/scalalib/PublishModule.scala | 77 ++- scalalib/src/mill/scalalib/publish/Ivy.scala | 26 +- scalalib/src/mill/scalalib/publish/Pom.scala | 75 ++- .../test/src/mill/scalalib/BomTests.scala | 484 ++++++++++++++++++ .../src/mill/scalalib/publish/PomTests.scala | 5 +- 16 files changed, 986 insertions(+), 42 deletions(-) create mode 100644 example/fundamentals/library-deps/bom-1-external-bom/build.mill create mode 100644 example/fundamentals/library-deps/bom-2-dependency-management/build.mill create mode 100644 scalalib/test/src/mill/scalalib/BomTests.scala diff --git a/docs/modules/ROOT/pages/fundamentals/library-deps.adoc b/docs/modules/ROOT/pages/fundamentals/library-deps.adoc index 8c703ab8175..00500ae6df5 100644 --- a/docs/modules/ROOT/pages/fundamentals/library-deps.adoc +++ b/docs/modules/ROOT/pages/fundamentals/library-deps.adoc @@ -96,6 +96,34 @@ def runIvyDeps = Agg( It is also possible to use a higher version of the same library dependencies already defined in `ivyDeps`, to ensure you compile against a minimal API version, but actually run with the latest available version. +== Dependency management + +Dependency management consists in listing dependencies whose versions we want to force. Having +a dependency in dependency management doesn't mean that this dependency will be fetched, only +that + +* if it ends up being fetched transitively, its version will be forced to the one in dependency management + +* if its version is empty in an `ivyDeps` section in Mill, the version from dependency management will be used + +Dependency management also allows to add exclusions to dependencies, both explicit dependencies and +transitive ones. + +Dependency management can be passed to Mill in two ways: + +* via external Maven BOMs, like https://repo1.maven.org/maven2/com/google/cloud/libraries-bom/26.50.0/libraries-bom-26.50.0.pom[this one], +whose Maven coordinates are `com.google.cloud:libraries-bom:26.50.0` + +* via the `dependencyManagement` task, that allows to directly list dependencies whose versions we want to enforce + +=== External BOMs + +include::partial$example/fundamentals/library-deps/bom-1-external-bom.adoc[] + +=== Dependency management task + +include::partial$example/fundamentals/library-deps/bom-2-dependency-management.adoc[] + == Searching For Dependency Updates include::partial$example/fundamentals/dependencies/1-search-updates.adoc[] diff --git a/docs/modules/ROOT/pages/javalib/dependencies.adoc b/docs/modules/ROOT/pages/javalib/dependencies.adoc index 76ebe8d4449..ac2c01aef22 100644 --- a/docs/modules/ROOT/pages/javalib/dependencies.adoc +++ b/docs/modules/ROOT/pages/javalib/dependencies.adoc @@ -13,6 +13,11 @@ include::partial$example/javalib/dependencies/1-ivy-deps.adoc[] include::partial$example/javalib/dependencies/2-run-compile-deps.adoc[] +== Dependency Management + +Mill has support for dependency management, see the +xref:fundamentals/library-deps.adoc#_dependency_management[Dependency Management section] +in xref:fundamentals/library-deps.adoc[]. == Unmanaged Jars diff --git a/docs/modules/ROOT/pages/kotlinlib/dependencies.adoc b/docs/modules/ROOT/pages/kotlinlib/dependencies.adoc index a970a256d4e..9670e311702 100644 --- a/docs/modules/ROOT/pages/kotlinlib/dependencies.adoc +++ b/docs/modules/ROOT/pages/kotlinlib/dependencies.adoc @@ -19,6 +19,12 @@ include::partial$example/kotlinlib/dependencies/1-ivy-deps.adoc[] include::partial$example/kotlinlib/dependencies/2-run-compile-deps.adoc[] +== Dependency Management + +Mill has support for dependency management, see the +xref:fundamentals/library-deps.adoc#_dependency_management[Dependency Management section] +in xref:fundamentals/library-deps.adoc[]. + == Unmanaged Jars include::partial$example/kotlinlib/dependencies/3-unmanaged-jars.adoc[] diff --git a/docs/modules/ROOT/pages/scalalib/dependencies.adoc b/docs/modules/ROOT/pages/scalalib/dependencies.adoc index c4c0bf2ef89..902d8b7be15 100644 --- a/docs/modules/ROOT/pages/scalalib/dependencies.adoc +++ b/docs/modules/ROOT/pages/scalalib/dependencies.adoc @@ -16,6 +16,11 @@ include::partial$example/scalalib/dependencies/1-ivy-deps.adoc[] include::partial$example/scalalib/dependencies/2-run-compile-deps.adoc[] +== Dependency Management + +Mill has support for dependency management, see the +xref:fundamentals/library-deps.adoc#_dependency_management[Dependency Management section] +in xref:fundamentals/library-deps.adoc[]. == Unmanaged Jars diff --git a/example/fundamentals/library-deps/bom-1-external-bom/build.mill b/example/fundamentals/library-deps/bom-1-external-bom/build.mill new file mode 100644 index 00000000000..eeb3a5344c5 --- /dev/null +++ b/example/fundamentals/library-deps/bom-1-external-bom/build.mill @@ -0,0 +1,21 @@ +// Pass an external BOM to a `JavaModule` / `ScalaModule` / `KotlinModule` with `bomDeps`, like + +//// SNIPPET:BUILD1 +package build +import mill._, javalib._ + +object bom extends JavaModule { + def bomDeps = Agg( + ivy"com.google.cloud:libraries-bom:26.50.0" + ) + def ivyDeps = Agg( + ivy"io.grpc:grpc-protobuf" + ) +} + +// The version of grpc-protobuf (`io.grpc:grpc-protobuf`) isn't written down here, so the version +// from the BOM, `1.67.1` is used. +// +// Also, by default, grpc-protobuf `1.67.1` pulls version `3.25.3` of protobuf-java (`com.google.protobuf:protobuf-java`) . +// But the BOM specifies another version for that dependency, `4.28.3`, so +// protobuf-java `4.28.3` ends up being pulled here. diff --git a/example/fundamentals/library-deps/bom-2-dependency-management/build.mill b/example/fundamentals/library-deps/bom-2-dependency-management/build.mill new file mode 100644 index 00000000000..22f1fe18de3 --- /dev/null +++ b/example/fundamentals/library-deps/bom-2-dependency-management/build.mill @@ -0,0 +1,54 @@ +// Pass dependencies to `dependencyManagement` in a `JavaModule` / `ScalaModule` / `KotlinModule`, like + +//// SNIPPET:BUILD1 +package build +import mill._, javalib._ + +object dependencyManagement extends JavaModule { + def dependencyManagement = Agg( + ivy"com.google.protobuf:protobuf-java:4.28.3", + ivy"io.grpc:grpc-protobuf:1.67.1" + ) + def ivyDeps = Agg( + ivy"io.grpc:grpc-protobuf" + ) +} + +// The version of grpc-protobuf (`io.grpc:grpc-protobuf`) isn't written down here, so the version +// found in `dependencyManagement`, `1.67.1` is used. +// +// Also, by default, grpc-protobuf `1.67.1` pulls version `3.25.3` of protobuf-java (`com.google.protobuf:protobuf-java`) . +// But `dependencyManagement` specifies another version for that dependency, `4.28.3`, so +// protobuf-java `4.28.3` ends up being pulled here. + +// One can also add exclusions via dependency management, like + +object dependencyManagementWithVersionAndExclusions extends JavaModule { + def dependencyManagement = Agg( + ivy"io.grpc:grpc-protobuf:1.67.1" + .exclude(("com.google.protobuf", "protobuf-java")) + ) + def ivyDeps = Agg( + ivy"io.grpc:grpc-protobuf" + ) +} + +// Here, grpc-protobuf has an empty version in `ivyDeps`, so the one in `dependencyManagement`, +// `1.67.1`, is used. Also, `com.google.protobuf:protobuf-java` is excluded from grpc-protobuf +// in `dependencyManagement`, so it ends up being excluded from it in `ivyDeps` too. + +// If one wants to add exclusions via `dependencyManagement`, specifying a version is optional, +// like + +object dependencyManagementWithExclusions extends JavaModule { + def dependencyManagement = Agg( + ivy"io.grpc:grpc-protobuf" + .exclude(("com.google.protobuf", "protobuf-java")) + ) + def ivyDeps = Agg( + ivy"io.grpc:grpc-protobuf:1.67.1" + ) +} + +// Here, given that grpc-protobuf is fetched during dependency resolution, +// `com.google.protobuf:protobuf-java` is excluded from it because of the dependency management. diff --git a/example/package.mill b/example/package.mill index b7f53d51527..1705260e539 100644 --- a/example/package.mill +++ b/example/package.mill @@ -81,6 +81,7 @@ object `package` extends RootModule with Module { object cross extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "cross")) object `out-dir` extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "out-dir")) object libraries extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "libraries")) + object `library-deps` extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "library-deps")) } object depth extends Module { diff --git a/main/util/src/mill/util/CoursierSupport.scala b/main/util/src/mill/util/CoursierSupport.scala index c44b32c3c65..6f6ccff62b2 100644 --- a/main/util/src/mill/util/CoursierSupport.scala +++ b/main/util/src/mill/util/CoursierSupport.scala @@ -31,6 +31,20 @@ trait CoursierSupport { ctx.fold(cache)(c => cache.withLogger(new TickerResolutionLogger(c))) } + private def isLocalTestDep(dep: Dependency): Option[Seq[PathRef]] = { + val org = dep.module.organization.value + val name = dep.module.name.value + val classpathKey = s"$org-$name" + + val classpathResourceText = + try Some(os.read( + os.resource(getClass.getClassLoader) / "mill/local-test-overrides" / classpathKey + )) + catch { case e: os.ResourceNotFoundException => None } + + classpathResourceText.map(_.linesIterator.map(s => PathRef(os.Path(s))).toSeq) + } + /** * Resolve dependencies using Coursier. * @@ -51,26 +65,8 @@ trait CoursierSupport { artifactTypes: Option[Set[Type]] = None, resolutionParams: ResolutionParams = ResolutionParams() ): Result[Agg[PathRef]] = { - def isLocalTestDep(dep: Dependency): Option[Seq[PathRef]] = { - val org = dep.module.organization.value - val name = dep.module.name.value - val classpathKey = s"$org-$name" - - val classpathResourceText = - try Some(os.read( - os.resource(getClass.getClassLoader) / "mill/local-test-overrides" / classpathKey - )) - catch { case e: os.ResourceNotFoundException => None } - - classpathResourceText.map(_.linesIterator.map(s => PathRef(os.Path(s))).toSeq) - } - - val (localTestDeps, remoteDeps) = deps.iterator.toSeq.partitionMap(d => - isLocalTestDep(d) match { - case None => Right(d) - case Some(vs) => Left(vs) - } - ) + val (localTestDeps, remoteDeps) = + deps.iterator.toSeq.partitionMap(d => isLocalTestDep(d).toLeft(d)) val resolutionRes = resolveDependenciesMetadataSafe( repositories, @@ -262,6 +258,7 @@ trait CoursierSupport { val rootDeps = deps.iterator .map(d => mapDependencies.fold(d)(_.apply(d))) + .filter(dep => isLocalTestDep(dep).isEmpty) .toSeq val forceVersions = force.iterator diff --git a/scalalib/src/mill/scalalib/CoursierModule.scala b/scalalib/src/mill/scalalib/CoursierModule.scala index 0b687f66646..c7c483cceb1 100644 --- a/scalalib/src/mill/scalalib/CoursierModule.scala +++ b/scalalib/src/mill/scalalib/CoursierModule.scala @@ -231,6 +231,37 @@ object CoursierModule { sources: Boolean ): Agg[PathRef] = resolveDeps(deps, sources, None) + + /** + * Processes dependencies and BOMs with coursier + * + * This makes coursier read and process BOM dependencies, and fill version placeholders + * in dependencies with the BOMs. + * + * Note that this doesn't throw when a version placeholder cannot be filled, and just leaves + * the placeholder behind. + * + * @param deps dependencies that might have placeholder versions ("_" as version) + * @param resolutionParams coursier resolution parameters + * @return dependencies with version placeholder filled + */ + def processDeps[T: CoursierModule.Resolvable]( + deps: IterableOnce[T], + resolutionParams: ResolutionParams = ResolutionParams() + ): Seq[Dependency] = { + val deps0 = deps + .map(implicitly[CoursierModule.Resolvable[T]].bind(_, bind)) + val res = Lib.resolveDependenciesMetadataSafe( + repositories = repositories, + deps = deps0, + mapDependencies = mapDependencies, + customizer = customizer, + coursierCacheCustomizer = coursierCacheCustomizer, + ctx = ctx, + resolutionParams = resolutionParams + ).getOrThrow + res.processedRootDependencies + } } sealed trait Resolvable[T] { diff --git a/scalalib/src/mill/scalalib/Dep.scala b/scalalib/src/mill/scalalib/Dep.scala index ac2569ada99..3b02d8daa4d 100644 --- a/scalalib/src/mill/scalalib/Dep.scala +++ b/scalalib/src/mill/scalalib/Dep.scala @@ -118,6 +118,8 @@ object Dep { } (module.split(':') match { + case Array(a, b) => Dep(a, b, "_", cross = empty(platformed = false)) + case Array(a, "", b) => Dep(a, b, "_", cross = Binary(platformed = false)) case Array(a, b, c) => Dep(a, b, c, cross = empty(platformed = false)) case Array(a, b, "", c) => Dep(a, b, c, cross = empty(platformed = true)) case Array(a, "", b, c) => Dep(a, b, c, cross = Binary(platformed = false)) diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 1c901525557..a2498b669e1 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -1,7 +1,7 @@ package mill package scalalib -import coursier.core.Resolution +import coursier.core.{BomDependency, Configuration, DependencyManagement, Resolution} import coursier.parse.JavaOrScalaModule import coursier.parse.ModuleParser import coursier.util.ModuleMatcher @@ -159,6 +159,139 @@ trait JavaModule */ def runIvyDeps: T[Agg[Dep]] = Task { Agg.empty[Dep] } + /** + * Any BOM dependencies you want to add to this Module, in the format + * ivy"org:name:version" + */ + def bomDeps: T[Agg[Dep]] = Task { Agg.empty[Dep] } + + def allBomDeps: Task[Agg[BomDependency]] = Task.Anon { + val modVerOrMalformed = + bomDeps().map(bindDependency()).map { bomDep => + val fromModVer = coursier.core.Dependency(bomDep.dep.module, bomDep.dep.version) + if (fromModVer == bomDep.dep) + Right(bomDep.dep.asBomDependency) + else + Left(bomDep) + } + + val malformed = modVerOrMalformed.collect { + case Left(malformedBomDep) => + malformedBomDep + } + if (malformed.isEmpty) + modVerOrMalformed.collect { + case Right(bomDep) => bomDep + } + else + throw new Exception( + "Found BOM dependencies with invalid parameters:" + System.lineSeparator() + + malformed.map("- " + _.dep + System.lineSeparator()).mkString + + "Only organization, name, and version are accepted." + ) + } + + /** + * Dependency management data + * + * Versions and exclusions in dependency management override those of transitive dependencies, + * while they have no effect if the corresponding dependency isn't pulled during dependency + * resolution. + * + * For example, the following forces com.lihaoyi::os-lib to version 0.11.3, and + * excludes org.slf4j:slf4j-api from com.lihaoyi::cask that it forces to version 0.9.4 + * {{{ + * def dependencyManagement = super.dependencyManagement() ++ Agg( + * ivy"com.lihaoyi::os-lib:0.11.3", + * ivy"com.lihaoyi::cask:0.9.4".exclude("org.slf4j", "slf4j-api") + * ) + * }}} + */ + def dependencyManagement: T[Agg[Dep]] = Task { Agg.empty[Dep] } + + private def addBoms( + dep: coursier.core.Dependency, + bomDeps: Seq[coursier.core.BomDependency], + depMgmt: Seq[(DependencyManagement.Key, DependencyManagement.Values)], + depMgmtMap: DependencyManagement.Map, + overrideVersions: Boolean = false + ): coursier.core.Dependency = { + val depMgmtKey = DependencyManagement.Key( + dep.module.organization, + dep.module.name, + coursier.core.Type.jar, + dep.publication.classifier + ) + val versionOverrideOpt = + if (dep.version == "_") depMgmtMap.get(depMgmtKey).map(_.version) + else None + val extraExclusions = depMgmtMap.get(depMgmtKey).map(_.minimizedExclusions) + dep + // add BOM coordinates - coursier will handle the rest + .addBomDependencies( + if (overrideVersions) bomDeps.map(_.withForceOverrideVersions(overrideVersions)) + else bomDeps + ) + // add dependency management ourselves: + // - overrides meant to apply to transitive dependencies + // - fill version if it's empty + // - add extra exclusions from dependency management + .withOverrides(dep.overrides ++ depMgmt) + .withVersion(versionOverrideOpt.getOrElse(dep.version)) + .withMinimizedExclusions( + extraExclusions.fold(dep.minimizedExclusions)(dep.minimizedExclusions.join(_)) + ) + } + + /** + * Data from dependencyManagement, converted to a type ready to be passed to coursier + * for dependency resolution + */ + private def processedDependencyManagement(deps: Seq[coursier.core.Dependency]) + : Seq[(DependencyManagement.Key, DependencyManagement.Values)] = { + val keyValuesOrErrors = + deps.map { depMgmt => + val fromUsedValues = coursier.core.Dependency(depMgmt.module, depMgmt.version) + .withPublication(coursier.core.Publication( + "", + depMgmt.publication.`type`, + coursier.core.Extension.empty, + depMgmt.publication.classifier + )) + .withMinimizedExclusions(depMgmt.minimizedExclusions) + .withOptional(depMgmt.optional) + if (fromUsedValues == depMgmt) { + val key = DependencyManagement.Key( + depMgmt.module.organization, + depMgmt.module.name, + if (depMgmt.publication.`type`.isEmpty) coursier.core.Type.jar + else depMgmt.publication.`type`, + depMgmt.publication.classifier + ) + val values = DependencyManagement.Values( + Configuration.empty, + if (depMgmt.version == "_") "" // shouldn't be needed with future coursier versions + else depMgmt.version, + depMgmt.minimizedExclusions, + depMgmt.optional + ) + Right(key -> values) + } else + Left(depMgmt) + } + + val errors = keyValuesOrErrors.collect { + case Left(errored) => errored + } + if (errors.isEmpty) + keyValuesOrErrors.collect { case Right(kv) => kv } + else + throw new Exception( + "Found dependency management entries with invalid values. Only organization, name, version, type, classifier, exclusions, and optionality can be specified" + System.lineSeparator() + + errors.map("- " + _ + System.lineSeparator()).mkString + ) + } + /** * Default artifact types to fetch and put in the classpath. Add extra types * here if you'd like fancy artifact extensions to be fetched. @@ -326,13 +459,45 @@ trait JavaModule */ def unmanagedClasspath: T[Agg[PathRef]] = Task { Agg.empty[PathRef] } + /** + * Returns a function adding BOM and dependency management details of + * this module to a `coursier.core.Dependency` + */ + def processDependency( + overrideVersions: Boolean = false + ): Task[coursier.core.Dependency => coursier.core.Dependency] = Task.Anon { + val bomDeps0 = allBomDeps().toSeq.map(_.withConfig(Configuration.compile)) + val depMgmt = processedDependencyManagement( + dependencyManagement().toSeq.map(bindDependency()).map(_.dep) + ) + val depMgmtMap = depMgmt.toMap + + dep => + addBoms(dep, bomDeps0, depMgmt, depMgmtMap, overrideVersions = overrideVersions) + } + + /** + * The Ivy dependencies of this module, with BOM and dependency management details + * added to them. This should be used when propagating the dependencies transitively + * to other modules. + */ + def processedIvyDeps: Task[Agg[BoundDep]] = Task.Anon { + val processDependency0 = processDependency()() + allIvyDeps().map(bindDependency()).map { dep => + dep.copy(dep = processDependency0(dep.dep)) + } + } + /** * The transitive ivy dependencies of this module and all it's upstream modules. * This is calculated from [[ivyDeps]], [[mandatoryIvyDeps]] and recursively from [[moduleDeps]]. */ def transitiveIvyDeps: T[Agg[BoundDep]] = Task { - allIvyDeps().map(bindDependency()) ++ - T.traverse(moduleDepsChecked)(_.transitiveIvyDeps)().flatten + val processDependency0 = processDependency(overrideVersions = true)() + processedIvyDeps() ++ + T.traverse(moduleDepsChecked)(_.transitiveIvyDeps)().flatten.map { dep => + dep.copy(dep = processDependency0(dep.dep)) + } } /** diff --git a/scalalib/src/mill/scalalib/PublishModule.scala b/scalalib/src/mill/scalalib/PublishModule.scala index 796c3ff10f9..75c4eb93c33 100644 --- a/scalalib/src/mill/scalalib/PublishModule.scala +++ b/scalalib/src/mill/scalalib/PublishModule.scala @@ -70,7 +70,7 @@ trait PublishModule extends JavaModule { outer => def publishXmlDeps: Task[Agg[Dependency]] = Task.Anon { val ivyPomDeps = - (ivyDeps() ++ mandatoryIvyDeps()).map(resolvePublishDependency.apply().apply(_)) + processedIvyDeps().map(_.toDep).map(resolvePublishDependency.apply().apply(_)) val compileIvyPomDeps = compileIvyDeps() .map(resolvePublishDependency.apply().apply(_)) @@ -89,6 +89,20 @@ trait PublishModule extends JavaModule { outer => compileModulePomDeps.map(Dependency(_, Scope.Provided)) } + /** + * BOM dependency to specify in the POM + */ + def publishXmlBomDeps: Task[Agg[Dependency]] = Task.Anon { + bomDeps().map(resolvePublishDependency.apply().apply(_)) + } + + /** + * Dependency management to specify in the POM + */ + def publishXmlDepMgmt: Task[Agg[Dependency]] = Task.Anon { + dependencyManagement().map(resolvePublishDependency.apply().apply(_)) + } + def pom: T[PathRef] = Task { val pom = Pom( artifactMetadata(), @@ -97,15 +111,72 @@ trait PublishModule extends JavaModule { outer => pomSettings(), publishProperties(), packagingType = pomPackagingType, - parentProject = pomParentProject() + parentProject = pomParentProject(), + bomDependencies = publishXmlBomDeps(), + dependencyManagement = publishXmlDepMgmt() ) val pomPath = T.dest / s"${artifactId()}-${publishVersion()}.pom" os.write.over(pomPath, pom) PathRef(pomPath) } + /** + * Dependencies with version placeholder filled from BOMs, alongside with BOM data + */ + def bomDetails: T[(Map[coursier.core.Module, String], coursier.core.DependencyManagement.Map)] = + Task { + val processedDeps = defaultResolver().processDeps( + transitiveCompileIvyDeps() ++ transitiveIvyDeps(), + resolutionParams = resolutionParams() + ) + val depMgmt: coursier.core.DependencyManagement.Map = + if (processedDeps.isEmpty) Map.empty + else { + val overrides = processedDeps.map(_.overrides) + overrides.tail.foldLeft(overrides.head) { (acc, map) => + acc.filter { + case (key, values) => + map.get(key).contains(values) + } + } + } + (processedDeps.map(_.moduleVersion).toMap, depMgmt) + } + def ivy: T[PathRef] = Task { - val ivy = Ivy(artifactMetadata(), publishXmlDeps(), extraPublish()) + val (rootDepVersions, bomDepMgmt) = bomDetails() + val publishXmlDeps0 = publishXmlDeps().map { dep => + if (dep.artifact.version == "_") + dep.copy( + artifact = dep.artifact.copy( + version = rootDepVersions.getOrElse( + coursier.core.Module( + coursier.core.Organization(dep.artifact.group), + coursier.core.ModuleName(dep.artifact.id), + Map.empty + ), + "" /* throw instead? */ + ) + ) + ) + else + dep + } + val overrides = + dependencyManagement().toSeq.map(bindDependency()).map(_.dep) + .filter(depMgmt => depMgmt.version.nonEmpty && depMgmt.version != "_") + .map { depMgmt => + Ivy.Override( + depMgmt.module.organization.value, + depMgmt.module.name.value, + depMgmt.version + ) + } ++ + bomDepMgmt.map { + case (key, values) => + Ivy.Override(key.organization.value, key.name.value, values.version) + } + val ivy = Ivy(artifactMetadata(), publishXmlDeps0, extraPublish(), overrides) val ivyPath = T.dest / "ivy.xml" os.write.over(ivyPath, ivy) PathRef(ivyPath) diff --git a/scalalib/src/mill/scalalib/publish/Ivy.scala b/scalalib/src/mill/scalalib/publish/Ivy.scala index ae340ac0d75..75ba33dbec1 100644 --- a/scalalib/src/mill/scalalib/publish/Ivy.scala +++ b/scalalib/src/mill/scalalib/publish/Ivy.scala @@ -8,10 +8,13 @@ object Ivy { val head = "\n" + case class Override(organization: String, name: String, version: String) + def apply( artifact: Artifact, dependencies: Agg[Dependency], - extras: Seq[PublishInfo] = Seq.empty + extras: Seq[PublishInfo] = Seq.empty, + overrides: Seq[Override] = Nil ): String = { def renderExtra(e: PublishInfo): Elem = { @@ -49,13 +52,29 @@ object Ivy { {extras.map(renderExtra)} - {dependencies.map(renderDependency).toSeq} + + {dependencies.map(renderDependency).toSeq} + {overrides.map(renderOverride)} + val pp = new PrettyPrinter(120, 4) head + pp.format(xml).replaceAll(">", ">") } + // bin-compat shim + def apply( + artifact: Artifact, + dependencies: Agg[Dependency], + extras: Seq[PublishInfo] + ): String = + apply( + artifact, + dependencies, + extras, + Nil + ) + private def renderDependency(dep: Dependency): Elem = { if (dep.exclusions.isEmpty) } + private def renderOverride(override0: Override): Elem = + + private def depIvyConf(d: Dependency): String = { if (d.optional) "optional" else d.scope match { diff --git a/scalalib/src/mill/scalalib/publish/Pom.scala b/scalalib/src/mill/scalalib/publish/Pom.scala index 76d1e728d36..6940f25c47f 100644 --- a/scalalib/src/mill/scalalib/publish/Pom.scala +++ b/scalalib/src/mill/scalalib/publish/Pom.scala @@ -39,10 +39,15 @@ object Pom { pomSettings = pomSettings, properties = properties, packagingType = pomSettings.packaging, - parentProject = None + parentProject = None, + bomDependencies = Agg.empty[Dependency], + dependencyManagement = Agg.empty[Dependency] ) - @deprecated("Use overload with parentProject parameter instead", "Mill 0.12.1") + @deprecated( + "Use overload with parentProject, bomDependencies, and dependencyManagement parameters instead", + "Mill 0.12.1" + ) def apply( artifact: Artifact, dependencies: Agg[Dependency], @@ -57,7 +62,9 @@ object Pom { pomSettings = pomSettings, properties = properties, packagingType = packagingType, - parentProject = None + parentProject = None, + bomDependencies = Agg.empty[Dependency], + dependencyManagement = Agg.empty[Dependency] ) def apply( @@ -68,6 +75,29 @@ object Pom { properties: Map[String, String], packagingType: String, parentProject: Option[Artifact] + ): String = + apply( + artifact, + dependencies, + name, + pomSettings, + properties, + packagingType, + parentProject, + Agg.empty[Dependency], + Agg.empty[Dependency] + ) + + def apply( + artifact: Artifact, + dependencies: Agg[Dependency], + name: String, + pomSettings: PomSettings, + properties: Map[String, String], + packagingType: String, + parentProject: Option[Artifact], + bomDependencies: Agg[Dependency], + dependencyManagement: Agg[Dependency] ): String = { val xml = - {dependencies.map(renderDependency).iterator} + { + dependencies.map(renderDependency(_)).iterator ++ + bomDependencies.map(renderDependency(_, isImport = true)).iterator + } + + + {dependencyManagement.map(renderDependency(_)).iterator} + + val pp = new PrettyPrinter(120, 4) @@ -143,29 +181,39 @@ object Pom { {property._2}.copy(label = property._1) } - private def renderDependency(d: Dependency): Elem = { - val scope = d.scope match { - case Scope.Compile => NodeSeq.Empty - case Scope.Provided => provided - case Scope.Test => test - case Scope.Runtime => runtime - } + private def renderDependency(d: Dependency, isImport: Boolean = false): Elem = { + val scope = + if (isImport) import + else + d.scope match { + case Scope.Compile => NodeSeq.Empty + case Scope.Provided => provided + case Scope.Test => test + case Scope.Runtime => runtime + } + + val `type` = if (isImport) pom else NodeSeq.Empty val optional = if (d.optional) true else NodeSeq.Empty + val version = + if (d.artifact.version == "_") NodeSeq.Empty + else {d.artifact.version} + if (d.exclusions.isEmpty) {d.artifact.group} {d.artifact.id} - {d.artifact.version} + {version} {scope} + {`type`} {optional} else {d.artifact.group} {d.artifact.id} - {d.artifact.version} + {version} { d.exclusions.map(ex => @@ -175,6 +223,7 @@ object Pom { } {scope} + {`type`} {optional} } diff --git a/scalalib/test/src/mill/scalalib/BomTests.scala b/scalalib/test/src/mill/scalalib/BomTests.scala new file mode 100644 index 00000000000..db3ce96b356 --- /dev/null +++ b/scalalib/test/src/mill/scalalib/BomTests.scala @@ -0,0 +1,484 @@ +package mill +package scalalib + +import mill.scalalib.publish._ +import mill.testkit.{TestBaseModule, UnitTester} +import utest._ + +import scala.jdk.CollectionConverters._ + +object BomTests extends TestSuite { + + trait TestPublishModule extends PublishModule { + def pomSettings = PomSettings( + description = artifactName(), + organization = "com.lihaoyi.mill-tests", + url = "https://github.com/com-lihaoyi/mill", + licenses = Seq(License.`Apache-2.0`), + versionControl = VersionControl.github("com-lihaoyi", "mill"), + developers = Nil + ) + def publishVersion = "0.1.0-SNAPSHOT" + } + + object modules extends TestBaseModule { + object bom extends Module { + object placeholder extends JavaModule with TestPublishModule { + def bomDeps = Agg( + ivy"com.google.cloud:libraries-bom:26.50.0" + ) + def ivyDeps = Agg( + ivy"com.google.protobuf:protobuf-java" + ) + + object dependee extends JavaModule with TestPublishModule { + def moduleDeps = Seq( + placeholder + ) + } + + object subDependee extends JavaModule with TestPublishModule { + def moduleDeps = Seq( + dependee + ) + } + + object check extends JavaModule { + def ivyDeps = Agg( + ivy"com.google.protobuf:protobuf-java" + ) + } + } + + object versionOverride extends JavaModule with TestPublishModule { + def bomDeps = Agg( + ivy"com.google.cloud:libraries-bom:26.50.0" + ) + def ivyDeps = Agg( + ivy"com.thesamet.scalapb:scalapbc_2.13:0.9.8" + ) + + object dependee extends JavaModule with TestPublishModule { + def moduleDeps = Seq( + versionOverride + ) + } + + object subDependee extends JavaModule with TestPublishModule { + def moduleDeps = Seq( + dependee + ) + } + + object check extends JavaModule { + def ivyDeps = Agg( + ivy"com.thesamet.scalapb:scalapbc_2.13:0.9.8" + ) + } + } + + object invalid extends TestBaseModule { + object exclude extends JavaModule { + def bomDeps = Agg( + ivy"com.google.cloud:libraries-bom:26.50.0".exclude(("foo", "thing")) + ) + } + } + } + + object depMgmt extends JavaModule with TestPublishModule { + def ivyDeps = Agg( + ivy"com.thesamet.scalapb:scalapbc_2.13:0.9.8" + ) + def dependencyManagement = Agg( + ivy"com.google.protobuf:protobuf-java:4.28.3" + ) + + object transitive extends JavaModule with TestPublishModule { + def moduleDeps = Seq(depMgmt) + } + + object extraExclude extends JavaModule with TestPublishModule { + def ivyDeps = Agg( + ivy"com.lihaoyi:cask_2.13:0.9.4" + ) + def dependencyManagement = Agg( + // The exclude should be automatically added to the dependency above + // thanks to dependency management, but the version should be left + // untouched + ivy"com.lihaoyi:cask_2.13:0.9.3" + .exclude(("org.slf4j", "slf4j-api")) + ) + + object transitive extends JavaModule with TestPublishModule { + def moduleDeps = Seq(extraExclude) + } + } + + object exclude extends JavaModule with TestPublishModule { + def ivyDeps = Agg( + ivy"com.lihaoyi:cask_2.13:0.9.4" + ) + def dependencyManagement = Agg( + ivy"org.java-websocket:Java-WebSocket:1.5.2" + .exclude(("org.slf4j", "slf4j-api")) + ) + + object transitive extends JavaModule with TestPublishModule { + def moduleDeps = Seq(exclude) + } + } + + object onlyExclude extends JavaModule with TestPublishModule { + def ivyDeps = Agg( + ivy"com.lihaoyi:cask_2.13:0.9.4" + ) + def dependencyManagement = Agg( + ivy"org.java-websocket:Java-WebSocket" + .exclude(("org.slf4j", "slf4j-api")) + ) + + object transitive extends JavaModule with TestPublishModule { + def moduleDeps = Seq(onlyExclude) + } + } + + object invalid extends TestBaseModule { + object transitive extends JavaModule { + def dependencyManagement = { + val dep = ivy"org.java-websocket:Java-WebSocket:1.5.3" + Agg( + dep.copy( + dep = dep.dep.withTransitive(false) + ) + ) + } + } + } + + object placeholder extends JavaModule with TestPublishModule { + def ivyDeps = Agg( + ivy"com.google.protobuf:protobuf-java" + ) + def dependencyManagement = Agg( + ivy"com.google.protobuf:protobuf-java:4.28.3" + ) + + object transitive extends JavaModule with TestPublishModule { + def moduleDeps = Seq(placeholder) + } + } + } + + object bomOnModuleDependency extends JavaModule with TestPublishModule { + def ivyDeps = Agg( + ivy"com.google.protobuf:protobuf-java:3.23.4" + ) + + object dependee extends JavaModule with TestPublishModule { + def bomDeps = Agg( + ivy"com.google.cloud:libraries-bom:26.50.0" + ) + def moduleDeps = Seq(bomOnModuleDependency) + } + } + } + + def expectedProtobufJavaVersion = "4.28.3" + def expectedCommonsCompressVersion = "1.23.0" + + def expectedProtobufJarName = s"protobuf-java-$expectedProtobufJavaVersion.jar" + def expectedCommonsCompressJarName = s"commons-compress-$expectedCommonsCompressVersion.jar" + + def compileClasspathFileNames(module: JavaModule)(implicit + eval: UnitTester + ): Seq[String] = + eval(module.compileClasspath).toTry.get.value + .toSeq.map(_.path.last) + + def compileClasspathContains( + module: JavaModule, + fileName: String, + jarCheck: Option[String => Boolean] + )(implicit + eval: UnitTester + ) = { + val fileNames = compileClasspathFileNames(module) + assert(fileNames.contains(fileName)) + for (check <- jarCheck; fileName <- fileNames) + assert(check(fileName)) + } + + def publishLocalAndResolve( + module: PublishModule, + dependencyModules: Seq[PublishModule], + scalaSuffix: String + )(implicit eval: UnitTester): Seq[os.Path] = { + val localIvyRepo = eval.evaluator.workspace / "ivy2Local" + eval(module.publishLocal(localIvyRepo.toString)).toTry.get + for (dependencyModule <- dependencyModules) + eval(dependencyModule.publishLocal(localIvyRepo.toString)).toTry.get + + val moduleString = eval(module.artifactName).toTry.get.value + + coursierapi.Fetch.create() + .addDependencies( + coursierapi.Dependency.of( + "com.lihaoyi.mill-tests", + moduleString.replace('.', '-') + scalaSuffix, + "0.1.0-SNAPSHOT" + ) + ) + .addRepositories( + coursierapi.IvyRepository.of(localIvyRepo.toNIO.toUri.toASCIIString + "[defaultPattern]") + ) + .fetch() + .asScala + .map(os.Path(_)) + .toVector + } + + def publishM2LocalAndResolve( + module: PublishModule, + dependencyModules: Seq[PublishModule], + scalaSuffix: String + )(implicit eval: UnitTester): Seq[os.Path] = { + val localM2Repo = eval.evaluator.workspace / "m2Local" + eval(module.publishM2Local(localM2Repo.toString)).toTry.get + for (dependencyModule <- dependencyModules) + eval(dependencyModule.publishM2Local(localM2Repo.toString)).toTry.get + + val moduleString = eval(module.artifactName).toTry.get.value + + coursierapi.Fetch.create() + .addDependencies( + coursierapi.Dependency.of( + "com.lihaoyi.mill-tests", + moduleString.replace('.', '-') + scalaSuffix, + "0.1.0-SNAPSHOT" + ) + ) + .addRepositories( + coursierapi.MavenRepository.of(localM2Repo.toNIO.toUri.toASCIIString) + ) + .fetch() + .asScala + .map(os.Path(_)) + .toVector + } + + def isInClassPath( + module: JavaModule with PublishModule, + jarName: String, + dependencyModules: Seq[PublishModule] = Nil, + jarCheck: Option[String => Boolean] = None, + ivy2LocalCheck: Boolean = true, + scalaSuffix: String = "" + )(implicit eval: UnitTester): Unit = { + compileClasspathContains(module, jarName, jarCheck) + + if (ivy2LocalCheck) { + val resolvedCp = publishLocalAndResolve(module, dependencyModules, scalaSuffix) + assert(resolvedCp.map(_.last).contains(jarName)) + for (check <- jarCheck; fileName <- resolvedCp.map(_.last)) + assert(check(fileName)) + } + + val resolvedM2Cp = publishM2LocalAndResolve(module, dependencyModules, scalaSuffix) + assert(resolvedM2Cp.map(_.last).contains(jarName)) + for (check <- jarCheck; fileName <- resolvedM2Cp.map(_.last)) + assert(check(fileName)) + } + + def tests = Tests { + + test("bom") { + test("placeholder") { + test("check") - UnitTester(modules, null).scoped { eval => + val res = eval(modules.bom.placeholder.check.compileClasspath) + assert( + res.left.exists(_.toString.contains( + "not found: https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java/_/protobuf-java-_.pom" + )) + ) + } + + test("simple") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath(modules.bom.placeholder, expectedProtobufJarName) + } + + test("dependee") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.bom.placeholder.dependee, + expectedProtobufJarName, + Seq(modules.bom.placeholder) + ) + } + + test("subDependee") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.bom.placeholder.subDependee, + expectedProtobufJarName, + Seq(modules.bom.placeholder, modules.bom.placeholder.dependee) + ) + } + } + + test("versionOverride") { + test("check") - UnitTester(modules, null).scoped { implicit eval => + val fileNames = compileClasspathFileNames(modules.bom.versionOverride.check) + assert(fileNames.exists(v => v.startsWith("protobuf-java-") && v.endsWith(".jar"))) + assert(!fileNames.contains(expectedProtobufJarName)) + } + + test("simple") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath(modules.bom.versionOverride, expectedProtobufJarName) + } + + test("dependee") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.bom.versionOverride.dependee, + expectedProtobufJarName, + Seq(modules.bom.versionOverride) + ) + } + + test("subDependee") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.bom.versionOverride.subDependee, + expectedProtobufJarName, + Seq(modules.bom.versionOverride, modules.bom.versionOverride.dependee) + ) + } + } + + test("invalid") { + test - UnitTester(modules, null).scoped { eval => + val res = eval(modules.bom.invalid.exclude.compileClasspath) + assert( + res.left.exists(_.toString.contains( + "Found BOM dependencies with invalid parameters:" + )) + ) + } + } + } + + test("depMgmt") { + test("override") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath(modules.depMgmt, expectedProtobufJarName) + } + + test("transitiveOverride") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath(modules.depMgmt.transitive, expectedProtobufJarName, Seq(modules.depMgmt)) + } + + test("extraExclude") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.depMgmt.extraExclude, + "cask_2.13-0.9.4.jar", + jarCheck = Some { jarName => + !jarName.startsWith("slf4j-api-") + } + ) + } + + test("transitiveExtraExclude") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.depMgmt.extraExclude.transitive, + "cask_2.13-0.9.4.jar", + Seq(modules.depMgmt.extraExclude), + jarCheck = Some { jarName => + !jarName.startsWith("slf4j-api-") + }, + ivy2LocalCheck = false // we could make that work + ) + } + + test("exclude") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.depMgmt.exclude, + "Java-WebSocket-1.5.2.jar", + jarCheck = Some { jarName => + !jarName.startsWith("slf4j-api-") + }, + ivy2LocalCheck = false // dep mgmt excludes can't be put in ivy.xml + ) + } + + test("transitiveExclude") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.depMgmt.exclude.transitive, + "Java-WebSocket-1.5.2.jar", + Seq(modules.depMgmt.exclude), + jarCheck = Some { jarName => + !jarName.startsWith("slf4j-api-") + }, + ivy2LocalCheck = false // dep mgmt excludes can't be put in ivy.xml + ) + } + + test("onlyExclude") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.depMgmt.onlyExclude, + "Java-WebSocket-1.5.3.jar", + jarCheck = Some { jarName => + !jarName.startsWith("slf4j-api-") + }, + ivy2LocalCheck = false // dep mgmt excludes can't be put in ivy.xml + ) + } + + test("transitiveOnlyExclude") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.depMgmt.onlyExclude.transitive, + "Java-WebSocket-1.5.3.jar", + Seq(modules.depMgmt.onlyExclude), + jarCheck = Some { jarName => + !jarName.startsWith("slf4j-api-") + }, + ivy2LocalCheck = false // dep mgmt excludes can't be put in ivy.xml + ) + } + + test("invalid") { + test - UnitTester(modules, null).scoped { eval => + val res = eval(modules.depMgmt.invalid.transitive.compileClasspath) + assert( + res.left.exists(_.toString.contains( + "Found dependency management entries with invalid values." + )) + ) + } + } + + test("placeholder") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath(modules.depMgmt.placeholder, expectedProtobufJarName) + } + + test("transitivePlaceholder") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.depMgmt.placeholder.transitive, + expectedProtobufJarName, + Seq(modules.depMgmt.placeholder) + ) + } + } + + test("bomOnModuleDependency") { + test("check") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.bomOnModuleDependency, + "protobuf-java-3.23.4.jar" + ) + } + test("dependee") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.bomOnModuleDependency.dependee, + expectedProtobufJarName, + Seq(modules.bomOnModuleDependency) + ) + } + } + } +} diff --git a/scalalib/test/src/mill/scalalib/publish/PomTests.scala b/scalalib/test/src/mill/scalalib/publish/PomTests.scala index d1b52b602aa..d8fb8d44bbd 100644 --- a/scalalib/test/src/mill/scalalib/publish/PomTests.scala +++ b/scalalib/test/src/mill/scalalib/publish/PomTests.scala @@ -222,7 +222,10 @@ object PomTests extends TestSuite { artifactId, pomSettings, properties, - PackagingType.Jar + PackagingType.Jar, + None, + Agg.empty[Dependency], + Agg.empty[Dependency] )) def singleText(seq: NodeSeq) = From 793be6cf405a8faa5de22f5198663f2f3b1da945 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Mon, 2 Dec 2024 11:59:22 +0100 Subject: [PATCH 02/18] Remove JavaModule#allIvyDeps --- scalalib/src/mill/scalalib/JavaModule.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index a2498b669e1..dd999c3eb0c 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -139,10 +139,10 @@ trait JavaModule */ def ivyDeps: T[Agg[Dep]] = Task { Agg.empty[Dep] } - /** - * Aggregation of mandatoryIvyDeps and ivyDeps. - * In most cases, instead of overriding this Target you want to override `ivyDeps` instead. - */ + @deprecated( + "Use processedIvyDeps or ivyDeps() ++ mandatoryIvyDeps() instead", + "Mill after 0.12.3" + ) def allIvyDeps: T[Agg[Dep]] = Task { ivyDeps() ++ mandatoryIvyDeps() } /** @@ -483,7 +483,7 @@ trait JavaModule */ def processedIvyDeps: Task[Agg[BoundDep]] = Task.Anon { val processDependency0 = processDependency()() - allIvyDeps().map(bindDependency()).map { dep => + (ivyDeps() ++ mandatoryIvyDeps()).map(bindDependency()).map { dep => dep.copy(dep = processDependency0(dep.dep)) } } From e920c4667aa84a1853652d4d6fd43e97a6b51681 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Mon, 2 Dec 2024 13:15:28 +0100 Subject: [PATCH 03/18] Re-organize things --- scalalib/src/mill/scalalib/JavaModule.scala | 68 ++++++++++----------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index dd999c3eb0c..ebaf43ab656 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -317,6 +317,18 @@ trait JavaModule */ def moduleDeps: Seq[JavaModule] = Seq.empty + /** + * The compile-only direct dependencies of this module. These are *not* + * transitive, and only take effect in the module that they are declared in. + */ + def compileModuleDeps: Seq[JavaModule] = Seq.empty + + /** + * The runtime-only direct dependencies of this module. These *are* transitive, + * and so get propagated to downstream modules automatically + */ + def runModuleDeps: Seq[JavaModule] = Seq.empty + /** * Same as [[moduleDeps]] but checked to not contain cycles. * Prefer this over using [[moduleDeps]] directly. @@ -327,6 +339,13 @@ trait JavaModule moduleDeps } + /** Same as [[compileModuleDeps]] but checked to not contain cycles. */ + final def compileModuleDepsChecked: Seq[JavaModule] = { + // trigger initialization to check for cycles + recCompileModuleDeps + compileModuleDeps + } + /** * Same as [[moduleDeps]] but checked to not contain cycles. * Prefer this over using [[moduleDeps]] directly. @@ -345,33 +364,6 @@ trait JavaModule _.moduleDeps ) - /** Should only be called from [[runModuleDepsChecked]] */ - private lazy val recRunModuleDeps: Seq[JavaModule] = - ModuleUtils.recursive[JavaModule]( - (millModuleSegments ++ Seq(Segment.Label("runModuleDeps"))).render, - this, - m => m.runModuleDeps ++ m.moduleDeps - ) - - /** - * The compile-only direct dependencies of this module. These are *not* - * transitive, and only take effect in the module that they are declared in. - */ - def compileModuleDeps: Seq[JavaModule] = Seq.empty - - /** - * The runtime-only direct dependencies of this module. These *are* transitive, - * and so get propagated to downstream modules automatically - */ - def runModuleDeps: Seq[JavaModule] = Seq.empty - - /** Same as [[compileModuleDeps]] but checked to not contain cycles. */ - final def compileModuleDepsChecked: Seq[JavaModule] = { - // trigger initialization to check for cycles - recCompileModuleDeps - compileModuleDeps - } - /** Should only be called from [[compileModuleDeps]] */ private lazy val recCompileModuleDeps: Seq[JavaModule] = ModuleUtils.recursive[JavaModule]( @@ -380,6 +372,14 @@ trait JavaModule _.compileModuleDeps ) + /** Should only be called from [[runModuleDepsChecked]] */ + private lazy val recRunModuleDeps: Seq[JavaModule] = + ModuleUtils.recursive[JavaModule]( + (millModuleSegments ++ Seq(Segment.Label("runModuleDeps"))).render, + this, + m => m.runModuleDeps ++ m.moduleDeps + ) + /** The direct and indirect dependencies of this module */ def recursiveModuleDeps: Seq[JavaModule] = { recModuleDeps } @@ -421,13 +421,6 @@ trait JavaModule (runModuleDepsChecked ++ moduleDepsChecked).flatMap(_.transitiveRunModuleDeps).distinct } - /** The compile-only transitive ivy dependencies of this module and all it's upstream compile-only modules. */ - def transitiveCompileIvyDeps: T[Agg[BoundDep]] = Task { - // We never include compile-only dependencies transitively, but we must include normal transitive dependencies! - compileIvyDeps().map(bindDependency()) ++ - T.traverse(compileModuleDepsChecked)(_.transitiveIvyDeps)().flatten - } - /** * Show the module dependencies. * @param recursive If `true` include all recursive module dependencies, else only show direct dependencies. @@ -500,6 +493,13 @@ trait JavaModule } } + /** The compile-only transitive ivy dependencies of this module and all it's upstream compile-only modules. */ + def transitiveCompileIvyDeps: T[Agg[BoundDep]] = Task { + // We never include compile-only dependencies transitively, but we must include normal transitive dependencies! + compileIvyDeps().map(bindDependency()) ++ + T.traverse(compileModuleDepsChecked)(_.transitiveIvyDeps)().flatten + } + /** * The transitive run ivy dependencies of this module and all it's upstream modules. * This is calculated from [[runIvyDeps]], [[mandatoryIvyDeps]] and recursively from [[moduleDeps]]. From 45fb34387184d607bf404348a643423cfb433d86 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Mon, 2 Dec 2024 11:45:58 +0100 Subject: [PATCH 04/18] Add JavaModule#compileDependencyManagement --- scalalib/src/mill/scalalib/JavaModule.scala | 23 ++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index ebaf43ab656..c5ff1051651 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -209,6 +209,8 @@ trait JavaModule */ def dependencyManagement: T[Agg[Dep]] = Task { Agg.empty[Dep] } + def compileDependencyManagement: T[Agg[Dep]] = Task { Agg.empty[Dep] } + private def addBoms( dep: coursier.core.Dependency, bomDeps: Seq[coursier.core.BomDependency], @@ -469,6 +471,18 @@ trait JavaModule addBoms(dep, bomDeps0, depMgmt, depMgmtMap, overrideVersions = overrideVersions) } + def processCompileDependency: Task[coursier.core.Dependency => coursier.core.Dependency] = + Task.Anon { + val bomDeps0 = allBomDeps().toSeq.map(_.withConfig(Configuration.provided)) + val depMgmt = processedDependencyManagement( + compileDependencyManagement().toSeq.map(bindDependency()).map(_.dep) + ) + val depMgmtMap = depMgmt.toMap + + dep => + addBoms(dep, bomDeps0, depMgmt, depMgmtMap, overrideVersions = false) + } + /** * The Ivy dependencies of this module, with BOM and dependency management details * added to them. This should be used when propagating the dependencies transitively @@ -481,6 +495,13 @@ trait JavaModule } } + def processedCompileIvyDeps: Task[Agg[Dep]] = Task.Anon { + val processDependency0 = processCompileDependency() + compileIvyDeps().map { dep => + dep.copy(dep = processDependency0(dep.dep)) + } + } + /** * The transitive ivy dependencies of this module and all it's upstream modules. * This is calculated from [[ivyDeps]], [[mandatoryIvyDeps]] and recursively from [[moduleDeps]]. @@ -496,7 +517,7 @@ trait JavaModule /** The compile-only transitive ivy dependencies of this module and all it's upstream compile-only modules. */ def transitiveCompileIvyDeps: T[Agg[BoundDep]] = Task { // We never include compile-only dependencies transitively, but we must include normal transitive dependencies! - compileIvyDeps().map(bindDependency()) ++ + processedCompileIvyDeps().map(bindDependency()) ++ T.traverse(compileModuleDepsChecked)(_.transitiveIvyDeps)().flatten } From a5af7ebe5e501387de83b8b387edbb4f82d84a34 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Mon, 2 Dec 2024 17:59:00 +0100 Subject: [PATCH 05/18] Take BOMs into account for tests --- scalalib/src/mill/scalalib/JavaModule.scala | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index c5ff1051651..2edb64f98fc 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -57,6 +57,10 @@ trait JavaModule } } + override def extraBomDeps = Task.Anon[Agg[BomDependency]] { + outer.allBomDeps().map(_.withConfig(Configuration.test)) + } + /** * JavaModule and its derivatives define inner test modules. * To avoid unexpected misbehavior due to the use of the wrong inner test trait @@ -191,6 +195,8 @@ trait JavaModule ) } + def extraBomDeps: Task[Agg[BomDependency]] = Task.Anon { Agg.empty[BomDependency] } + /** * Dependency management data * @@ -461,7 +467,8 @@ trait JavaModule def processDependency( overrideVersions: Boolean = false ): Task[coursier.core.Dependency => coursier.core.Dependency] = Task.Anon { - val bomDeps0 = allBomDeps().toSeq.map(_.withConfig(Configuration.compile)) + val bomDeps0 = + allBomDeps().toSeq.map(_.withConfig(Configuration.compile)) ++ extraBomDeps().toSeq val depMgmt = processedDependencyManagement( dependencyManagement().toSeq.map(bindDependency()).map(_.dep) ) From 3bee9eb0afa6a20fe4a425aa35a0849590dfac92 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Mon, 2 Dec 2024 16:37:40 +0100 Subject: [PATCH 06/18] Add JavaModule#runDependencyManagement --- scalalib/src/mill/scalalib/JavaModule.scala | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 2edb64f98fc..e1e653db748 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -217,6 +217,8 @@ trait JavaModule def compileDependencyManagement: T[Agg[Dep]] = Task { Agg.empty[Dep] } + def runDependencyManagement: T[Agg[Dep]] = Task { Agg.empty[Dep] } + private def addBoms( dep: coursier.core.Dependency, bomDeps: Seq[coursier.core.BomDependency], @@ -490,6 +492,17 @@ trait JavaModule addBoms(dep, bomDeps0, depMgmt, depMgmtMap, overrideVersions = false) } + def processRunDependency: Task[coursier.core.Dependency => coursier.core.Dependency] = Task.Anon { + val bomDeps0 = allBomDeps().toSeq.map(_.withConfig(Configuration.defaultCompile)) + val depMgmt = processedDependencyManagement( + runDependencyManagement().toSeq.map(bindDependency()).map(_.dep) + ) + val depMgmtMap = depMgmt.toMap + + dep => + addBoms(dep, bomDeps0, depMgmt, depMgmtMap, overrideVersions = false) + } + /** * The Ivy dependencies of this module, with BOM and dependency management details * added to them. This should be used when propagating the dependencies transitively @@ -509,6 +522,13 @@ trait JavaModule } } + def processedRunIvyDeps: Task[Agg[Dep]] = Task.Anon { + val processDependency0 = processRunDependency() + runIvyDeps().map { dep => + dep.copy(dep = processDependency0(dep.dep)) + } + } + /** * The transitive ivy dependencies of this module and all it's upstream modules. * This is calculated from [[ivyDeps]], [[mandatoryIvyDeps]] and recursively from [[moduleDeps]]. From 383d065ba2796333e8a96156679c5a3aadd261c8 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Mon, 2 Dec 2024 18:19:46 +0100 Subject: [PATCH 07/18] Apply BOM and dep mgmt to transitive dependencies from module deps --- scalalib/src/mill/scalalib/JavaModule.scala | 50 +++++++++++++-------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index e1e653db748..f1f8cc4ce79 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -480,19 +480,22 @@ trait JavaModule addBoms(dep, bomDeps0, depMgmt, depMgmtMap, overrideVersions = overrideVersions) } - def processCompileDependency: Task[coursier.core.Dependency => coursier.core.Dependency] = - Task.Anon { - val bomDeps0 = allBomDeps().toSeq.map(_.withConfig(Configuration.provided)) - val depMgmt = processedDependencyManagement( - compileDependencyManagement().toSeq.map(bindDependency()).map(_.dep) - ) - val depMgmtMap = depMgmt.toMap + def processCompileDependency( + overrideVersions: Boolean = false + ): Task[coursier.core.Dependency => coursier.core.Dependency] = Task.Anon { + val bomDeps0 = allBomDeps().toSeq.map(_.withConfig(Configuration.provided)) + val depMgmt = processedDependencyManagement( + compileDependencyManagement().toSeq.map(bindDependency()).map(_.dep) + ) + val depMgmtMap = depMgmt.toMap - dep => - addBoms(dep, bomDeps0, depMgmt, depMgmtMap, overrideVersions = false) - } + dep => + addBoms(dep, bomDeps0, depMgmt, depMgmtMap, overrideVersions = overrideVersions) + } - def processRunDependency: Task[coursier.core.Dependency => coursier.core.Dependency] = Task.Anon { + def processRunDependency( + overrideVersions: Boolean = false + ): Task[coursier.core.Dependency => coursier.core.Dependency] = Task.Anon { val bomDeps0 = allBomDeps().toSeq.map(_.withConfig(Configuration.defaultCompile)) val depMgmt = processedDependencyManagement( runDependencyManagement().toSeq.map(bindDependency()).map(_.dep) @@ -500,7 +503,7 @@ trait JavaModule val depMgmtMap = depMgmt.toMap dep => - addBoms(dep, bomDeps0, depMgmt, depMgmtMap, overrideVersions = false) + addBoms(dep, bomDeps0, depMgmt, depMgmtMap, overrideVersions = overrideVersions) } /** @@ -516,14 +519,14 @@ trait JavaModule } def processedCompileIvyDeps: Task[Agg[Dep]] = Task.Anon { - val processDependency0 = processCompileDependency() + val processDependency0 = processCompileDependency()() compileIvyDeps().map { dep => dep.copy(dep = processDependency0(dep.dep)) } } def processedRunIvyDeps: Task[Agg[Dep]] = Task.Anon { - val processDependency0 = processRunDependency() + val processDependency0 = processRunDependency()() runIvyDeps().map { dep => dep.copy(dep = processDependency0(dep.dep)) } @@ -543,9 +546,12 @@ trait JavaModule /** The compile-only transitive ivy dependencies of this module and all it's upstream compile-only modules. */ def transitiveCompileIvyDeps: T[Agg[BoundDep]] = Task { + val processDependency0 = processCompileDependency(overrideVersions = true)() // We never include compile-only dependencies transitively, but we must include normal transitive dependencies! processedCompileIvyDeps().map(bindDependency()) ++ - T.traverse(compileModuleDepsChecked)(_.transitiveIvyDeps)().flatten + T.traverse(compileModuleDepsChecked)(_.transitiveIvyDeps)().flatten.map { dep => + dep.copy(dep = processDependency0(dep.dep)) + } } /** @@ -553,10 +559,16 @@ trait JavaModule * This is calculated from [[runIvyDeps]], [[mandatoryIvyDeps]] and recursively from [[moduleDeps]]. */ def transitiveRunIvyDeps: T[Agg[BoundDep]] = Task { - runIvyDeps().map(bindDependency()) ++ - T.traverse(moduleDepsChecked)(_.transitiveRunIvyDeps)().flatten ++ - T.traverse(runModuleDepsChecked)(_.transitiveIvyDeps)().flatten ++ - T.traverse(runModuleDepsChecked)(_.transitiveRunIvyDeps)().flatten + val processDependency0 = processRunDependency(overrideVersions = true)() + runIvyDeps().map(bindDependency()) ++ { + val viaModules = + T.traverse(moduleDepsChecked)(_.transitiveRunIvyDeps)().flatten ++ + T.traverse(runModuleDepsChecked)(_.transitiveIvyDeps)().flatten ++ + T.traverse(runModuleDepsChecked)(_.transitiveRunIvyDeps)().flatten + viaModules.map { dep => + dep.copy(dep = processDependency0(dep.dep)) + } + } } /** From b8e50e16b7acd9f107d13924a66c5bd2e52911b2 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Mon, 2 Dec 2024 16:40:10 +0100 Subject: [PATCH 08/18] ivy.xml stuff --- .../src/mill/scalalib/PublishModule.scala | 32 ++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/scalalib/src/mill/scalalib/PublishModule.scala b/scalalib/src/mill/scalalib/PublishModule.scala index 75c4eb93c33..acad9c736ab 100644 --- a/scalalib/src/mill/scalalib/PublishModule.scala +++ b/scalalib/src/mill/scalalib/PublishModule.scala @@ -162,9 +162,28 @@ trait PublishModule extends JavaModule { outer => else dep } + def rootDepsAdjustment = publishXmlDeps0.iterator.flatMap { dep => + val key = coursier.core.DependencyManagement.Key( + coursier.core.Organization(dep.artifact.group), + coursier.core.ModuleName(dep.artifact.id), + coursier.core.Type.jar, + coursier.core.Classifier.empty + ) + bomDepMgmt.get(key).flatMap { values => + if (values.version.nonEmpty && values.version != dep.artifact.version) + Some(key -> values.withVersion("")) + else + None + } + } + val bomDepMgmt0 = bomDepMgmt ++ rootDepsAdjustment + lazy val moduleSet = publishXmlDeps0.map(dep => (dep.artifact.group, dep.artifact.id)).toSet val overrides = dependencyManagement().toSeq.map(bindDependency()).map(_.dep) .filter(depMgmt => depMgmt.version.nonEmpty && depMgmt.version != "_") + .filter { depMgmt => + !moduleSet.contains((depMgmt.module.organization.value, depMgmt.module.name.value)) + } .map { depMgmt => Ivy.Override( depMgmt.module.organization.value, @@ -172,10 +191,15 @@ trait PublishModule extends JavaModule { outer => depMgmt.version ) } ++ - bomDepMgmt.map { - case (key, values) => - Ivy.Override(key.organization.value, key.name.value, values.version) - } + bomDepMgmt0 + .filter { + case (key, _) => + !moduleSet.contains((key.organization.value, key.name.value)) + } + .map { + case (key, values) => + Ivy.Override(key.organization.value, key.name.value, values.version) + } val ivy = Ivy(artifactMetadata(), publishXmlDeps0, extraPublish(), overrides) val ivyPath = T.dest / "ivy.xml" os.write.over(ivyPath, ivy) From 1570f37264edea6e8ae3fd2338272f0f5fa7070f Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Mon, 2 Dec 2024 17:01:02 +0100 Subject: [PATCH 09/18] Add BOM scope tests --- .../test/src/mill/scalalib/BomTests.scala | 58 +++++++++++++++++-- 1 file changed, 54 insertions(+), 4 deletions(-) diff --git a/scalalib/test/src/mill/scalalib/BomTests.scala b/scalalib/test/src/mill/scalalib/BomTests.scala index db3ce96b356..f367d5b2fd9 100644 --- a/scalalib/test/src/mill/scalalib/BomTests.scala +++ b/scalalib/test/src/mill/scalalib/BomTests.scala @@ -170,6 +170,26 @@ object BomTests extends TestSuite { } } + object bomScope extends JavaModule with TestPublishModule { + def bomDeps = Agg( + ivy"org.apache.spark:spark-parent_2.13:3.5.3" + ) + def compileIvyDeps = Agg( + ivy"com.google.protobuf:protobuf-java-util", + ivy"org.scala-lang.modules:scala-parallel-collections_2.13" + ) + + object fail extends JavaModule with TestPublishModule { + def bomDeps = Agg( + ivy"org.apache.spark:spark-parent_2.13:3.5.3" + ) + def ivyDeps = Agg( + ivy"com.google.protobuf:protobuf-java-util", + ivy"org.scala-lang.modules:scala-parallel-collections_2.13" + ) + } + } + object bomOnModuleDependency extends JavaModule with TestPublishModule { def ivyDeps = Agg( ivy"com.google.protobuf:protobuf-java:3.23.4" @@ -273,6 +293,7 @@ object BomTests extends TestSuite { dependencyModules: Seq[PublishModule] = Nil, jarCheck: Option[String => Boolean] = None, ivy2LocalCheck: Boolean = true, + m2LocalCheck: Boolean = true, scalaSuffix: String = "" )(implicit eval: UnitTester): Unit = { compileClasspathContains(module, jarName, jarCheck) @@ -284,10 +305,12 @@ object BomTests extends TestSuite { assert(check(fileName)) } - val resolvedM2Cp = publishM2LocalAndResolve(module, dependencyModules, scalaSuffix) - assert(resolvedM2Cp.map(_.last).contains(jarName)) - for (check <- jarCheck; fileName <- resolvedM2Cp.map(_.last)) - assert(check(fileName)) + if (m2LocalCheck) { + val resolvedM2Cp = publishM2LocalAndResolve(module, dependencyModules, scalaSuffix) + assert(resolvedM2Cp.map(_.last).contains(jarName)) + for (check <- jarCheck; fileName <- resolvedM2Cp.map(_.last)) + assert(check(fileName)) + } } def tests = Tests { @@ -465,6 +488,33 @@ object BomTests extends TestSuite { } } + test("bomScope") { + test("provided") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.bomScope, + "protobuf-java-3.23.4.jar", + ivy2LocalCheck = false, + m2LocalCheck = false + ) + } + test("providedFromBomRuntimeScope") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.bomScope, + "scala-parallel-collections_2.13-1.0.4.jar", + ivy2LocalCheck = false, + m2LocalCheck = false + ) + } + test("ignoreProvidedForCompile") - UnitTester(modules, null).scoped { implicit eval => + val res = eval(modules.bomScope.fail.resolvedIvyDeps) + assert( + res.left.exists(_.toString.contains( + "not found: https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java-util/_/protobuf-java-util-_.pom" + )) + ) + } + } + test("bomOnModuleDependency") { test("check") - UnitTester(modules, null).scoped { implicit eval => isInClassPath( From 93e1d5219cefe18fb1daef87f0110f2c9bb3804a Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Thu, 5 Dec 2024 15:41:28 +0100 Subject: [PATCH 10/18] more --- scalalib/src/mill/scalalib/JavaModule.scala | 30 ++++++++----------- .../src/mill/scalalib/PublishModule.scala | 2 +- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 03ffb26d9a7..023df834edb 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -143,10 +143,10 @@ trait JavaModule */ def ivyDeps: T[Agg[Dep]] = Task { Agg.empty[Dep] } - @deprecated( - "Use processedIvyDeps or ivyDeps() ++ mandatoryIvyDeps() instead", - "Mill after 0.12.3" - ) + /** + * Aggregation of mandatoryIvyDeps and ivyDeps. + * In most cases, instead of overriding this Target you want to override `ivyDeps` instead. + */ def allIvyDeps: T[Agg[Dep]] = Task { ivyDeps() ++ mandatoryIvyDeps() } /** @@ -207,7 +207,7 @@ trait JavaModule * For example, the following forces com.lihaoyi::os-lib to version 0.11.3, and * excludes org.slf4j:slf4j-api from com.lihaoyi::cask that it forces to version 0.9.4 * {{{ - * def dependencyManagement = super.dependencyManagement() ++ Agg( + * def depManagement = super.depManagement() ++ Agg( * ivy"com.lihaoyi::os-lib:0.11.3", * ivy"com.lihaoyi::cask:0.9.4".exclude("org.slf4j", "slf4j-api") * ) @@ -472,7 +472,7 @@ trait JavaModule val bomDeps0 = allBomDeps().toSeq.map(_.withConfig(Configuration.compile)) ++ extraBomDeps().toSeq val depMgmt = processedDependencyManagement( - dependencyManagement().toSeq.map(bindDependency()).map(_.dep) + depManagement().toSeq.map(bindDependency()).map(_.dep) ) addBoms(bomDeps0, depMgmt, overrideVersions = overrideVersions) @@ -485,10 +485,8 @@ trait JavaModule val depMgmt = processedDependencyManagement( compileDependencyManagement().toSeq.map(bindDependency()).map(_.dep) ) - val depMgmtMap = depMgmt.toMap - dep => - addBoms(dep, bomDeps0, depMgmt, depMgmtMap, overrideVersions = overrideVersions) + addBoms(bomDeps0, depMgmt, overrideVersions = overrideVersions) } def processRunDependency( @@ -498,10 +496,8 @@ trait JavaModule val depMgmt = processedDependencyManagement( runDependencyManagement().toSeq.map(bindDependency()).map(_.dep) ) - val depMgmtMap = depMgmt.toMap - dep => - addBoms(dep, bomDeps0, depMgmt, depMgmtMap, overrideVersions = overrideVersions) + addBoms(bomDeps0, depMgmt, overrideVersions = overrideVersions) } /** @@ -516,16 +512,16 @@ trait JavaModule } } - def processedCompileIvyDeps: Task[Agg[Dep]] = Task.Anon { + def processedCompileIvyDeps: Task[Agg[BoundDep]] = Task.Anon { val processDependency0 = processCompileDependency()() - compileIvyDeps().map { dep => + compileIvyDeps().map(bindDependency()).map { dep => dep.copy(dep = processDependency0(dep.dep)) } } - def processedRunIvyDeps: Task[Agg[Dep]] = Task.Anon { + def processedRunIvyDeps: Task[Agg[BoundDep]] = Task.Anon { val processDependency0 = processRunDependency()() - runIvyDeps().map { dep => + runIvyDeps().map(bindDependency()).map { dep => dep.copy(dep = processDependency0(dep.dep)) } } @@ -546,7 +542,7 @@ trait JavaModule def transitiveCompileIvyDeps: T[Agg[BoundDep]] = Task { val processDependency0 = processCompileDependency(overrideVersions = true)() // We never include compile-only dependencies transitively, but we must include normal transitive dependencies! - processedCompileIvyDeps().map(bindDependency()) ++ + processedCompileIvyDeps() ++ T.traverse(compileModuleDepsChecked)(_.transitiveIvyDeps)().flatten.map { dep => dep.copy(dep = processDependency0(dep.dep)) } diff --git a/scalalib/src/mill/scalalib/PublishModule.scala b/scalalib/src/mill/scalalib/PublishModule.scala index a3967bd256d..181c3049dc8 100644 --- a/scalalib/src/mill/scalalib/PublishModule.scala +++ b/scalalib/src/mill/scalalib/PublishModule.scala @@ -191,7 +191,7 @@ trait PublishModule extends JavaModule { outer => depMgmt.version ) } ++ - bomDepMgmt0 + bomDepMgmt .filter { case (key, _) => !moduleSet.contains((key.organization.value, key.name.value)) From 140b7877edd70b2bc3798dced9fbe1fb6ccd6db7 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Fri, 6 Dec 2024 14:31:04 +0100 Subject: [PATCH 11/18] Tweaking / more tests --- scalalib/src/mill/scalalib/JavaModule.scala | 10 +- .../src/mill/scalalib/PublishModule.scala | 13 +- scalalib/src/mill/scalalib/publish/Ivy.scala | 8 +- .../test/src/mill/scalalib/BomTests.scala | 191 ++++++++++++++---- 4 files changed, 168 insertions(+), 54 deletions(-) diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 1809b90bbbd..6dacc893770 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -215,9 +215,9 @@ trait JavaModule */ def depManagement: T[Agg[Dep]] = Task { Agg.empty[Dep] } - def compileDependencyManagement: T[Agg[Dep]] = Task { Agg.empty[Dep] } + def compileDepManagement: T[Agg[Dep]] = Task { Agg.empty[Dep] } - def runDependencyManagement: T[Agg[Dep]] = Task { Agg.empty[Dep] } + def runDepManagement: T[Agg[Dep]] = Task { Agg.empty[Dep] } private def addBoms( bomDeps: Seq[coursier.core.BomDependency], @@ -483,7 +483,7 @@ trait JavaModule ): Task[coursier.core.Dependency => coursier.core.Dependency] = Task.Anon { val bomDeps0 = allBomDeps().toSeq.map(_.withConfig(Configuration.provided)) val depMgmt = processedDependencyManagement( - compileDependencyManagement().toSeq.map(bindDependency()).map(_.dep) + compileDepManagement().toSeq.map(bindDependency()).map(_.dep) ) addBoms(bomDeps0, depMgmt, overrideVersions = overrideVersions) @@ -494,7 +494,7 @@ trait JavaModule ): Task[coursier.core.Dependency => coursier.core.Dependency] = Task.Anon { val bomDeps0 = allBomDeps().toSeq.map(_.withConfig(Configuration.defaultCompile)) val depMgmt = processedDependencyManagement( - runDependencyManagement().toSeq.map(bindDependency()).map(_.dep) + runDepManagement().toSeq.map(bindDependency()).map(_.dep) ) addBoms(bomDeps0, depMgmt, overrideVersions = overrideVersions) @@ -554,7 +554,7 @@ trait JavaModule */ def transitiveRunIvyDeps: T[Agg[BoundDep]] = Task { val processDependency0 = processRunDependency(overrideVersions = true)() - runIvyDeps().map(bindDependency()) ++ { + processedRunIvyDeps() ++ { val viaModules = T.traverse(moduleDepsChecked)(_.transitiveRunIvyDeps)().flatten ++ T.traverse(runModuleDepsChecked)(_.transitiveIvyDeps)().flatten ++ diff --git a/scalalib/src/mill/scalalib/PublishModule.scala b/scalalib/src/mill/scalalib/PublishModule.scala index 46308bef30b..c0ecbf1bcbd 100644 --- a/scalalib/src/mill/scalalib/PublishModule.scala +++ b/scalalib/src/mill/scalalib/PublishModule.scala @@ -137,9 +137,10 @@ trait PublishModule extends JavaModule { outer => def bomDetails: T[(Map[coursier.core.Module, String], coursier.core.DependencyManagement.Map)] = Task { val (processedDeps, depMgmt) = defaultResolver().processDeps( - processedIvyDeps(), + processedRunIvyDeps() ++ processedIvyDeps(), resolutionParams = resolutionParams(), - boms = allBomDeps().toSeq.map(_.withConfig(Configuration.compile)) + boms = + allBomDeps().toSeq.map(_.withConfig(Configuration.defaultCompile)) ++ extraBomDeps().toSeq ) (processedDeps.map(_.moduleVersion).toMap, depMgmt) } @@ -191,7 +192,7 @@ trait PublishModule extends JavaModule { outer => ) val entries = coursier.core.DependencyManagement.add( Map.empty, - depMgmtEntries ++ bomDepMgmt + depMgmtEntries ++ bomDepMgmt0 .filter { case (key, _) => !moduleSet.contains((key.organization.value, key.name.value)) @@ -200,10 +201,14 @@ trait PublishModule extends JavaModule { outer => entries.toVector .map { case (key, values) => + val confString = + if (values.config == Configuration.runtime) "runtime->runtime" + else "" Ivy.Override( key.organization.value, key.name.value, - values.version + values.version, + confString ) } .sortBy(value => (value.organization, value.name, value.version)) diff --git a/scalalib/src/mill/scalalib/publish/Ivy.scala b/scalalib/src/mill/scalalib/publish/Ivy.scala index 29bf840694f..20867337197 100644 --- a/scalalib/src/mill/scalalib/publish/Ivy.scala +++ b/scalalib/src/mill/scalalib/publish/Ivy.scala @@ -8,7 +8,7 @@ object Ivy { val head = "\n" - case class Override(organization: String, name: String, version: String) + case class Override(organization: String, name: String, version: String, confString: String) def apply( artifact: Artifact, @@ -89,7 +89,11 @@ object Ivy { } private def renderOverride(override0: Override): Elem = - + if (override0.confString.isEmpty) + + else + private def depIvyConf(d: Dependency): String = { def target(value: String) = d.configuration.getOrElse(value) diff --git a/scalalib/test/src/mill/scalalib/BomTests.scala b/scalalib/test/src/mill/scalalib/BomTests.scala index 7162b731390..0427e3f571b 100644 --- a/scalalib/test/src/mill/scalalib/BomTests.scala +++ b/scalalib/test/src/mill/scalalib/BomTests.scala @@ -77,7 +77,7 @@ object BomTests extends TestSuite { } } - object invalid extends TestBaseModule { + object invalid extends Module { object exclude extends JavaModule { def bomDeps = Agg( ivy"com.google.cloud:libraries-bom:26.50.0".exclude(("foo", "thing")) @@ -143,7 +143,7 @@ object BomTests extends TestSuite { } } - object invalid extends TestBaseModule { + object invalid extends Module { object transitive extends JavaModule { def depManagement = { val dep = ivy"org.java-websocket:Java-WebSocket:1.5.3" @@ -272,23 +272,79 @@ object BomTests extends TestSuite { } } - object bomScope extends JavaModule with TestPublishModule { - def bomDeps = Agg( - ivy"org.apache.spark:spark-parent_2.13:3.5.3" - ) - def compileIvyDeps = Agg( - ivy"com.google.protobuf:protobuf-java-util", - ivy"org.scala-lang.modules:scala-parallel-collections_2.13" - ) - - object fail extends JavaModule with TestPublishModule { + object bomScope extends Module { + object provided extends JavaModule with TestPublishModule { + // This BOM has a versions for protobuf-java-util marked as provided, + // and one for scala-parallel-collections_2.13 in the default scope. + // Both should be taken into account here. def bomDeps = Agg( ivy"org.apache.spark:spark-parent_2.13:3.5.3" ) - def ivyDeps = Agg( + def compileIvyDeps = Agg( ivy"com.google.protobuf:protobuf-java-util", ivy"org.scala-lang.modules:scala-parallel-collections_2.13" ) + + object fail extends JavaModule with TestPublishModule { + // Same as above, except the dependencies are in the + // default scope for us here, so the protobuf-java-util version + // shouldn't be read, as it's in provided scope in the BOM. + def bomDeps = Agg( + ivy"org.apache.spark:spark-parent_2.13:3.5.3" + ) + def ivyDeps = Agg( + ivy"com.google.protobuf:protobuf-java-util", + ivy"org.scala-lang.modules:scala-parallel-collections_2.13" + ) + } + } + + object runtimeScope extends JavaModule with TestPublishModule { + // BOM has a version for org.mvnpm.at.hpcc-js:wasm marked as runtime. + // This version should be taken into account in runtime deps here. + def bomDeps = Agg( + ivy"io.quarkus:quarkus-bom:3.15.1" + ) + def runIvyDeps = Agg( + ivy"org.mvnpm.at.hpcc-js:wasm" + ) + } + + object runtimeScopeFail extends JavaModule with TestPublishModule { + // BOM has a version for org.mvnpm.at.hpcc-js:wasm marked as runtime. + // This version shouldn't be taken into account in main deps here. + def bomDeps = Agg( + ivy"io.quarkus:quarkus-bom:3.15.1" + ) + def ivyDeps = Agg( + ivy"org.mvnpm.at.hpcc-js:wasm" + ) + } + + object testScope extends JavaModule with TestPublishModule { + // BOM has a version for scalatest_2.13 marked as test scope. + // This version should be taken into account in test modules here. + def bomDeps = Agg( + ivy"org.apache.spark:spark-parent_2.13:3.5.3" + ) + object test extends JavaTests { + def testFramework = "com.novocode.junit.JUnitFramework" + def ivyDeps = Agg( + ivy"com.novocode:junit-interface:0.11", + ivy"org.scalatest:scalatest_2.13" + ) + } + } + + object testScopeFail extends JavaModule with TestPublishModule { + // BOM has a version for scalatest_2.13 marked as test scope. + // This version shouldn't be taken into account in main module here. + def bomDeps = Agg( + ivy"org.apache.spark:spark-parent_2.13:3.5.3" + ) + def ivyDeps = Agg( + ivy"org.scalatest:scalatest_2.13" + ) } } @@ -321,7 +377,7 @@ object BomTests extends TestSuite { def compileClasspathContains( module: JavaModule, fileName: String, - jarCheck: Option[String => Boolean] + jarCheck: Option[String => Boolean] = None )(implicit eval: UnitTester ) = { @@ -331,10 +387,30 @@ object BomTests extends TestSuite { assert(check(fileName)) } + def runtimeClasspathFileNames(module: JavaModule)(implicit + eval: UnitTester + ): Seq[String] = + eval(module.runClasspath).toTry.get.value + .toSeq.map(_.path.last) + + def runtimeClasspathContains( + module: JavaModule, + fileName: String, + jarCheck: Option[String => Boolean] = None + )(implicit + eval: UnitTester + ) = { + val fileNames = runtimeClasspathFileNames(module) + assert(fileNames.contains(fileName)) + for (check <- jarCheck; fileName <- fileNames) + assert(check(fileName)) + } + def publishLocalAndResolve( module: PublishModule, dependencyModules: Seq[PublishModule], - scalaSuffix: String + scalaSuffix: String, + fetchRuntime: Boolean )(implicit eval: UnitTester): Seq[os.Path] = { val localIvyRepo = eval.evaluator.workspace / "ivy2Local" eval(module.publishLocal(localIvyRepo.toString)).toTry.get @@ -343,17 +419,23 @@ object BomTests extends TestSuite { val moduleString = eval(module.artifactName).toTry.get.value + val dep = coursierapi.Dependency.of( + "com.lihaoyi.mill-tests", + moduleString.replace('.', '-') + scalaSuffix, + "0.1.0-SNAPSHOT" + ) coursierapi.Fetch.create() - .addDependencies( - coursierapi.Dependency.of( - "com.lihaoyi.mill-tests", - moduleString.replace('.', '-') + scalaSuffix, - "0.1.0-SNAPSHOT" - ) - ) + .addDependencies(dep) .addRepositories( coursierapi.IvyRepository.of(localIvyRepo.toNIO.toUri.toASCIIString + "[defaultPattern]") ) + .withResolutionParams { + val defaultParams = coursierapi.ResolutionParams.create() + defaultParams.withDefaultConfiguration( + if (fetchRuntime) "runtime" + else defaultParams.getDefaultConfiguration + ) + } .fetch() .asScala .map(os.Path(_)) @@ -395,24 +477,26 @@ object BomTests extends TestSuite { dependencyModules: Seq[PublishModule] = Nil, jarCheck: Option[String => Boolean] = None, ivy2LocalCheck: Boolean = true, - m2LocalCheck: Boolean = true, - scalaSuffix: String = "" + scalaSuffix: String = "", + runtimeOnly: Boolean = false )(implicit eval: UnitTester): Unit = { - compileClasspathContains(module, jarName, jarCheck) + if (runtimeOnly) + runtimeClasspathContains(module, jarName, jarCheck) + else + compileClasspathContains(module, jarName, jarCheck) if (ivy2LocalCheck) { - val resolvedCp = publishLocalAndResolve(module, dependencyModules, scalaSuffix) + val resolvedCp = + publishLocalAndResolve(module, dependencyModules, scalaSuffix, fetchRuntime = runtimeOnly) assert(resolvedCp.map(_.last).contains(jarName)) for (check <- jarCheck; fileName <- resolvedCp.map(_.last)) assert(check(fileName)) } - if (m2LocalCheck) { - val resolvedM2Cp = publishM2LocalAndResolve(module, dependencyModules, scalaSuffix) - assert(resolvedM2Cp.map(_.last).contains(jarName)) - for (check <- jarCheck; fileName <- resolvedM2Cp.map(_.last)) - assert(check(fileName)) - } + val resolvedM2Cp = publishM2LocalAndResolve(module, dependencyModules, scalaSuffix) + assert(resolvedM2Cp.map(_.last).contains(jarName)) + for (check <- jarCheck; fileName <- resolvedM2Cp.map(_.last)) + assert(check(fileName)) } def tests = Tests { @@ -667,29 +751,50 @@ object BomTests extends TestSuite { test("bomScope") { test("provided") - UnitTester(modules, null).scoped { implicit eval => - isInClassPath( - modules.bomScope, - "protobuf-java-3.23.4.jar", - ivy2LocalCheck = false, - m2LocalCheck = false + // test about provided scope, nothing to see in published stuff + compileClasspathContains( + modules.bomScope.provided, + "protobuf-java-3.23.4.jar" ) } test("providedFromBomRuntimeScope") - UnitTester(modules, null).scoped { implicit eval => - isInClassPath( - modules.bomScope, - "scala-parallel-collections_2.13-1.0.4.jar", - ivy2LocalCheck = false, - m2LocalCheck = false + // test about provided scope, nothing to see in published stuff + compileClasspathContains( + modules.bomScope.provided, + "scala-parallel-collections_2.13-1.0.4.jar" ) } test("ignoreProvidedForCompile") - UnitTester(modules, null).scoped { implicit eval => - val res = eval(modules.bomScope.fail.resolvedIvyDeps) + val res = eval(modules.bomScope.provided.fail.resolvedIvyDeps) assert( res.left.exists(_.toString.contains( "not found: https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java-util/_/protobuf-java-util-_.pom" )) ) } + + test("test") - UnitTester(modules, null).scoped { implicit eval => + compileClasspathContains( + modules.bomScope.testScope.test, + "scalatest_2.13-3.2.16.jar" + ) + } + test("testCheck") - UnitTester(modules, null).scoped { eval => + val res = eval(modules.bomScope.testScopeFail.compileClasspath) + assert( + res.left.exists(_.toString.contains( + "not found: https://repo1.maven.org/maven2/org/scalatest/scalatest_2.13/_/scalatest_2.13-_.pom" + )) + ) + } + + test("runtime") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.bomScope.runtimeScope, + "wasm-2.15.3.jar", + runtimeOnly = true + ) + } } test("bomOnModuleDependency") { From 578587481388bcb4b7ce7d77e38402bcfe7b4189 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Fri, 6 Dec 2024 15:52:01 +0100 Subject: [PATCH 12/18] some more --- scalalib/src/mill/scalalib/JavaModule.scala | 46 ++++++- .../src/mill/scalalib/PublishModule.scala | 11 +- .../test/src/mill/scalalib/BomTests.scala | 122 +++++++++++++++++- 3 files changed, 167 insertions(+), 12 deletions(-) diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 6dacc893770..d93f3ef2929 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -58,7 +58,12 @@ trait JavaModule } override def extraBomDeps = Task.Anon[Agg[BomDependency]] { - outer.allBomDeps().map(_.withConfig(Configuration.test)) + super.extraBomDeps() ++ + outer.allBomDeps().map(_.withConfig(Configuration.test)) + } + override def extraDepManagement = Task.Anon[Agg[Dep]] { + super.extraDepManagement() ++ + outer.depManagement() } /** @@ -195,8 +200,23 @@ trait JavaModule ) } + /** + * BOM dependencies that are not meant to be overridden or changed by users. + * + * This is mainly used to add BOM dependencies of the main module to its test + * modules, while ensuring test dependencies of the BOM are taken into account too + * in the test module. + */ def extraBomDeps: Task[Agg[BomDependency]] = Task.Anon { Agg.empty[BomDependency] } + /** + * Dependency management entries that are not meant to be overridden or changed by users. + * + * This is mainly used to add dependency management entries of the main module to its test + * modules. + */ + def extraDepManagement: Task[Agg[Dep]] = Task.Anon { Agg.empty[Dep] } + /** * Dependency management data * @@ -215,8 +235,20 @@ trait JavaModule */ def depManagement: T[Agg[Dep]] = Task { Agg.empty[Dep] } + /** + * Dependency management data for the compile only or "provided" scope. + * + * Versions and exclusions specified here apply only to dependencies pulled via + * `compileIvyDeps`. + */ def compileDepManagement: T[Agg[Dep]] = Task { Agg.empty[Dep] } + /** + * Dependency management data for the runtime scope. + * + * Versions and exclusions specified here apply only to dependencies pulled via + * `runIvyDeps`. + */ def runDepManagement: T[Agg[Dep]] = Task { Agg.empty[Dep] } private def addBoms( @@ -472,7 +504,9 @@ trait JavaModule val bomDeps0 = allBomDeps().toSeq.map(_.withConfig(Configuration.compile)) ++ extraBomDeps().toSeq val depMgmt = processedDependencyManagement( - depManagement().toSeq.map(bindDependency()).map(_.dep) + (extraDepManagement().toSeq ++ depManagement().toSeq) + .map(bindDependency()) + .map(_.dep) ) addBoms(bomDeps0, depMgmt, overrideVersions = overrideVersions) @@ -483,7 +517,9 @@ trait JavaModule ): Task[coursier.core.Dependency => coursier.core.Dependency] = Task.Anon { val bomDeps0 = allBomDeps().toSeq.map(_.withConfig(Configuration.provided)) val depMgmt = processedDependencyManagement( - compileDepManagement().toSeq.map(bindDependency()).map(_.dep) + (compileDepManagement().toSeq ++ extraDepManagement().toSeq ++ depManagement().toSeq) + .map(bindDependency()) + .map(_.dep) ) addBoms(bomDeps0, depMgmt, overrideVersions = overrideVersions) @@ -494,7 +530,9 @@ trait JavaModule ): Task[coursier.core.Dependency => coursier.core.Dependency] = Task.Anon { val bomDeps0 = allBomDeps().toSeq.map(_.withConfig(Configuration.defaultCompile)) val depMgmt = processedDependencyManagement( - runDepManagement().toSeq.map(bindDependency()).map(_.dep) + (runDepManagement().toSeq ++ depManagement()) + .map(bindDependency()) + .map(_.dep) ) addBoms(bomDeps0, depMgmt, overrideVersions = overrideVersions) diff --git a/scalalib/src/mill/scalalib/PublishModule.scala b/scalalib/src/mill/scalalib/PublishModule.scala index c0ecbf1bcbd..6bf773474cb 100644 --- a/scalalib/src/mill/scalalib/PublishModule.scala +++ b/scalalib/src/mill/scalalib/PublishModule.scala @@ -111,7 +111,14 @@ trait PublishModule extends JavaModule { outer => * Dependency management to specify in the POM */ def publishXmlDepMgmt: Task[Agg[Dependency]] = Task.Anon { - depManagement().map(resolvePublishDependency.apply().apply(_)) + (extraDepManagement() ++ depManagement()) + .map(resolvePublishDependency.apply().apply(_)) ++ + compileDepManagement() + .map(resolvePublishDependency.apply().apply(_)) + .map(_.copy(scope = Scope.Provided)) ++ + runDepManagement() + .map(resolvePublishDependency.apply().apply(_)) + .map(_.copy(scope = Scope.Runtime)) } def pom: T[PathRef] = Task { @@ -182,7 +189,7 @@ trait PublishModule extends JavaModule { outer => lazy val moduleSet = publishXmlDeps0.map(dep => (dep.artifact.group, dep.artifact.id)).toSet val overrides = { val depMgmtEntries = processedDependencyManagement( - depManagement().toSeq + (extraDepManagement().toSeq ++ depManagement().toSeq) .map(bindDependency()) .map(_.dep) .filter(depMgmt => depMgmt.version.nonEmpty && depMgmt.version != "_") diff --git a/scalalib/test/src/mill/scalalib/BomTests.scala b/scalalib/test/src/mill/scalalib/BomTests.scala index 0427e3f571b..bdfe6166b58 100644 --- a/scalalib/test/src/mill/scalalib/BomTests.scala +++ b/scalalib/test/src/mill/scalalib/BomTests.scala @@ -348,6 +348,75 @@ object BomTests extends TestSuite { } } + object depMgmtScope extends Module { + object provided extends JavaModule with TestPublishModule { + // Version in compileDepManagement should be used in compileIvyDeps + def compileDepManagement = Agg( + ivy"com.google.protobuf:protobuf-java-util:3.23.4" + ) + def depManagement = Agg( + ivy"org.scala-lang.modules:scala-parallel-collections_2.13:1.0.4" + ) + def compileIvyDeps = Agg( + ivy"com.google.protobuf:protobuf-java-util", + ivy"org.scala-lang.modules:scala-parallel-collections_2.13" + ) + + object fail extends JavaModule with TestPublishModule { + // Same as above, except the dependencies are in the + // default scope for us here, so the protobuf-java-util version + // shouldn't be read, as it's in provided scope in the BOM. + def compileDepManagement = Agg( + ivy"com.google.protobuf:protobuf-java-util:3.23.4" + ) + def depManagement = Agg( + ivy"org.scala-lang.modules:scala-parallel-collections_2.13:1.0.4" + ) + def ivyDeps = Agg( + ivy"com.google.protobuf:protobuf-java-util", + ivy"org.scala-lang.modules:scala-parallel-collections_2.13" + ) + } + } + + object runtimeScope extends JavaModule with TestPublishModule { + // Dep mgmt has a version for org.mvnpm.at.hpcc-js:wasm marked as runtime. + // This version should be taken into account in runtime deps here. + def runDepManagement = Agg( + ivy"org.mvnpm.at.hpcc-js:wasm:2.15.3" + ) + def runIvyDeps = Agg( + ivy"org.mvnpm.at.hpcc-js:wasm" + ) + } + + object runtimeScopeFail extends JavaModule with TestPublishModule { + // Dep mgmt has a version for org.mvnpm.at.hpcc-js:wasm marked as runtime. + // This version shouldn't be taken into account in main deps here. + def runDepManagement = Agg( + ivy"org.mvnpm.at.hpcc-js:wasm:2.15.3" + ) + def ivyDeps = Agg( + ivy"org.mvnpm.at.hpcc-js:wasm" + ) + } + + object testScope extends JavaModule with TestPublishModule { + // Dep mgmt in main module has a version for scalatest_2.13. + // This version should be taken into account in test modules here. + def depManagement = Agg( + ivy"org.scalatest:scalatest_2.13:3.2.16" + ) + object test extends JavaTests { + def testFramework = "com.novocode.junit.JUnitFramework" + def ivyDeps = Agg( + ivy"com.novocode:junit-interface:0.11", + ivy"org.scalatest:scalatest_2.13" + ) + } + } + } + object bomOnModuleDependency extends JavaModule with TestPublishModule { def ivyDeps = Agg( ivy"com.google.protobuf:protobuf-java:3.23.4" @@ -419,13 +488,14 @@ object BomTests extends TestSuite { val moduleString = eval(module.artifactName).toTry.get.value - val dep = coursierapi.Dependency.of( - "com.lihaoyi.mill-tests", - moduleString.replace('.', '-') + scalaSuffix, - "0.1.0-SNAPSHOT" - ) coursierapi.Fetch.create() - .addDependencies(dep) + .addDependencies( + coursierapi.Dependency.of( + "com.lihaoyi.mill-tests", + moduleString.replace('.', '-') + scalaSuffix, + "0.1.0-SNAPSHOT" + ) + ) .addRepositories( coursierapi.IvyRepository.of(localIvyRepo.toNIO.toUri.toASCIIString + "[defaultPattern]") ) @@ -812,5 +882,45 @@ object BomTests extends TestSuite { ) } } + + test("depMgmtScope") { + test("provided") - UnitTester(modules, null).scoped { implicit eval => + // test about provided scope, nothing to see in published stuff + compileClasspathContains( + modules.depMgmtScope.provided, + "protobuf-java-3.23.4.jar" + ) + } + test("providedFromBomRuntimeScope") - UnitTester(modules, null).scoped { implicit eval => + // test about provided scope, nothing to see in published stuff + compileClasspathContains( + modules.depMgmtScope.provided, + "scala-parallel-collections_2.13-1.0.4.jar" + ) + } + test("ignoreProvidedForCompile") - UnitTester(modules, null).scoped { implicit eval => + val res = eval(modules.depMgmtScope.provided.fail.resolvedIvyDeps) + assert( + res.left.exists(_.toString.contains( + "not found: https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java-util/_/protobuf-java-util-_.pom" + )) + ) + } + + test("test") - UnitTester(modules, null).scoped { implicit eval => + compileClasspathContains( + modules.depMgmtScope.testScope.test, + "scalatest_2.13-3.2.16.jar" + ) + } + + test("runtime") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.depMgmtScope.runtimeScope, + "wasm-2.15.3.jar", + runtimeOnly = true + ) + } + } } } From a34b2eeff378d382acced1fba4103ebbcda02a09 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Fri, 6 Dec 2024 16:07:43 +0100 Subject: [PATCH 13/18] Add comments --- scalalib/src/mill/scalalib/JavaModule.scala | 42 +++++++++++++++++++ .../src/mill/scalalib/PublishModule.scala | 32 ++++++++------ 2 files changed, 61 insertions(+), 13 deletions(-) diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index d93f3ef2929..910fdf4c5b2 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -248,6 +248,9 @@ trait JavaModule * * Versions and exclusions specified here apply only to dependencies pulled via * `runIvyDeps`. + * + * @param overrideVersions Whether versions in BOMs should override those of the passed + * dependencies themselves */ def runDepManagement: T[Agg[Dep]] = Task { Agg.empty[Dep] } @@ -497,6 +500,8 @@ trait JavaModule /** * Returns a function adding BOM and dependency management details of * this module to a `coursier.core.Dependency` + * + * Meant to be called on values originating from `ivyDeps` */ def processDependency( overrideVersions: Boolean = false @@ -512,6 +517,15 @@ trait JavaModule addBoms(bomDeps0, depMgmt, overrideVersions = overrideVersions) } + /** + * Returns a function adding BOM and dependency management details of + * this module to a `coursier.core.Dependency` + * + * Meant to be called on values originating from `compileIvyDeps` + * + * @param overrideVersions Whether versions in BOMs should override those of the passed + * dependencies themselves + */ def processCompileDependency( overrideVersions: Boolean = false ): Task[coursier.core.Dependency => coursier.core.Dependency] = Task.Anon { @@ -525,6 +539,15 @@ trait JavaModule addBoms(bomDeps0, depMgmt, overrideVersions = overrideVersions) } + /** + * Returns a function adding BOM and dependency management details of + * this module to a `coursier.core.Dependency` + * + * Meant to be called on values originating from `runIvyDeps` + * + * @param overrideVersions Whether versions in BOMs should override those of the passed + * dependencies themselves + */ def processRunDependency( overrideVersions: Boolean = false ): Task[coursier.core.Dependency => coursier.core.Dependency] = Task.Anon { @@ -550,6 +573,11 @@ trait JavaModule } } + /** + * The compile-time Ivy dependencies of this module, with BOM and dependency management details + * added to them. This should be used when propagating the dependencies transitively + * to other modules. + */ def processedCompileIvyDeps: Task[Agg[BoundDep]] = Task.Anon { val processDependency0 = processCompileDependency()() compileIvyDeps().map(bindDependency()).map { dep => @@ -557,6 +585,11 @@ trait JavaModule } } + /** + * The runtime Ivy dependencies of this module, with BOM and dependency management details + * added to them. This should be used when propagating the dependencies transitively + * to other modules. + */ def processedRunIvyDeps: Task[Agg[BoundDep]] = Task.Anon { val processDependency0 = processRunDependency()() runIvyDeps().map(bindDependency()).map { dep => @@ -569,6 +602,9 @@ trait JavaModule * This is calculated from [[ivyDeps]], [[mandatoryIvyDeps]] and recursively from [[moduleDeps]]. */ def transitiveIvyDeps: T[Agg[BoundDep]] = Task { + // overrideVersions is true for dependencies originating from moduleDeps, + // as these are transitive (the direct dependency is the one in moduleDeps), + // so versions in BOMs should override their versions val processDependency0 = processDependency(overrideVersions = true)() processedIvyDeps() ++ T.traverse(moduleDepsChecked)(_.transitiveIvyDeps)().flatten.map { dep => @@ -578,6 +614,9 @@ trait JavaModule /** The compile-only transitive ivy dependencies of this module and all it's upstream compile-only modules. */ def transitiveCompileIvyDeps: T[Agg[BoundDep]] = Task { + // overrideVersions is true for dependencies originating from moduleDeps, + // as these are transitive (the direct dependency is the one in moduleDeps), + // so versions in BOMs should override their versions val processDependency0 = processCompileDependency(overrideVersions = true)() // We never include compile-only dependencies transitively, but we must include normal transitive dependencies! processedCompileIvyDeps() ++ @@ -591,6 +630,9 @@ trait JavaModule * This is calculated from [[runIvyDeps]], [[mandatoryIvyDeps]] and recursively from [[moduleDeps]]. */ def transitiveRunIvyDeps: T[Agg[BoundDep]] = Task { + // overrideVersions is true for dependencies originating from moduleDeps, + // as these are transitive (the direct dependency is the one in moduleDeps), + // so versions in BOMs should override their versions val processDependency0 = processRunDependency(overrideVersions = true)() processedRunIvyDeps() ++ { val viaModules = diff --git a/scalalib/src/mill/scalalib/PublishModule.scala b/scalalib/src/mill/scalalib/PublishModule.scala index 6bf773474cb..69869007e81 100644 --- a/scalalib/src/mill/scalalib/PublishModule.scala +++ b/scalalib/src/mill/scalalib/PublishModule.scala @@ -171,21 +171,24 @@ trait PublishModule extends JavaModule { outer => else dep } - def rootDepsAdjustment = publishXmlDeps0.iterator.flatMap { dep => - val key = coursier.core.DependencyManagement.Key( - coursier.core.Organization(dep.artifact.group), - coursier.core.ModuleName(dep.artifact.id), - coursier.core.Type.jar, - coursier.core.Classifier.empty - ) - bomDepMgmt.get(key).flatMap { values => - if (values.version.nonEmpty && values.version != dep.artifact.version) - Some(key -> values.withVersion("")) - else - None + val bomDepMgmt0 = { + // Ensure we don't override versions of root dependencies with overrides from the BOM + val rootDepsAdjustment = publishXmlDeps0.iterator.flatMap { dep => + val key = coursier.core.DependencyManagement.Key( + coursier.core.Organization(dep.artifact.group), + coursier.core.ModuleName(dep.artifact.id), + coursier.core.Type.jar, + coursier.core.Classifier.empty + ) + bomDepMgmt.get(key).flatMap { values => + if (values.version.nonEmpty && values.version != dep.artifact.version) + Some(key -> values.withVersion("")) + else + None + } } + bomDepMgmt ++ rootDepsAdjustment } - val bomDepMgmt0 = bomDepMgmt ++ rootDepsAdjustment lazy val moduleSet = publishXmlDeps0.map(dep => (dep.artifact.group, dep.artifact.id)).toSet val overrides = { val depMgmtEntries = processedDependencyManagement( @@ -194,6 +197,7 @@ trait PublishModule extends JavaModule { outer => .map(_.dep) .filter(depMgmt => depMgmt.version.nonEmpty && depMgmt.version != "_") .filter { depMgmt => + // Ensure we don't override versions of root dependencies with overrides from the BOM !moduleSet.contains((depMgmt.module.organization.value, depMgmt.module.name.value)) } ) @@ -202,6 +206,7 @@ trait PublishModule extends JavaModule { outer => depMgmtEntries ++ bomDepMgmt0 .filter { case (key, _) => + // Ensure we don't override versions of root dependencies with overrides from the BOM !moduleSet.contains((key.organization.value, key.name.value)) } ) @@ -209,6 +214,7 @@ trait PublishModule extends JavaModule { outer => .map { case (key, values) => val confString = + // we pull the runtime dependency of runtime dependencies, like Maven does if (values.config == Configuration.runtime) "runtime->runtime" else "" Ivy.Override( From c4b5ef851245eab03b734fbc70f91ea97a1878d8 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Fri, 6 Dec 2024 16:19:35 +0100 Subject: [PATCH 14/18] mima --- build.mill | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build.mill b/build.mill index c3cb65ff74d..d55f29dded2 100644 --- a/build.mill +++ b/build.mill @@ -548,6 +548,9 @@ trait MillStableScalaModule extends MillPublishScalaModule with Mima { ProblemFilter.exclude[ReversedMissingMethodProblem]( "mill.scalalib.JavaModule.mill$scalalib$JavaModule$$super$runMain" ), + // Added overrides for a new targets, should be safe + ProblemFilter.exclude[ReversedMissingMethodProblem]("mill.scalalib.JavaModule#JavaModuleTests.mill$scalalib$JavaModule$JavaModuleTests$$super$extraBomDeps"), + ProblemFilter.exclude[ReversedMissingMethodProblem]("mill.scalalib.JavaModule#JavaModuleTests.mill$scalalib$JavaModule$JavaModuleTests$$super$extraDepManagement"), // Terminal is sealed, not sure why MIMA still complains ProblemFilter.exclude[ReversedMissingMethodProblem]("mill.eval.Terminal.task"), From 674e23f731f6d8c6a1a8abccff064e30fd711472 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Fri, 6 Dec 2024 16:26:20 +0100 Subject: [PATCH 15/18] Add comment --- scalalib/src/mill/scalalib/JavaModule.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 910fdf4c5b2..fabddd8d970 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -232,6 +232,10 @@ trait JavaModule * ivy"com.lihaoyi::cask:0.9.4".exclude("org.slf4j", "slf4j-api") * ) * }}} + * + * Versions and exclusions specified here apply to dependencies pulled via + * `ivyDeps`, `compileIvyDeps` (compile-only or "provided" dependencies), and + * `runIvyDeps` (runtime dependencies). */ def depManagement: T[Agg[Dep]] = Task { Agg.empty[Dep] } From 6bbbaca41a468917a4bf664e3d8ae8f9ca5d25e6 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Wed, 11 Dec 2024 15:16:52 +0100 Subject: [PATCH 16/18] Rename {all,extra}BomDeps to {all,extra}BomIvyDeps --- build.mill | 2 +- scalalib/src/mill/scalalib/JavaModule.scala | 16 ++++++++-------- scalalib/src/mill/scalalib/PublishModule.scala | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/build.mill b/build.mill index d55f29dded2..10c082539d8 100644 --- a/build.mill +++ b/build.mill @@ -549,7 +549,7 @@ trait MillStableScalaModule extends MillPublishScalaModule with Mima { "mill.scalalib.JavaModule.mill$scalalib$JavaModule$$super$runMain" ), // Added overrides for a new targets, should be safe - ProblemFilter.exclude[ReversedMissingMethodProblem]("mill.scalalib.JavaModule#JavaModuleTests.mill$scalalib$JavaModule$JavaModuleTests$$super$extraBomDeps"), + ProblemFilter.exclude[ReversedMissingMethodProblem]("mill.scalalib.JavaModule#JavaModuleTests.mill$scalalib$JavaModule$JavaModuleTests$$super$extraBomIvyDeps"), ProblemFilter.exclude[ReversedMissingMethodProblem]("mill.scalalib.JavaModule#JavaModuleTests.mill$scalalib$JavaModule$JavaModuleTests$$super$extraDepManagement"), // Terminal is sealed, not sure why MIMA still complains ProblemFilter.exclude[ReversedMissingMethodProblem]("mill.eval.Terminal.task"), diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index edb70e0e4f3..cc1873af648 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -57,9 +57,9 @@ trait JavaModule } } - override def extraBomDeps = Task.Anon[Agg[BomDependency]] { - super.extraBomDeps() ++ - outer.allBomDeps().map(_.withConfig(Configuration.test)) + override def extraBomIvyDeps = Task.Anon[Agg[BomDependency]] { + super.extraBomIvyDeps() ++ + outer.allBomIvyDeps().map(_.withConfig(Configuration.test)) } override def extraDepManagement = Task.Anon[Agg[Dep]] { super.extraDepManagement() ++ @@ -174,7 +174,7 @@ trait JavaModule */ def bomIvyDeps: T[Agg[Dep]] = Task { Agg.empty[Dep] } - def allBomDeps: Task[Agg[BomDependency]] = Task.Anon { + def allBomIvyDeps: Task[Agg[BomDependency]] = Task.Anon { val modVerOrMalformed = bomIvyDeps().map(bindDependency()).map { bomDep => val fromModVer = coursier.core.Dependency(bomDep.dep.module, bomDep.dep.version) @@ -207,7 +207,7 @@ trait JavaModule * modules, while ensuring test dependencies of the BOM are taken into account too * in the test module. */ - def extraBomDeps: Task[Agg[BomDependency]] = Task.Anon { Agg.empty[BomDependency] } + def extraBomIvyDeps: Task[Agg[BomDependency]] = Task.Anon { Agg.empty[BomDependency] } /** * Dependency management entries that are not meant to be overridden or changed by users. @@ -511,7 +511,7 @@ trait JavaModule overrideVersions: Boolean = false ): Task[coursier.core.Dependency => coursier.core.Dependency] = Task.Anon { val bomDeps0 = - allBomDeps().toSeq.map(_.withConfig(Configuration.compile)) ++ extraBomDeps().toSeq + allBomIvyDeps().toSeq.map(_.withConfig(Configuration.compile)) ++ extraBomIvyDeps().toSeq val depMgmt = processedDependencyManagement( (extraDepManagement().toSeq ++ depManagement().toSeq) .map(bindDependency()) @@ -533,7 +533,7 @@ trait JavaModule def processCompileDependency( overrideVersions: Boolean = false ): Task[coursier.core.Dependency => coursier.core.Dependency] = Task.Anon { - val bomDeps0 = allBomDeps().toSeq.map(_.withConfig(Configuration.provided)) + val bomDeps0 = allBomIvyDeps().toSeq.map(_.withConfig(Configuration.provided)) val depMgmt = processedDependencyManagement( (compileDepManagement().toSeq ++ extraDepManagement().toSeq ++ depManagement().toSeq) .map(bindDependency()) @@ -555,7 +555,7 @@ trait JavaModule def processRunDependency( overrideVersions: Boolean = false ): Task[coursier.core.Dependency => coursier.core.Dependency] = Task.Anon { - val bomDeps0 = allBomDeps().toSeq.map(_.withConfig(Configuration.defaultCompile)) + val bomDeps0 = allBomIvyDeps().toSeq.map(_.withConfig(Configuration.defaultCompile)) val depMgmt = processedDependencyManagement( (runDepManagement().toSeq ++ depManagement()) .map(bindDependency()) diff --git a/scalalib/src/mill/scalalib/PublishModule.scala b/scalalib/src/mill/scalalib/PublishModule.scala index e4f32277f5a..488224587fe 100644 --- a/scalalib/src/mill/scalalib/PublishModule.scala +++ b/scalalib/src/mill/scalalib/PublishModule.scala @@ -146,8 +146,8 @@ trait PublishModule extends JavaModule { outer => val (processedDeps, depMgmt) = defaultResolver().processDeps( processedRunIvyDeps() ++ processedIvyDeps(), resolutionParams = resolutionParams(), - boms = - allBomDeps().toSeq.map(_.withConfig(Configuration.defaultCompile)) ++ extraBomDeps().toSeq + boms = allBomIvyDeps().toSeq.map(_.withConfig(Configuration.defaultCompile)) ++ + extraBomIvyDeps().toSeq ) (processedDeps.map(_.moduleVersion).toMap, depMgmt) } From 70a85e13a570338bc3ab3a5ed220636dc72737b2 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Sun, 15 Dec 2024 23:15:06 +0100 Subject: [PATCH 17/18] fmt --- scalalib/src/mill/scalalib/PublishModule.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/scalalib/src/mill/scalalib/PublishModule.scala b/scalalib/src/mill/scalalib/PublishModule.scala index 7a6dfbb41d0..d9aeb813b61 100644 --- a/scalalib/src/mill/scalalib/PublishModule.scala +++ b/scalalib/src/mill/scalalib/PublishModule.scala @@ -200,7 +200,6 @@ trait PublishModule extends JavaModule { outer => // Ensure we don't override versions of root dependencies with overrides from the BOM !moduleSet.contains((depMgmt.module.organization.value, depMgmt.module.name.value)) } - ) val entries = coursier.core.DependencyManagement.add( Map.empty, From baf330be637ed4cb90ee5fbe9ae3e0bc3ce36b81 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Sun, 15 Dec 2024 23:15:11 +0100 Subject: [PATCH 18/18] Fix tests --- scalalib/test/src/mill/scalalib/BomTests.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scalalib/test/src/mill/scalalib/BomTests.scala b/scalalib/test/src/mill/scalalib/BomTests.scala index 3448c37d0d0..3fc34307cc4 100644 --- a/scalalib/test/src/mill/scalalib/BomTests.scala +++ b/scalalib/test/src/mill/scalalib/BomTests.scala @@ -838,7 +838,7 @@ object BomTests extends TestSuite { val res = eval(modules.bomScope.provided.fail.resolvedIvyDeps) assert( res.left.exists(_.toString.contains( - "not found: https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java-util/_/protobuf-java-util-_.pom" + "not found: https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java-util//protobuf-java-util-.pom" )) ) } @@ -853,7 +853,7 @@ object BomTests extends TestSuite { val res = eval(modules.bomScope.testScopeFail.compileClasspath) assert( res.left.exists(_.toString.contains( - "not found: https://repo1.maven.org/maven2/org/scalatest/scalatest_2.13/_/scalatest_2.13-_.pom" + "not found: https://repo1.maven.org/maven2/org/scalatest/scalatest_2.13//scalatest_2.13-.pom" )) ) } @@ -902,7 +902,7 @@ object BomTests extends TestSuite { val res = eval(modules.depMgmtScope.provided.fail.resolvedIvyDeps) assert( res.left.exists(_.toString.contains( - "not found: https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java-util/_/protobuf-java-util-_.pom" + "not found: https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java-util//protobuf-java-util-.pom" )) ) }