Merge pull request #234 from Leonidas-from-XIV/hybrid-mode

Allow marking certain packages as "installed via OPAM"

CHANGES.md

@@ -8,6 +8,10 @@
 - Show an error message when the solver can't find any version that satisfies
   the requested version constraint in the user's OPAM file (#215, #248,
   @Leonidas-from-XIV)
+- Allow packages to be marked as being provided by Opam and not to be pulled by
+  `opam-monorepo`. To control this a new optional Opam file field,
+  `x-opam-monorepo-opam-provided` is introduced. Its value is a list of package
+  names that are to be excluded from being pulled (#234, @Leonidas-from-XIV)
 
 ### Changed
 

README.md

@@ -79,9 +79,20 @@ opam-repository) thanks to the `pin-depends`.
 You can use that property to your advantage by allowing one to choose between a "monorepo" or
 regular opam workflow depending on the situation.
 
+You can also exclude packages from the set of packages to
+be vendored by `opam-monorepo`. To do so you can specify an additional field in
+your Opam file:
+
+```
+x-opam-monorepo-opam-provided: ["ocamlformat" "patdiff"]
+```
+
+This will exclude the packages from the list of packages `opam-monorepo` will
+pull, so they can be installed via `opam` manually.
+
 ### opam monorepo pull
 
-The `pull` command fetches the sources using the URLs in the lockfile. It benefits from the opam
+The `pull` command fetches the sources using the URLs in the lockfile. It benefits from the Opam
 cache but its outcome does not depend on your opam configuration.
 
 ## Monorepo projects

cli/lock.ml

@@ -65,11 +65,11 @@ let opam_to_git_remote remote =
   | Some ("git", remote) -> remote
   | _ -> remote
 
-let compute_duniverse ~package_summaries =
+let compute_duniverse ~dependency_entries =
   let get_default_branch remote =
     Exec.git_default_branch ~remote:(opam_to_git_remote remote) ()
   in
-  Duniverse.from_package_summaries ~get_default_branch package_summaries
+  Duniverse.from_dependency_entries ~get_default_branch dependency_entries
 
 let resolve_ref deps =
   let resolve_ref ~repo ~ref =
@@ -256,8 +256,24 @@ let extract_opam_env ~source_config global_state =
   | { global_vars = Some env; _ } -> env
   | { global_vars = None; _ } -> opam_env_from_global_state global_state
 
+let opam_provided_packages ~opam_monorepo_cwd local_packages target_packages =
+  let open Result.O in
+  OpamPackage.Name.Set.fold
+    (fun name acc ->
+      let* acc = acc in
+      match OpamPackage.Name.Map.find_opt name local_packages with
+      | Some (_version, opam) -> (
+          match Source_opam_config.get ~opam_monorepo_cwd opam with
+          | Ok config -> (
+              match config.opam_provided with
+              | None -> Ok acc
+              | Some provided -> Ok (OpamPackage.Name.Set.union provided acc))
+          | Error (`Msg msg) -> Error (`Msg msg))
+      | None -> Ok acc)
+    target_packages (Ok OpamPackage.Name.Set.empty)
+
 let calculate_opam ~source_config ~build_only ~allow_jbuilder ~local_opam_files
-    ~ocaml_version ~target_packages =
+    ~ocaml_version ~target_packages ~opam_provided =
   let open Result.O in
   OpamGlobalState.with_ `Lock_none (fun global_state ->
       let* pin_depends = get_pin_depends ~global_state local_opam_files in
@@ -274,7 +290,7 @@ let calculate_opam ~source_config ~build_only ~allow_jbuilder ~local_opam_files
           let opam_env = extract_opam_env ~source_config global_state in
           let solver = Opam_solve.explicit_repos_solver in
           Opam_solve.calculate ~build_only ~allow_jbuilder ~local_opam_files
-            ~target_packages ~pin_depends ?ocaml_version solver
+            ~target_packages ~opam_provided ~pin_depends ?ocaml_version solver
             (opam_env, local_repo_dirs)
           |> Result.map_error ~f:(interpret_solver_error ~repositories solver)
       | { repositories = None; _ } ->
@@ -284,7 +300,8 @@ let calculate_opam ~source_config ~build_only ~allow_jbuilder ~local_opam_files
                     (OpamSwitch.to_string switch_state.switch));
               let solver = Opam_solve.local_opam_config_solver in
               Opam_solve.calculate ~build_only ~allow_jbuilder ~local_opam_files
-                ~target_packages ~pin_depends ?ocaml_version solver switch_state
+                ~target_packages ~opam_provided ~pin_depends ?ocaml_version
+                solver switch_state
               |> Result.map_error ~f:(fun err ->
                      let repositories = current_repos ~switch_state in
                      interpret_solver_error ~repositories solver err)))
@@ -425,16 +442,19 @@ let run (`Root root) (`Recurse_opam recurse) (`Build_only build_only)
   let* source_config =
     extract_source_config ~opam_monorepo_cwd:root ~opam_files target_packages
   in
-  let* package_summaries =
+  let* opam_provided =
+    opam_provided_packages ~opam_monorepo_cwd:root opam_files target_packages
+  in
+  let* dependency_entries =
     calculate_opam ~source_config ~build_only ~allow_jbuilder ~ocaml_version
-      ~local_opam_files:opam_files ~target_packages
+      ~local_opam_files:opam_files ~target_packages ~opam_provided
   in
   Common.Logs.app (fun l -> l "Calculating exact pins for each of them.");
-  let* duniverse = compute_duniverse ~package_summaries >>= resolve_ref in
+  let* duniverse = compute_duniverse ~dependency_entries >>= resolve_ref in
   let target_depexts = target_depexts opam_files target_packages in
   let lockfile =
     Lockfile.create ~source_config ~root_packages:target_packages
-      ~package_summaries ~root_depexts:target_depexts ~duniverse ()
+      ~dependency_entries ~root_depexts:target_depexts ~duniverse ()
   in
   let* () =
     Lockfile.save ~opam_monorepo_cwd:root ~file:lockfile_path lockfile

doc/index.mld

@@ -9,3 +9,5 @@ Table of contents:
 - {{!page-faq}A FAQ} to answer common questions about [opam monorepo]
 - {{!page-concepts}A description of the concepts} behind [opam monorepo].
 - Some {{!page-workflows}common workflows} used in [opam monorepo] projects.
+- A {{!page-"opam-provided"} way to use [opam-monorepo] with non-[dune] dependencies}
+  for when it is not possible to use [dun]e

doc/opam-provided.mld

@@ -0,0 +1,116 @@
+{1 [opam]-provided dependencies}
+
+This section documents a feature that’s useful when some dependencies cannot be
+installed via [opam-monorepo]. This workflow isn’t meant for regular use but
+only as a way to use [opam-monorepo] in cases where it wouldn't be otherwise
+possible.
+
+We suggest minimizing its usage and prefer to use (and contribute) [dune-ports]
+from the [opam-overlays] repository. It's usage is safest with
+leaf-dependencies and those that don’t have dependency intersections with
+vendored packages. Due to possibly of unexpected interactions, this feature is
+only meant for advanced users, and the solutions it produces might be in flux.
+
+{2 Usecases}
+
+Sometimes it is not possible to put all your dependencies into the [duniverse],
+be it due to your dependencies not building with [dune] or some tooling that
+should be just installed via [opam]. Such examples include:
+
+{ul
+{- Packages not building with Dune}
+{- Packages that should not be vendored for legal reasons}
+}
+
+These users can opt-in to having some of their dependencies provided by [opam]
+instead of [opam-monorepo] pulling them into the [duniverse].
+
+{2 Initial setup}
+
+With the default configuration, [opam-monorepo] runs in full-[duniverse] mode,
+i.e., requiring all the dependencies of your [opam] files to be able to be
+built as part of the [duniverse].
+
+As such it is only recommended that packages are installed via [opam] for which
+there is no way to build them with [dune], e.g., by using the
+{{:https://github.com/dune-universe/opam-overlays}opam-overlays} repository
+with Dune ports.
+
+To let [opam-monorepo] know a package is to be installed via [opam], it needs
+to be marked as such in the Opam file. For this, it needs to be added as normal
+in the [depends] field, as well in addition into the new
+[x-opam-monorepo-opam-provided] field. This field takes a single package or a
+list of packages to be installed via [opam] instead of being pulled into the
+[duniverse].
+
+To configure [opam-monorepo] to avoid pulling in package [foo], specify it in
+the list of packages provided by Opam:
+
+{[
+depends: [
+  "foo"
+]
+x-opam-monorepo-opam-provided: ["foo"]
+]}
+
+This can either be specified directly in the Opam file or, if you use Dune to
+generate Opam files, in the [.template] file. In such case, make sure to
+regenerate the Opam file.
+
+As a shortcut syntax, if there’s only one package, it’s possible to leave out
+the list and just specify the package itself:
+
+{[
+depends: [
+  "foo"
+]
+x-opam-monorepo-opam-provided: "foo"
+]}
+
+{2 Usage}
+
+After you configured your Opam file, you have to {e lock} the environment.
+
+{[
+$ opam monorepo lock
+]}
+
+After this succeeds, you will have an [.opam.locked] file, which contains all
+your project’s dependencies (including transitive dependencies). The ones that
+[opam-monorepo] will pull into the [duniverse] are marked as [vendor].
+
+{[
+$ opam install --ignore-pin-depends --deps-only ./ --locked
+$ opam-monorepo pull
+]}
+
+{2 How it works}
+
+In addition to calculating the dependencies of the packages that will be
+included in the [duniverse], [opam-monorepo] calculates the dependencies of the
+packages to be installed via [opam]. [opam-monorepo] then writes a lockfile
+that sets a variable on all the dependencies it will pull, whereas
+[opam]-provided dependencies don't have variables set. In Opam, unset variables
+are false; thus Opam will ignore the packages that [opam-monorepo] will pull.
+
+The [opam]-provided subset of dependencies is then {e excluded} from the
+[duniverse] packages.
+
+{3 Resolution of dependency overlaps}
+
+There are cases where a dependency package appears multiple times in the
+project’s dependency tree. It’s possible that a package is part of the packages
+to be included in the [duniverse], as well as packages installed by [opam].
+
+In such cases there are two possibilities:
+
+{ol
+{- Install the package in [opam], exclude it from duniverse}
+{- Install the package in [opam], include a duplicate in the duniverse}
+}
+
+Both these approaches have advantages and disadvantages. [opam-monorepo]
+currently chooses #2 to maximize the amount of packages that are vendored, thus
+increasing the size of the reproducible set. Yet this doesn’t mean that two
+different versions of the package can be installed. The set of packages in Opam
+and the [duniverse] must be co-installable!

lib/duniverse.ml

@@ -205,8 +205,14 @@ let dev_repo_map_from_packages packages =
         | Some pkgs -> Some (pkg :: pkgs)
         | None -> Some [ pkg ]))
 
-let from_package_summaries ~get_default_branch summaries =
+let from_dependency_entries ~get_default_branch dependencies =
   let open Result.O in
+  let summaries =
+    List.map
+      ~f:(fun Opam.Dependency_entry.{ package_summary; vendored = _ } ->
+        package_summary)
+      dependencies
+  in
   let results =
     List.map
       ~f:(Repo.Package.from_package_summary ~get_default_branch)

lib/duniverse.mli

@@ -63,9 +63,9 @@ type t = resolved Repo.t list
 
 val equal : t -> t -> bool
 
-val from_package_summaries :
+val from_dependency_entries :
   get_default_branch:(string -> (string, Rresult.R.msg) result) ->
-  Opam.Package_summary.t list ->
+  Opam.Dependency_entry.t list ->
   (unresolved Repo.t list, [ `Msg of string ]) result
 (** Build opamverse and duniverse from a list of [Types.Opam.entry] values.
     It filters out virtual packages and packages with unknown dev-repo.  *)

lib/lockfile.ml

@@ -90,13 +90,15 @@ module Depends = struct
 
   type t = dependency list
 
-  let from_package_summaries l =
-    List.map l ~f:(fun summary ->
+  let from_dependency_entries dependency_entries =
+    List.map dependency_entries
+      ~f:(fun Opam.Dependency_entry.{ vendored; package_summary } ->
         let vendored =
-          (not @@ Opam.Package_summary.is_base_package summary)
-          && (not @@ Opam.Package_summary.is_virtual summary)
+          (not @@ Opam.Package_summary.is_base_package package_summary)
+          && (not @@ Opam.Package_summary.is_virtual package_summary)
+          && vendored
         in
-        { vendored; package = summary.package })
+        { vendored; package = package_summary.package })
 
   let variable_equal a b =
     String.equal (OpamVariable.to_string a) (OpamVariable.to_string b)
@@ -240,7 +242,13 @@ module Depexts = struct
     let c = OpamSysPkg.Set.compare pkg_set pkg_set' in
     if c = 0 then compare filter filter' else c
 
-  let all ~root_depexts ~package_summaries =
+  let all ~root_depexts ~dependency_entries =
+    let package_summaries =
+      List.map
+        ~f:(fun Opam.Dependency_entry.{ package_summary; vendored = _ } ->
+          package_summary)
+        dependency_entries
+    in
     let transitive_depexts =
       List.map
         ~f:(fun { Opam.Package_summary.depexts; _ } -> depexts)
@@ -262,13 +270,13 @@ type t = {
 
 let depexts t = t.depexts
 
-let create ~source_config ~root_packages ~package_summaries ~root_depexts
+let create ~source_config ~root_packages ~dependency_entries ~root_depexts
     ~duniverse () =
   let version = Version.current in
-  let depends = Depends.from_package_summaries package_summaries in
+  let depends = Depends.from_dependency_entries dependency_entries in
   let pin_depends = Pin_depends.from_duniverse duniverse in
   let duniverse_dirs = Duniverse_dirs.from_duniverse duniverse in
-  let depexts = Depexts.all ~root_depexts ~package_summaries in
+  let depexts = Depexts.all ~root_depexts ~dependency_entries in
   {
     version;
     root_packages;

lib/lockfile.mli

@@ -3,7 +3,7 @@ type t
 val create :
   source_config:Source_opam_config.t ->
   root_packages:OpamPackage.Name.Set.t ->
-  package_summaries:Opam.Package_summary.t list ->
+  dependency_entries:Opam.Dependency_entry.t list ->
   root_depexts:(OpamSysPkg.Set.t * OpamTypes.filter) list list ->
   duniverse:Duniverse.t ->
   unit ->

lib/opam.ml

@@ -127,9 +127,27 @@ module Depexts = struct
 end
 
 module Pp = struct
-  let package = Fmt.using OpamPackage.to_string Fmt.string
+  module Package : Pp_combinators.Opam.Printable with type t = OpamPackage.t =
+  struct
+    type t = OpamPackage.t
 
-  let package_name = Fmt.using OpamPackage.Name.to_string Fmt.string
+    let pp = Fmt.using OpamPackage.to_string Fmt.string
+  end
+
+  module Package_name :
+    Pp_combinators.Opam.Printable with type t = OpamPackage.Name.t = struct
+    type t = OpamPackage.Name.t
+
+    let pp = Fmt.using OpamPackage.Name.to_string Fmt.string
+  end
+
+  module Package_name_set =
+    Pp_combinators.Opam.Make_Set (OpamPackage.Name.Set) (Package_name)
+  module Package_set = Pp_combinators.Opam.Make_Set (OpamPackage.Set) (Package)
+
+  let package = Package.pp
+
+  let package_name = Package_name.pp
 
   let version = Fmt.using OpamPackage.Version.to_string Fmt.string
 
@@ -169,7 +187,7 @@ module Package_summary = struct
       (option ~brackets:true string)
       dev_repo Depexts.pp depexts
 
-  let from_opam ~pkg:package opam_file =
+  let from_opam package opam_file =
     let url_field = OpamFile.OPAM.url opam_file in
     let url_src = Option.map ~f:Url.from_opam_field url_field in
     let hashes =
@@ -193,6 +211,10 @@ module Package_summary = struct
     | _ -> false
 end
 
+module Dependency_entry = struct
+  type t = { package_summary : Package_summary.t; vendored : bool }
+end
+
 let local_package_version opam_file ~explicit_version =
   match explicit_version with
   | Some v -> v