diff --git a/CHANGES.md b/CHANGES.md index 9391ea0b4..3c94fa382 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,11 @@ ### Changed +- Mark packages to be pulled by opam-monorepo with the `vendor` variable so + using OPAM with `opam install --deps-only --locked .` will not install + packages that will be installed with `opam-monorepo pull` (#237, + @Leonidas-from-XIV) + ### Deprecated ### Fixed diff --git a/cli/list_cmd.ml b/cli/list_cmd.ml index 586032e5b..12072a195 100644 --- a/cli/list_cmd.ml +++ b/cli/list_cmd.ml @@ -2,8 +2,8 @@ open Import open Duniverse type t = { - name : string; - version : string; + name : OpamPackage.Name.t; + version : OpamPackage.Version.t; loc : string; pinned : bool; descr : string option; @@ -14,6 +14,7 @@ type t = { let pad s max_len = Printf.sprintf "%-*s" max_len s let guess_pin ~version ~loc = + let version = OpamPackage.Version.to_string version in (* opam-overlays *) String.is_suffix ~suffix:"+dune" version || String.is_prefix ~prefix:"https://github.com/dune-universe" loc @@ -29,13 +30,15 @@ let pp_pin_version = Fmt.(styled `Blue string) let pp_pin_loc ppf s = Fmt.pf ppf "pinned at %a" Fmt.(styled `Underline string) s -let compare_pkg x y = String.compare x.name y.name +let compare_pkg x y = OpamPackage.Name.compare x.name y.name let pp ~max_name ~max_version ~short ppf t = - if short then Fmt.string ppf t.name + if short then Duniverse_lib.Opam.Pp.package_name ppf t.name else - let padded_name = pad t.name max_name in - let padded_version = pad t.version max_version in + let padded_name = pad (OpamPackage.Name.to_string t.name) max_name in + let padded_version = + pad (OpamPackage.Version.to_string t.version) max_version + in if t.pinned then Fmt.pf ppf "%a %a %a" pp_name padded_name pp_pin_version padded_version pp_pin_loc t.loc @@ -45,7 +48,7 @@ let pp ~max_name ~max_version ~short ppf t = let pkgs_of_repo (t : resolved Repo.t) = List.map - ~f:(fun (pkg : Opam.t) -> + ~f:(fun (pkg : OpamPackage.t) -> let name = pkg.name in let version = pkg.version in let loc = Repo.Url.to_string t.url in @@ -66,11 +69,7 @@ let with_descr pkgs = OpamSwitchState.with_ `Lock_none global_state (fun switch_state -> List.map ~f:(fun pkg -> - let opam = - OpamPackage.create - (OpamPackage.Name.of_string pkg.name) - (OpamPackage.Version.of_string pkg.version) - in + let opam = OpamPackage.create pkg.name pkg.version in match OpamSwitchState.opam switch_state opam with | opam -> { pkg with descr = OpamFile.OPAM.synopsis opam } | exception Not_found -> @@ -88,8 +87,10 @@ let run (`Root root) (`Lockfile explicit_lockfile) short () = let max_name, max_version = List.fold_left ~f:(fun (max_name, max_version) t -> - ( max (String.length t.name) max_name, - max (String.length t.version) max_version )) + ( max (String.length (OpamPackage.Name.to_string t.name)) max_name, + max + (String.length (OpamPackage.Version.to_string t.version)) + max_version )) ~init:(0, 0) pkgs in let pp = pp ~max_name ~max_version ~short in diff --git a/lib/config.ml b/lib/config.ml index 767e9c005..ec95e70af 100644 --- a/lib/config.ml +++ b/lib/config.ml @@ -13,6 +13,7 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. * *) +open Import let base_packages = [ @@ -26,6 +27,8 @@ let base_packages = "ocaml-base-compiler"; "ocaml-variants"; ] + |> List.map ~f:OpamPackage.Name.of_string + |> OpamPackage.Name.Set.of_list let duniverse_opam_repo = "git+https://github.com/dune-universe/opam-overlays.git" @@ -49,3 +52,6 @@ let dune_src_dir = Fpath.(bootstrap_src_dir / "dune") let dune_latest_tag = "2.6.0" (* TODO get from opam metadata *) let lockfile_ext = ".opam.locked" + +(* variable to use for vendoring *) +let vendor_variable = OpamVariable.of_string "vendor" diff --git a/lib/duniverse.ml b/lib/duniverse.ml index 8ce9008f3..eb5709461 100644 --- a/lib/duniverse.ml +++ b/lib/duniverse.ml @@ -1,34 +1,9 @@ open Import -module O = Opam type unresolved = Git.Ref.t type resolved = Git.Ref.resolved -module Opam = struct - type t = { name : string; version : string } - - let equal t t' = - let { name; version } = t in - let { name = name'; version = version' } = t' in - String.equal name name' && String.equal version version' - - let pp fmt { name; version } = Format.fprintf fmt "%s.%s" name version - - let raw_pp fmt { name; version } = - let open Pp_combinators.Ocaml in - Format.fprintf fmt "@[{ name = %a;@ version = %a }@]" string name - string version - - let to_opam { name = n; version = v } = - OpamPackage.(create (Name.of_string n) (Version.of_string v)) - - let from_opam pkg = - let name = OpamPackage.name_to_string pkg in - let version = OpamPackage.version_to_string pkg in - { name; version } -end - module Repo = struct module Url = struct type 'ref t = Git of { repo : string; ref : 'ref } | Other of string @@ -71,23 +46,23 @@ module Repo = struct let to_opam_url t = opam_url_from_string (to_string t) let from_opam_url opam_url = - match O.Url.from_opam opam_url with - | O.Url.Other s -> Ok (Other s) - | O.Url.Git { repo; ref = Some commit } -> + match Opam.Url.from_opam opam_url with + | Opam.Url.Other s -> Ok (Other s) + | Opam.Url.Git { repo; ref = Some commit } -> Ok (Git { repo; ref = { Git.Ref.t = commit; commit } }) | _ -> Error (`Msg "Git URL must be resolved to a commit hash") end module Package = struct type t = { - opam : Opam.t; + opam : OpamPackage.t; dev_repo : string; url : unresolved Url.t; hashes : OpamHash.t list; } let equal t t' = - Opam.equal t.opam t'.opam + OpamPackage.equal t.opam t'.opam && String.equal t.dev_repo t'.dev_repo && Url.equal Git.Ref.equal t.url t'.url @@ -95,14 +70,14 @@ module Repo = struct let open Pp_combinators.Ocaml in Format.fprintf fmt "@[{ opam = %a;@ dev_repo = %a;@ url = %a;@ hashes = %a }@]" - Opam.raw_pp opam string dev_repo (Url.pp Git.Ref.pp) url - (list O.Pp.hash) hashes + Opam.Pp.raw_package opam string dev_repo (Url.pp Git.Ref.pp) url + (list Opam.Pp.hash) hashes let from_package_summary ~get_default_branch ps = - let open O.Package_summary in + let open Opam.Package_summary in let open Result.O in let url ourl = - match (ourl : O.Url.t) with + match (ourl : Opam.Url.t) with | Other s -> Ok (Url.Other s) | Git { repo; ref = Some ref } -> Ok (Url.Git { repo; ref }) | Git { repo; ref = None } -> @@ -111,23 +86,17 @@ module Repo = struct match ps with | _ when is_base_package ps -> Ok None | { url_src = None; _ } | { dev_repo = None; _ } -> Ok None - | { - url_src = Some url_src; - name; - version; - dev_repo = Some dev_repo; - hashes; - _; - } -> + | { url_src = Some url_src; package; dev_repo = Some dev_repo; hashes; _ } + -> url url_src >>= fun url -> - Ok (Some { opam = { name; version }; dev_repo; url; hashes }) + Ok (Some { opam = package; dev_repo; url; hashes }) end type 'ref t = { dir : string; url : 'ref Url.t; hashes : OpamHash.t list; - provided_packages : Opam.t list; + provided_packages : OpamPackage.t list; } let log_url_selection ~dev_repo ~packages ~highest_version_package = @@ -136,7 +105,8 @@ module Repo = struct | Other s -> s in let pp_package fmt { Package.opam = { name; version }; url; _ } = - Format.fprintf fmt "%s.%s: %s" name version (url_to_string url) + Format.fprintf fmt "%a.%a: %s" Opam.Pp.package_name name Opam.Pp.version + version (url_to_string url) in let sep fmt () = Format.fprintf fmt "\n" in Logs.info (fun l -> @@ -179,7 +149,7 @@ module Repo = struct need dune to provide that feature. *) let highest_version_package = List.max_exn packages ~compare:(fun p p' -> - OpamVersionCompare.compare p.Package.opam.version p'.opam.version) + OpamPackage.Version.compare p.Package.opam.version p'.opam.version) in log_url_selection ~dev_repo ~packages ~highest_version_package; let url = highest_version_package.url in @@ -198,16 +168,16 @@ module Repo = struct in String.equal dir dir' && Url.equal equal_ref url url' - && List.equal O.Hash.equal hashes hashes' - && List.equal Opam.equal provided_packages provided_packages' + && List.equal Opam.Hash.equal hashes hashes' + && List.equal OpamPackage.equal provided_packages provided_packages' let pp pp_ref fmt { dir; url; hashes; provided_packages } = let open Pp_combinators.Ocaml in Format.fprintf fmt "@[{ dir = %a;@ url = %a;@ hashes = %a;@ provided_packages = %a \ }@]" - string dir (Url.pp pp_ref) url (list O.Pp.hash) hashes (list Opam.raw_pp) - provided_packages + string dir (Url.pp pp_ref) url (list Opam.Pp.hash) hashes + (list Opam.Pp.raw_package) provided_packages let resolve ~resolve_ref ({ url; _ } as t) = let open Result.O in diff --git a/lib/duniverse.mli b/lib/duniverse.mli index 60606611c..f6f1cbd60 100644 --- a/lib/duniverse.mli +++ b/lib/duniverse.mli @@ -1,22 +1,7 @@ -module O = Opam - type unresolved = Git.Ref.t type resolved = Git.Ref.resolved -module Opam : sig - type t = { name : string; version : string } - (** Type of dependencies to install through opam *) - - val equal : t -> t -> bool - - val pp : t Fmt.t - - val to_opam : t -> OpamPackage.t - - val from_opam : OpamPackage.t -> t -end - module Repo : sig module Url : sig type 'ref t = Git of { repo : string; ref : 'ref } | Other of string @@ -38,7 +23,7 @@ module Repo : sig dir : string; url : 'ref Url.t; hashes : OpamHash.t list; - provided_packages : Opam.t list; + provided_packages : OpamPackage.t list; } (** Type of dependencies to clone in the duniverse *) @@ -52,7 +37,7 @@ module Repo : sig module Package : sig type t = { - opam : Opam.t; + opam : OpamPackage.t; dev_repo : string; url : unresolved Url.t; hashes : OpamHash.t list; @@ -64,7 +49,7 @@ module Repo : sig val from_package_summary : get_default_branch:(string -> (string, Rresult.R.msg) result) -> - O.Package_summary.t -> + Opam.Package_summary.t -> (t option, [ `Msg of string ]) result end @@ -80,7 +65,7 @@ val equal : t -> t -> bool val from_package_summaries : get_default_branch:(string -> (string, Rresult.R.msg) result) -> - O.Package_summary.t list -> + Opam.Package_summary.t list -> (unresolved Repo.t list, [ `Msg of string ]) result (** Build opamverse and duniverse from a list of [Types.Opam.entry] values. It filters out virtual packages and packages with unknown dev-repo. *) diff --git a/lib/lockfile.ml b/lib/lockfile.ml index b13f36d08..b57bfacab 100644 --- a/lib/lockfile.ml +++ b/lib/lockfile.ml @@ -131,29 +131,75 @@ module Root_packages = struct end module Depends = struct - type t = (string * string) list + type dependency = { package : OpamPackage.t; vendored : bool } + + type t = dependency list let from_package_summaries l = - List.map l ~f:(fun (p : Opam.Package_summary.t) -> (p.name, p.version)) + List.map l ~f:(fun summary -> + let vendored = + (not @@ Opam.Package_summary.is_base_package summary) + && (not @@ Opam.Package_summary.is_virtual summary) + in + { vendored; package = summary.package }) + + let variable_equal a b = + String.equal (OpamVariable.to_string a) (OpamVariable.to_string b) let from_filtered_formula formula = let open OpamTypes in let atoms = OpamFormula.ands_to_list formula in Result.List.map atoms ~f:(function | Atom (name, Atom (Constraint (`Eq, FString version))) -> - Ok (OpamPackage.Name.to_string name, version) + let version = OpamPackage.Version.of_string version in + let package = OpamPackage.create name version in + Ok { package; vendored = false } + | Atom + ( name, + And + ( Atom (Constraint (`Eq, FString version)), + Atom (Filter (FIdent ([], var, None))) ) ) + | Atom + ( name, + And + ( Atom (Filter (FIdent ([], var, None))), + Atom (Constraint (`Eq, FString version)) ) ) + when variable_equal var Config.vendor_variable -> + let version = OpamPackage.Version.of_string version in + let package = OpamPackage.create name version in + Ok { package; vendored = true } | _ -> Error (`Msg "Invalid opam-monorepo lockfile: depends should be expressed as \ - a list equality constraints")) - - let one_to_formula (name, version) : OpamTypes.filtered_formula = - Atom - (OpamPackage.Name.of_string name, Atom (Constraint (`Eq, FString version))) + a list equality constraints optionally with a `vendor` variable")) + + let one_to_formula { package; vendored } : OpamTypes.filtered_formula = + let name = package.name in + let version = package.version in + let variable = + OpamFormula.Atom + (OpamTypes.Filter (OpamTypes.FIdent ([], Config.vendor_variable, None))) + in + let version_constraint = + OpamFormula.Atom + (OpamTypes.Constraint + (`Eq, OpamTypes.FString (OpamPackage.Version.to_string version))) + in + let formula = + match vendored with + | true -> OpamFormula.And (version_constraint, variable) + | false -> version_constraint + in + Atom (name, formula) - let to_filtered_formula t = - let sorted = List.sort ~cmp:(fun (n, _) (n', _) -> String.compare n n') t in + let to_filtered_formula xs = + let sorted = + List.sort + ~cmp:(fun { package; _ } { package = package'; _ } -> + OpamPackage.compare package package') + xs + in match sorted with | [] -> OpamFormula.Empty | hd :: tl -> @@ -169,7 +215,7 @@ module Pin_depends = struct let open Duniverse.Repo in List.concat_map l ~f:(fun { provided_packages; url; _ } -> let url = Url.to_opam_url url in - List.map provided_packages ~f:(fun p -> (Duniverse.Opam.to_opam p, url))) + List.map provided_packages ~f:(fun p -> (p, url))) let sort t = List.sort ~cmp:(fun (pkg, _) (pkg', _) -> OpamPackage.compare pkg pkg') t @@ -311,7 +357,7 @@ let to_duniverse { duniverse_dirs; pin_depends; _ } = OpamUrl.Map.update url (fun l -> package :: l) [] acc) |> OpamUrl.Map.bindings in - Result.List.map packages_per_url ~f:(fun (url, packages) -> + Result.List.map packages_per_url ~f:(fun (url, provided_packages) -> match OpamUrl.Map.find_opt url duniverse_dirs with | None -> let msg = @@ -321,9 +367,6 @@ let to_duniverse { duniverse_dirs; pin_depends; _ } = in Error (`Msg msg) | Some (dir, hashes) -> - let provided_packages = - List.map packages ~f:Duniverse.Opam.from_opam - in url_to_duniverse_url url >>= fun url -> Ok { Duniverse.Repo.dir; url; hashes; provided_packages }) diff --git a/lib/opam.ml b/lib/opam.ml index 3fa0673d7..f5782facf 100644 --- a/lib/opam.ml +++ b/lib/opam.ml @@ -121,38 +121,50 @@ module Depexts = struct t end +module Pp = struct + let package = Fmt.using OpamPackage.to_string Fmt.string + + let package_name = Fmt.using OpamPackage.Name.to_string Fmt.string + + let version = Fmt.using OpamPackage.Version.to_string Fmt.string + + let raw_package fmt pkg = + Format.fprintf fmt "@[{ name = %a;@ version = %a }@]" package_name + pkg.OpamPackage.name version pkg.version + + let hash = Hash.pp + + let url = Fmt.using OpamUrl.to_string Fmt.string +end + module Package_summary = struct type t = { - name : string; - version : string; + package : OpamPackage.t; url_src : Url.t option; hashes : OpamHash.t list; dev_repo : string option; depexts : (OpamSysPkg.Set.t * OpamTypes.filter) list; } - let equal { name; version; url_src; hashes; dev_repo; depexts } t' = - String.equal name t'.name - && String.equal version t'.version + let equal { package; url_src; hashes; dev_repo; depexts } t' = + OpamPackage.equal package t'.package && Option.equal Url.equal url_src t'.url_src && List.equal Hash.equal hashes t'.hashes && Option.equal String.equal dev_repo t'.dev_repo && Depexts.equal depexts t'.depexts - let pp fmt { name; version; url_src; hashes; dev_repo; depexts } = + let pp fmt { package; url_src; hashes; dev_repo; depexts } = let open Pp_combinators.Ocaml in Format.fprintf fmt "@[{ name = %a;@ version = %a;@ url_src = %a;@ hashes = %a;@ \ dev_repo = %a;@ depexts = %a }@]" - string name string version + Pp.package_name package.name Pp.version package.version (option ~brackets:true Url.pp) url_src (list Hash.pp) hashes (option ~brackets:true string) dev_repo Depexts.pp depexts - let from_opam ~pkg opam_file = - let name = OpamPackage.name_to_string pkg in - let version = OpamPackage.version_to_string pkg in + let from_opam ~pkg:package opam_file = let url_field = OpamFile.OPAM.url opam_file in let url_src = Option.map ~f:Url.from_opam_field url_field in let hashes = @@ -162,7 +174,7 @@ module Package_summary = struct Option.map ~f:OpamUrl.to_string (OpamFile.OPAM.dev_repo opam_file) in let depexts = OpamFile.OPAM.depexts opam_file in - { name; version; url_src; hashes; dev_repo; depexts } + { package; url_src; hashes; dev_repo; depexts } let is_virtual = function | { url_src = None; _ } -> true @@ -170,20 +182,12 @@ module Package_summary = struct | _ -> false let is_base_package = function - | { name; _ } when List.mem name ~set:Config.base_packages -> true + | { package; _ } + when OpamPackage.Name.Set.mem package.name Config.base_packages -> + true | _ -> false end -module Pp = struct - let package = Fmt.using OpamPackage.to_string Fmt.string - - let package_name = Fmt.using OpamPackage.Name.to_string Fmt.string - - let hash = Hash.pp - - let url = Fmt.using OpamUrl.to_string Fmt.string -end - let local_package_version opam_file ~explicit_version = match explicit_version with | Some v -> v diff --git a/lib/opam.mli b/lib/opam.mli index 9cd8995fb..02369fa77 100644 --- a/lib/opam.mli +++ b/lib/opam.mli @@ -14,8 +14,7 @@ end module Package_summary : sig type t = { - name : string; - version : string; + package : OpamPackage.t; url_src : Url.t option; hashes : OpamHash.t list; dev_repo : string option; @@ -41,8 +40,12 @@ end module Pp : sig val package : OpamPackage.t Fmt.t + val raw_package : OpamPackage.t Fmt.t + val package_name : OpamPackage.Name.t Fmt.t + val version : OpamPackage.Version.t Fmt.t + val hash : OpamHash.t Fmt.t val url : OpamUrl.t Fmt.t diff --git a/test/lib/test_duniverse.ml b/test/lib/test_duniverse.ml index 108e4fa00..16bf92122 100644 --- a/test/lib/test_duniverse.ml +++ b/test/lib/test_duniverse.ml @@ -17,9 +17,15 @@ module Testable = struct end end -let summary_factory ?(name = "") ?(version = "") ?dev_repo ?url_src +let opam_factory ~name ~version = + let name = OpamPackage.Name.of_string name in + let version = OpamPackage.Version.of_string version in + OpamPackage.create name version + +let summary_factory ?(name = "undefined") ?(version = "1") ?dev_repo ?url_src ?(hashes = []) ?(depexts = []) () = - { Opam.Package_summary.name; version; dev_repo; url_src; hashes; depexts } + let package = opam_factory ~name ~version in + { Opam.Package_summary.package; dev_repo; url_src; hashes; depexts } module Repo = struct module Package = struct @@ -56,7 +62,7 @@ module Repo = struct (Ok (Some { - opam = { name = "y"; version = "v" }; + opam = opam_factory ~name:"y" ~version:"v"; dev_repo = "d"; url = Other "u"; hashes = []; @@ -73,7 +79,7 @@ module Repo = struct (Ok (Some { - opam = { name = "y"; version = "v" }; + opam = opam_factory ~name:"y" ~version:"v"; dev_repo = "d"; url = Git { repo = "r"; ref = "master" }; hashes = []; @@ -85,7 +91,8 @@ module Repo = struct let package_factory ?(name = "") ?(version = "") ?(dev_repo = "") ?(url = Duniverse.Repo.Url.Other "") ?(hashes = []) () = let open Duniverse.Repo.Package in - { opam = { name; version }; dev_repo; url; hashes } + let opam = opam_factory ~name ~version in + { opam; dev_repo; url; hashes } let test_from_packages = let make_test ~name ~dev_repo ~packages ~expected () = @@ -111,7 +118,7 @@ module Repo = struct dir = "d"; url = Other "u"; hashes = []; - provided_packages = [ { name = "p"; version = "v" } ]; + provided_packages = [ opam_factory ~name:"p" ~version:"v" ]; } (); make_test ~name:"Uses repository name as dir" @@ -126,7 +133,7 @@ module Repo = struct dir = "repo"; url = Other "u"; hashes = []; - provided_packages = [ { name = "p"; version = "v" } ]; + provided_packages = [ opam_factory ~name:"p" ~version:"v" ]; } (); make_test ~name:"Expection for dune" @@ -141,7 +148,7 @@ module Repo = struct dir = "dune_"; url = Other "u"; hashes = []; - provided_packages = [ { name = "p"; version = "v" } ]; + provided_packages = [ opam_factory ~name:"p" ~version:"v" ]; } (); make_test ~name:"Add all to provided packages" ~dev_repo:"d" @@ -159,8 +166,8 @@ module Repo = struct hashes = []; provided_packages = [ - { name = "d"; version = "zdev" }; - { name = "d-lwt"; version = "zdev" }; + opam_factory ~name:"d" ~version:"zdev"; + opam_factory ~name:"d-lwt" ~version:"zdev"; ]; } (); @@ -179,7 +186,8 @@ module Repo = struct hashes = []; provided_packages = [ - { name = "d"; version = "1" }; { name = "d-lwt"; version = "2" }; + opam_factory ~name:"d" ~version:"1"; + opam_factory ~name:"d-lwt" ~version:"2"; ]; } (); @@ -222,7 +230,7 @@ let test_from_package_summaries = dir = "d"; url = Other "u"; hashes = []; - provided_packages = [ { name = "x"; version = "v" } ]; + provided_packages = [ opam_factory ~name:"x" ~version:"v" ]; }; ]) (); @@ -243,8 +251,8 @@ let test_from_package_summaries = hashes = []; provided_packages = [ - { name = "y-lwt"; version = "v" }; - { name = "y"; version = "v" }; + opam_factory ~name:"y-lwt" ~version:"v"; + opam_factory ~name:"y" ~version:"v"; ]; }; ])