diff --git a/CHANGES.md b/CHANGES.md index 6ca6220f00..cad669168d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -31,7 +31,9 @@ #1187, #1243) - Add a `@short_title` tag to specify the short title of a page for use in the sidebar / breadcrumbs (@panglesd, #1246) -- Add a 'remap' option to HTML generation for partial docsets (@jonludlam, #1189) +- Add a frontmatter syntax for mld pages (@panglesd, #1187) +- Add 'remap' and 'remap-file' options to HTML generation for partial docsets + (@jonludlam, #1189, #1248) - Added an `html-generate-asset` command (@panglesd, #1185) - Added syntax for images, videos, audio (@panglesd, #1184) - Added the ability to order pages in the table of content (@panglesd, #1193) diff --git a/src/driver/common_args.ml b/src/driver/common_args.ml index 1aea27f7e4..d58149c77c 100644 --- a/src/driver/common_args.ml +++ b/src/driver/common_args.ml @@ -52,6 +52,10 @@ let generate_grep = let doc = "Show html-generate commands containing the string" in Arg.(value & opt (some string) None & info [ "html-grep" ] ~doc) +let remap = + let doc = "Remap paths in non-selected packages to ocaml.org" in + Arg.(value & flag & info [ "remap" ] ~doc) + type t = { verbose : bool; odoc_dir : Fpath.t; @@ -65,6 +69,7 @@ type t = { compile_grep : string option; link_grep : string option; generate_grep : string option; + remap : bool; } let term = @@ -82,7 +87,8 @@ let term = and+ odoc_bin = odoc_bin and+ compile_grep = compile_grep and+ link_grep = link_grep - and+ generate_grep = generate_grep in + and+ generate_grep = generate_grep + and+ remap = remap in { verbose; odoc_dir; @@ -96,4 +102,5 @@ let term = compile_grep; link_grep; generate_grep; + remap; } diff --git a/src/driver/compile.ml b/src/driver/compile.ml index e8d184771b..df065acd17 100644 --- a/src/driver/compile.ml +++ b/src/driver/compile.ml @@ -242,7 +242,7 @@ let link : compiled list -> _ = c | _ -> Logs.debug (fun m -> m "linking %a" Fpath.pp c.odoc_file); - link c.odoc_file c.odocl_file c.enable_warnings; + if c.to_output then link c.odoc_file c.odocl_file c.enable_warnings; (match c.kind with | `Intf _ -> Atomic.incr Stats.stats.linked_units | `Mld -> Atomic.incr Stats.stats.linked_mlds @@ -262,7 +262,7 @@ let sherlodoc_index_one ~output_dir (index : Odoc_unit.index) = Sherlodoc.index ~format:`js ~inputs ~dst (); rel_path -let html_generate ~occurrence_file output_dir linked = +let html_generate ~occurrence_file ~remaps output_dir linked = let tbl = Hashtbl.create 10 in let _ = OS.Dir.create output_dir |> Result.get_ok in Sherlodoc.js Fpath.(output_dir // Sherlodoc.js_file); @@ -289,34 +289,43 @@ let html_generate ~occurrence_file output_dir linked = rel_path | Some p -> Promise.await p in - let html_generate : linked -> unit = - fun l -> - let output_dir = Fpath.to_string output_dir in - let input_file = l.odocl_file in - match l.kind with - | `Intf { hidden = true; _ } -> () - | `Impl { src_path; _ } -> - Odoc.html_generate_source ~search_uris:[] ~output_dir ~input_file - ~source:src_path (); - Odoc.html_generate_source ~search_uris:[] ~output_dir ~input_file - ~source:src_path ~as_json:true (); - Atomic.incr Stats.stats.generated_units - | `Asset -> - Odoc.html_generate_asset ~output_dir ~input_file:l.odoc_file - ~asset_path:l.input_file () - | _ -> - let search_uris, index = - match l.index with - | None -> (None, None) - | Some index -> - let db_path = compile_index index in - let search_uris = [ db_path; Sherlodoc.js_file ] in - let index = index.output_file in - (Some search_uris, Some index) - in - Odoc.html_generate ?search_uris ?index ~output_dir ~input_file (); - Odoc.html_generate ?search_uris ?index ~output_dir ~input_file - ~as_json:true (); - Atomic.incr Stats.stats.generated_units + let html_generate : Fpath.t option -> linked -> unit = + fun remap_file l -> + (if l.to_output then + let output_dir = Fpath.to_string output_dir in + let input_file = l.odocl_file in + match l.kind with + | `Intf { hidden = true; _ } -> () + | `Impl { src_path; _ } -> + Odoc.html_generate_source ~search_uris:[] ~output_dir ~input_file + ~source:src_path (); + Odoc.html_generate_source ~search_uris:[] ~output_dir ~input_file + ~source:src_path ~as_json:true (); + Atomic.incr Stats.stats.generated_units + | `Asset -> + Odoc.html_generate_asset ~output_dir ~input_file:l.odoc_file + ~asset_path:l.input_file () + | _ -> + let search_uris, index = + match l.index with + | None -> (None, None) + | Some index -> + let db_path = compile_index index in + let search_uris = [ db_path; Sherlodoc.js_file ] in + let index = index.output_file in + (Some search_uris, Some index) + in + Odoc.html_generate ?search_uris ?index ?remap:remap_file ~output_dir + ~input_file (); + Odoc.html_generate ?search_uris ?index ~output_dir ~input_file + ~as_json:true ()); + Atomic.incr Stats.stats.generated_units in - Fiber.List.iter html_generate linked + if List.length remaps = 0 then Fiber.List.iter (html_generate None) linked + else + Bos.OS.File.with_tmp_oc "remap.%s.txt" + (fun fpath oc () -> + List.iter (fun (a, b) -> Printf.fprintf oc "%s:%s\n%!" a b) remaps; + Fiber.List.iter (html_generate (Some fpath)) linked) + () + |> ignore diff --git a/src/driver/compile.mli b/src/driver/compile.mli index ef120179fa..bf919d5f08 100644 --- a/src/driver/compile.mli +++ b/src/driver/compile.mli @@ -14,4 +14,9 @@ type linked val link : compiled list -> linked list -val html_generate : occurrence_file:Fpath.t -> Fpath.t -> linked list -> unit +val html_generate : + occurrence_file:Fpath.t -> + remaps:(string * string) list -> + Fpath.t -> + linked list -> + unit diff --git a/src/driver/dune_style.ml b/src/driver/dune_style.ml index 32e0a3d4c2..d1416cc3ef 100644 --- a/src/driver/dune_style.ml +++ b/src/driver/dune_style.ml @@ -122,7 +122,8 @@ let of_dune_build dir = assets = [] (* When dune has a notion of doc assets, do something *); - enable_warnings = false; + selected = false; + remaps = []; pkg_dir; other_docs = []; config = Global_config.empty; diff --git a/src/driver/landing_pages.ml b/src/driver/landing_pages.ml index f122265ddd..45f35b5e5b 100644 --- a/src/driver/landing_pages.ml +++ b/src/driver/landing_pages.ml @@ -18,6 +18,7 @@ let make_index ~dirs ~rel_dir ?index ~content () = odoc_file; odocl_file; enable_warnings = false; + to_output = true; kind = `Mld; index; } diff --git a/src/driver/odoc.ml b/src/driver/odoc.ml index 14f72f505f..e34394c3f5 100644 --- a/src/driver/odoc.ml +++ b/src/driver/odoc.ml @@ -180,7 +180,7 @@ let compile_index ?(ignore_output = false) ~output_file ?occurrence_file ~json ignore @@ Cmd_outputs.submit log desc cmd (Some output_file) let html_generate ~output_dir ?index ?(ignore_output = false) - ?(search_uris = []) ?(as_json = false) ~input_file:file () = + ?(search_uris = []) ?remap ?(as_json = false) ~input_file:file () = let open Cmd in let index = match index with None -> empty | Some idx -> v "--index" % p idx @@ -193,6 +193,9 @@ let html_generate ~output_dir ?index ?(ignore_output = false) let cmd = !odoc % "html-generate" % p file %% index %% search_uris % "-o" % output_dir in + let cmd = + match remap with None -> cmd | Some f -> cmd % "--remap-file" % p f + in let cmd = if as_json then cmd % "--as-json" else cmd in let desc = Printf.sprintf "Generating HTML for %s" (Fpath.to_string file) in let log = diff --git a/src/driver/odoc.mli b/src/driver/odoc.mli index f1a201f0a7..bcfb718c6c 100644 --- a/src/driver/odoc.mli +++ b/src/driver/odoc.mli @@ -55,6 +55,7 @@ val html_generate : ?index:Fpath.t -> ?ignore_output:bool -> ?search_uris:Fpath.t list -> + ?remap:Fpath.t -> ?as_json:bool -> input_file:Fpath.t -> unit -> diff --git a/src/driver/odoc_driver.ml b/src/driver/odoc_driver.ml index eaad6e4d62..bb0f21cda8 100644 --- a/src/driver/odoc_driver.ml +++ b/src/driver/odoc_driver.ml @@ -133,6 +133,7 @@ let run mode compile_grep; link_grep; generate_grep; + remap; } = Option.iter (fun odoc_bin -> Odoc.odoc := Bos.Cmd.v odoc_bin) odoc_bin; let _ = Voodoo.find_universe_and_version "foo" in @@ -195,6 +196,14 @@ let run mode | _ -> failwith "Error, expecting singleton library in voodoo mode") | _ -> None in + let remaps = + if remap then + List.concat_map + (fun (_, pkg) -> pkg.Packages.remaps) + (Util.StringMap.bindings all) + else [] + in + Logs.debug (fun m -> m "XXXX Remaps length: %d" (List.length remaps)); let () = Eio.Fiber.both (fun () -> @@ -204,7 +213,7 @@ let run mode let odocl_dir = Option.value odocl_dir ~default:odoc_dir in { Odoc_unit.odoc_dir; odocl_dir; index_dir; mld_dir } in - Odoc_units_of.packages ~dirs ~extra_paths all + Odoc_units_of.packages ~dirs ~extra_paths ~remap all in Compile.init_stats units; let compiled = @@ -229,7 +238,9 @@ let run mode let () = Odoc.count_occurrences ~input:[ odoc_dir ] ~output in output in - let () = Compile.html_generate ~occurrence_file html_dir linked in + let () = + Compile.html_generate ~occurrence_file ~remaps html_dir linked + in let _ = Odoc.support_files html_dir in ()) (fun () -> render_stats env nb_workers) diff --git a/src/driver/odoc_unit.ml b/src/driver/odoc_unit.ml index 3409f8377c..9c40809c78 100644 --- a/src/driver/odoc_unit.ml +++ b/src/driver/odoc_unit.ml @@ -58,6 +58,7 @@ type 'a unit = { pkgname : string option; index : index option; enable_warnings : bool; + to_output : bool; kind : 'a; } diff --git a/src/driver/odoc_unit.mli b/src/driver/odoc_unit.mli index dc0e2f306b..950cb7bf2d 100644 --- a/src/driver/odoc_unit.mli +++ b/src/driver/odoc_unit.mli @@ -33,6 +33,7 @@ type 'a unit = { pkgname : string option; index : index option; enable_warnings : bool; + to_output : bool; kind : 'a; } diff --git a/src/driver/odoc_units_of.ml b/src/driver/odoc_units_of.ml index 02e4b7b73a..bc7ba9d8ad 100644 --- a/src/driver/odoc_units_of.ml +++ b/src/driver/odoc_units_of.ml @@ -1,6 +1,6 @@ open Odoc_unit -let packages ~dirs ~extra_paths (pkgs : Packages.t list) : t list = +let packages ~dirs ~extra_paths ~remap (pkgs : Packages.t list) : t list = let { odoc_dir; odocl_dir; index_dir; mld_dir = _ } = dirs in (* [module_of_hash] Maps a hash to the corresponding [Package.t], library name and [Packages.modulety]. [lib_dirs] maps a library name to the odoc dir containing its @@ -88,7 +88,9 @@ let packages ~dirs ~extra_paths (pkgs : Packages.t list) : t list = in let make_unit ~name ~kind ~rel_dir ~input_file ~pkg ~lib_deps ~enable_warnings - : _ unit = + ~to_output : _ unit = + let to_output = to_output || not remap in + (* If we haven't got active remapping, we output everything *) let ( // ) = Fpath.( // ) in let ( / ) = Fpath.( / ) in let pkg_args = args_of pkg lib_deps in @@ -109,6 +111,7 @@ let packages ~dirs ~extra_paths (pkgs : Packages.t list) : t list = odoc_file; odocl_file; kind; + to_output; enable_warnings; index = Some (index_of pkg); } @@ -142,7 +145,7 @@ let packages ~dirs ~extra_paths (pkgs : Packages.t list) : t list = in let name = intf.mif_path |> Fpath.rem_ext |> Fpath.basename in make_unit ~name ~kind ~rel_dir ~input_file:intf.mif_path ~pkg ~lib_deps - ~enable_warnings:pkg.enable_warnings + ~enable_warnings:pkg.selected ~to_output:pkg.selected in match Hashtbl.find_opt intf_cache intf.mif_hash with | Some unit -> unit @@ -170,7 +173,7 @@ let packages ~dirs ~extra_paths (pkgs : Packages.t list) : t list = in let unit = make_unit ~name ~kind ~rel_dir ~input_file:impl.mip_path ~pkg - ~lib_deps ~enable_warnings:pkg.enable_warnings + ~lib_deps ~enable_warnings:pkg.selected ~to_output:pkg.selected in Some unit in @@ -185,9 +188,13 @@ let packages ~dirs ~extra_paths (pkgs : Packages.t list) : t list = in let of_lib pkg (lib : Packages.libty) = let lib_deps = Util.StringSet.add lib.lib_name lib.lib_deps in - let index = index_of pkg in - let landing_page :> t = Landing_pages.library ~dirs ~pkg ~index lib in - landing_page :: List.concat_map (of_module pkg lib lib_deps) lib.modules + let landing_page :> t list = + if pkg.Packages.selected then + let index = index_of pkg in + [ Landing_pages.library ~dirs ~pkg ~index lib ] + else [] + in + landing_page @ List.concat_map (of_module pkg lib lib_deps) lib.modules in let of_mld pkg (mld : Packages.mld) : mld unit list = let open Fpath in @@ -202,7 +209,7 @@ let packages ~dirs ~extra_paths (pkgs : Packages.t list) : t list = in let unit = make_unit ~name ~kind ~rel_dir ~input_file:mld_path ~pkg ~lib_deps - ~enable_warnings:pkg.enable_warnings + ~enable_warnings:pkg.selected ~to_output:pkg.selected in [ unit ] in @@ -216,7 +223,7 @@ let packages ~dirs ~extra_paths (pkgs : Packages.t list) : t list = let lib_deps = Util.StringSet.empty in let unit = make_unit ~name ~kind ~rel_dir ~input_file:md ~pkg ~lib_deps - ~enable_warnings:pkg.enable_warnings + ~enable_warnings:pkg.selected ~to_output:pkg.selected in [ unit ] | _ -> @@ -233,7 +240,7 @@ let packages ~dirs ~extra_paths (pkgs : Packages.t list) : t list = let unit = let name = asset_path |> Fpath.basename |> ( ^ ) "asset-" in make_unit ~name ~kind ~rel_dir ~input_file:asset_path ~pkg - ~lib_deps:Util.StringSet.empty ~enable_warnings:false + ~lib_deps:Util.StringSet.empty ~enable_warnings:false ~to_output:true in [ unit ] in @@ -252,7 +259,7 @@ let packages ~dirs ~extra_paths (pkgs : Packages.t list) : t list = (Fpath.normalize (Fpath.v "./index.mld"))) pkg.mlds in - if has_index_page then [] + if has_index_page || not pkg.selected then [] else let index = index_of pkg in [ Landing_pages.package ~dirs ~pkg ~index ] diff --git a/src/driver/odoc_units_of.mli b/src/driver/odoc_units_of.mli index d47158681c..67a7b74b05 100644 --- a/src/driver/odoc_units_of.mli +++ b/src/driver/odoc_units_of.mli @@ -1,4 +1,8 @@ open Odoc_unit val packages : - dirs:dirs -> extra_paths:Voodoo.extra_paths -> Packages.t list -> t list + dirs:dirs -> + extra_paths:Voodoo.extra_paths -> + remap:bool -> + Packages.t list -> + t list diff --git a/src/driver/packages.ml b/src/driver/packages.ml index 9bfaa24933..6c05b0c0c8 100644 --- a/src/driver/packages.ml +++ b/src/driver/packages.ml @@ -87,7 +87,8 @@ type t = { libraries : libty list; mlds : mld list; assets : asset list; - enable_warnings : bool; + selected : bool; + remaps : (string * string) list; other_docs : Fpath.t list; pkg_dir : Fpath.t; config : Global_config.t; @@ -101,13 +102,13 @@ let pp fmt t = libraries: %a;@,\ mlds: %a;@,\ assets: %a;@,\ - enable_warnings: %b;@,\ + selected: %b;@,\ other_docs: %a;@,\ pkg_dir: %a@,\ }@]" t.name t.version (Fmt.Dump.list pp_libty) t.libraries (Fmt.Dump.list pp_mld) - t.mlds (Fmt.Dump.list pp_asset) t.assets t.enable_warnings - (Fmt.Dump.list Fpath.pp) t.other_docs Fpath.pp t.pkg_dir + t.mlds (Fmt.Dump.list pp_asset) t.assets t.selected (Fmt.Dump.list Fpath.pp) + t.other_docs Fpath.pp t.pkg_dir let maybe_prepend_top top_dir dir = match top_dir with None -> dir | Some d -> Fpath.(d // dir) @@ -412,7 +413,8 @@ let of_libs ~packages_dir libs = libraries; mlds; assets; - enable_warnings = false; + selected = false; + remaps = []; other_docs; pkg_dir; config; @@ -470,7 +472,28 @@ let of_packages ~packages_dir packages = files.docs |> Fpath.Set.of_list in - let enable_warnings = List.mem pkg.name packages in + let selected = List.mem pkg.name packages in + let remaps = + if selected then [] + else + let local_pkg_path = Fpath.to_string (Fpath.to_dir_path pkg_dir) in + let pkg_path = + Printf.sprintf "https://ocaml.org/p/%s/%s/doc/" pkg.name + pkg.version + in + let lib_paths = + List.map + (fun libty -> + let lib_name = libty.lib_name in + let local_lib_path = + Printf.sprintf "%s%s/" local_pkg_path lib_name + in + let lib_path = pkg_path in + (local_lib_path, lib_path)) + libraries + in + (local_pkg_path, pkg_path) :: lib_paths + in let other_docs = Fpath.Set.elements other_docs in Util.StringMap.add pkg.name { @@ -479,7 +502,8 @@ let of_packages ~packages_dir packages = libraries; mlds; assets; - enable_warnings; + selected; + remaps; other_docs; pkg_dir; config; @@ -487,11 +511,6 @@ let of_packages ~packages_dir packages = acc) Util.StringMap.empty all in - let result = fix_missing_deps packages in - Logs.debug (fun m -> - m "ZZZZ Result: %a" - Fmt.(Dump.list (pair string pp)) - (Util.StringMap.bindings result)); - result + fix_missing_deps packages type set = t Util.StringMap.t diff --git a/src/driver/packages.mli b/src/driver/packages.mli index 3fbed55783..61ed731de9 100644 --- a/src/driver/packages.mli +++ b/src/driver/packages.mli @@ -73,7 +73,8 @@ type t = { libraries : libty list; mlds : mld list; assets : asset list; - enable_warnings : bool; + selected : bool; + remaps : (string * string) list; other_docs : Fpath.t list; pkg_dir : Fpath.t; config : Global_config.t; diff --git a/src/driver/voodoo.ml b/src/driver/voodoo.ml index a7d266ee3c..52eb6b2dd0 100644 --- a/src/driver/voodoo.ml +++ b/src/driver/voodoo.ml @@ -229,7 +229,8 @@ let process_package pkg = libraries; mlds; assets; - enable_warnings = false; + selected = true; + remaps = []; other_docs = []; pkg_dir = top_dir pkg; config; diff --git a/src/html/link.ml b/src/html/link.ml index 4b3a09f250..d63745b519 100644 --- a/src/html/link.ml +++ b/src/html/link.ml @@ -13,17 +13,22 @@ module Path = struct let remap config f = let l = String.concat "/" f in - try - let prefix, replacement = - List.find - (fun (prefix, _replacement) -> - Astring.String.is_prefix ~affix:prefix l) - (Config.remap config) - in - let len = String.length prefix in - let l = String.sub l len (String.length l - len) in - Some (replacement ^ l) - with Not_found -> None + let remaps = + List.filter + (fun (prefix, _replacement) -> Astring.String.is_prefix ~affix:prefix l) + (Config.remap config) + in + let remaps = + List.sort + (fun (a, _) (b, _) -> compare (String.length b) (String.length a)) + remaps + in + match remaps with + | [] -> None + | (prefix, replacement) :: _ -> + let len = String.length prefix in + let l = String.sub l len (String.length l - len) in + Some (replacement ^ l) let get_dir_and_file ~config url = let l = Url.Path.to_list url in diff --git a/src/odoc/bin/main.ml b/src/odoc/bin/main.ml index 2549eb3ce2..21c3ef234a 100644 --- a/src/odoc/bin/main.ml +++ b/src/odoc/bin/main.ml @@ -1151,10 +1151,31 @@ module Odoc_html_args = struct let doc = "Remap an identifier to an external URL." in Arg.(value & opt_all convert_remap [] & info [ "R" ] ~doc) + let remap_file = + let doc = "File containing remap rules." in + Arg.(value & opt (some file) None & info ~docv:"FILE" ~doc [ "remap-file" ]) + let extra_args = let config semantic_uris closed_details indent theme_uri support_uri - search_uris flat as_json remap = + search_uris flat as_json remap remap_file = let open_details = not closed_details in + let remap = + match remap_file with + | None -> remap + | Some f -> + let ic = open_in f in + let rec loop acc = + match input_line ic with + | exception _ -> + close_in ic; + acc + | line -> ( + match Astring.String.cut ~sep:":" line with + | Some (orig, mapped) -> loop ((orig, mapped) :: acc) + | None -> loop acc) + in + loop [] + in let html_config = Odoc_html.Config.v ~theme_uri ~support_uri ~search_uris ~semantic_uris ~indent ~flat ~open_details ~as_json ~remap () @@ -1163,7 +1184,7 @@ module Odoc_html_args = struct in Term.( const config $ semantic_uris $ closed_details $ indent $ theme_uri - $ support_uri $ search_uri $ flat $ as_json $ remap) + $ support_uri $ search_uri $ flat $ as_json $ remap $ remap_file) end module Odoc_html = Make_renderer (Odoc_html_args) diff --git a/test/integration/remap.t/remap.txt b/test/integration/remap.t/remap.txt new file mode 100644 index 0000000000..c852aecf75 --- /dev/null +++ b/test/integration/remap.t/remap.txt @@ -0,0 +1,3 @@ +prefix/otherpkg/:https://mysite.org/bar/ +prefix/:https://mysite.org/foo/ + diff --git a/test/integration/remap.t/run.t b/test/integration/remap.t/run.t index dad872a34f..e71214afe1 100644 --- a/test/integration/remap.t/run.t +++ b/test/integration/remap.t/run.t @@ -18,3 +18,16 @@ This shouldn't stop us from outputting the remapped package though, and the foll $ odoc html-generate -o _html3 _odoc/prefix/otherpkg/otherlib.odocl -R prefix/otherpkg/:https://mysite.org/p/otherpkg/1.2.3/ +The order shouldn't matter, the longest prefix ought to win + $ odoc html-generate -o _html3 --indent _odoc/prefix/mypkg/test.odocl -R prefix/:https://mysite.org/foo/ -R prefix/otherpkg/:https://mysite.org/bar/ + $ odoc html-generate -o _html4 --indent _odoc/prefix/mypkg/test.odocl -R prefix/otherpkg/:https://mysite.org/bar/ -R prefix/:https://mysite.org/foo/ + + $ grep Otherlib/index.html _html3/prefix/mypkg/Test/index.html _html4/prefix/mypkg/Test/index.html + _html3/prefix/mypkg/Test/index.html: Otherlib.t + _html4/prefix/mypkg/Test/index.html: Otherlib.t + + $ odoc html-generate -o _html5 --indent _odoc/prefix/mypkg/test.odocl --remap-file remap.txt + $ grep Otherlib/index.html _html5/prefix/mypkg/Test/index.html + Otherlib.t + +