[Sites] Fix installation through opam

        - Substitute the files in an intermediate directory if needed
        - Backward compatible

        Fixes ocaml#4198 ocaml#4212

Signed-off-by: François Bobot <francois.bobot@cea.fr>

bin/common.ml

@@ -427,6 +427,18 @@ module Options_implied_by_dash_p = struct
                  "Select the build profile, for instance $(b,dev) or \
                   $(b,release). The default is $(b,%s)."
                  (Profile.to_string Dune_rules.Profile.default)))
+    and+ no_promote_install_files =
+      let doc =
+        "Do not promote the generated <package>.install files to the source \
+         tree"
+      in
+      Arg.(value & flag & info [ "no-promote-install-files" ] ~docs ~doc)
+    in
+    let t =
+      { t with
+        promote_install_files =
+          (not no_promote_install_files) && t.promote_install_files
+      }
     in
     match profile with
     | None -> t

bin/install_uninstall.ml

@@ -379,6 +379,14 @@ let install_uninstall ~what =
             ~doc:
               "Make the binaries relocatable (the installation directory can \
                be moved).")
+    and+ create_install_files =
+      Arg.(
+        value & flag
+        & info [ "create-install-files" ]
+            ~doc:
+              "Do not directly install, but create install files in the root \
+               directory and create substituted files if needed in destdir \
+               (_destdir by default).")
     and+ pkgs = Arg.(value & pos_all package_name [] name_)
     and+ context =
       Arg.(
@@ -474,6 +482,13 @@ let install_uninstall ~what =
                  in
                  (context, entries_per_package))
         in
+        let destdir =
+          if create_install_files then
+            Some (Option.value ~default:"_destdir" destdir)
+          else
+            destdir
+        in
+        let open Fiber.O in
         let (module Ops) = file_operations ~dry_run ~workspace in
         let files_deleted_in = ref Path.Set.empty in
         let+ () =
@@ -500,28 +515,57 @@ let install_uninstall ~what =
                     Install.Section.Paths.make ~package ~destdir:prefix ?libdir
                       ?mandir ()
                   in
-                  Fiber.sequential_iter entries ~f:(fun entry ->
-                      let special_file = Special_file.of_entry entry in
-                      let dst =
-                        Install.Entry.relative_installed_path entry ~paths
-                        |> interpret_destdir ~destdir
-                      in
-                      let dir = Path.parent_exn dst in
-                      match what with
-                      | Install ->
-                        Ops.remove_if_exists dst;
-                        Printf.eprintf "Installing %s\n%!"
-                          (Path.to_string_maybe_quoted dst);
-                        Ops.mkdir_p dir;
-                        let executable =
-                          Section.should_set_executable_bit entry.section
+                  let+ entries =
+                    Fiber.sequential_map entries ~f:(fun entry ->
+                        let special_file = Special_file.of_entry entry in
+                        let dst =
+                          Install.Entry.relative_installed_path entry ~paths
+                          |> interpret_destdir ~destdir
                         in
-                        Ops.copy_file ~src:entry.src ~dst ~executable
-                          ~special_file ~package ~conf
-                      | Uninstall ->
-                        Ops.remove_if_exists dst;
-                        files_deleted_in := Path.Set.add !files_deleted_in dir;
-                        Fiber.return ())))
+                        let dir = Path.parent_exn dst in
+                        match what with
+                        | Install ->
+                          let* copy =
+                            match special_file with
+                            | _ when not create_install_files ->
+                              Fiber.return true
+                            | None ->
+                              Dune_rules.Artifact_substitution.test_file
+                                ~src:entry.src ()
+                            | Some Special_file.META
+                            | Some Special_file.Dune_package ->
+                              Fiber.return true
+                          in
+                          let msg =
+                            if create_install_files then
+                              "Copying to"
+                            else
+                              "Installing"
+                          in
+                          if copy then
+                            let* () =
+                              Ops.remove_if_exists dst;
+                              Printf.eprintf "%s %s\n%!" msg
+                                (Path.to_string_maybe_quoted dst);
+                              Ops.mkdir_p dir;
+                              let executable =
+                                Section.should_set_executable_bit entry.section
+                              in
+                              Ops.copy_file ~src:entry.src ~dst ~executable
+                                ~special_file ~package ~conf
+                            in
+                            Fiber.return (Install.Entry.set_src entry dst)
+                          else
+                            Fiber.return entry
+                        | Uninstall ->
+                          Ops.remove_if_exists dst;
+                          files_deleted_in := Path.Set.add !files_deleted_in dir;
+                          Fiber.return entry)
+                  in
+                  if create_install_files then
+                    let fn = resolve_package_install workspace package in
+                    Io.write_file (Path.source fn)
+                      (Install.gen_install_file entries)))
         in
         Path.Set.to_list !files_deleted_in
         (* This [List.rev] is to ensure we process children directories before

doc/dune-files.rst

@@ -190,6 +190,8 @@ where ``<setting>`` is one of:
 - ``(enabled_for <languages>)`` can be used to restrict the languages that are
   considered for formatting.
 
+.. _generate_opam_files:
+
 generate_opam_files
 -------------------
 

doc/sites.rst

@@ -140,6 +140,12 @@ install --relocatable --prefix $dir``. The files will be copied to the directory
 to its location. So even if the directory ``$dir`` is moved,
 ``themes_locations`` will be correct.
 
+For installation trough opam, ``dune install`` must be invoked with the option
+``--create-install-files`` which creates an install file ``<pkg>.install`` and
+copy the file that needs subtistution to an intermediary directory. The
+``<pkg>.opam`` file generated by dune :ref:`generate_opam_files` does the right
+invocation.
+
 Implementation details
 ----------------------
 

otherlibs/site/test/run.t

@@ -6,7 +6,7 @@ Test embedding of sites locations information
   $ for i in a b d; do
   > mkdir -p $i
   > cat >$i/dune-project <<EOF
-  > (lang dune 2.8)
+  > (lang dune 2.9)
   > (generate_opam_files true)
   > (using dune_site 0.1)
   > (name $i)
@@ -17,7 +17,8 @@ Test embedding of sites locations information
   $ for i in c; do
   >   mkdir -p $i
   >   cat >$i/dune-project <<EOF
-  > (lang dune 2.8)
+  > (lang dune 2.9)
+  > (generate_opam_files true)
   > (using dune_site 0.1)
   > (name $i)
   > (package (name $i) (sites (share data) (lib plugins)))
@@ -125,6 +126,7 @@ Test embedding of sites locations information
   > (lang dune 2.2)
   > EOF
 
+
 Test with an opam like installation
 --------------------------------
 
@@ -134,27 +136,38 @@ Test with an opam like installation
   # This file is generated by dune, edit dune-project instead
   opam-version: "2.0"
   depends: [
-    "dune" {>= "2.8"}
+    "dune" {>= "2.9"}
     "odoc" {with-doc}
   ]
   build: [
-    ["dune" "subst"] {dev}
+    ["dune" "subst" "--root" "."] {dev}
     [
       "dune"
       "build"
       "-p"
       name
       "-j"
       jobs
+      "--no-promote-install-files"
       "@install"
       "@runtest" {with-test}
       "@doc" {with-doc}
     ]
+    ["dune" "install" "-p" name "--create-install-files" name]
   ]
 
-  $ dune build -p a @install
+  $ dune build -p a --no-promote-install-files @install
+
+  $ test -e a/a.install
+  [1]
+
+  $ dune install -p a --create-install-files a 2> /dev/null
 
+  $ grep "_destdir" a/a.install -c
+  7
 
+  $ grep "_build" a/a.install -c
+  12
 
 Build everything
 ----------------

src/dune_rules/opam_create.ml

@@ -41,6 +41,21 @@ let default_build_command =
       "@doc" {with-doc}
   ]
 ]
+|}))
+  and from_2_9 =
+    lazy
+      (Opam_file.parse_value
+         (Lexbuf.from_string ~fname:"<internal>"
+            {|
+[
+  [ "dune" "subst" "--root" "." ] {dev}
+  [ "dune" "build" "-p" name "-j" jobs "--no-promote-install-files"
+      "@install"
+      "@runtest" {with-test}
+      "@doc" {with-doc}
+  ]
+  [ "dune" "install" "-p" name "--create-install-files" name ]
+]
 |}))
   in
   fun project ->
@@ -49,8 +64,10 @@ let default_build_command =
         before_1_11
       else if Dune_project.dune_version project < (2, 7) then
         from_1_11_before_2_7
+      else if Dune_project.dune_version project < (2, 9) then
+        from_2_7
       else
-        from_2_7)
+        from_2_9)
 
 let package_fields
     { Package.synopsis