Support exporting Windows Dockerfiles

dune-project

@@ -16,7 +16,7 @@
  astring
  (fmt (>= 0.8.9))
  logs
- cmdliner
+ (cmdliner (> 1.0.4))
  (tar (>= 2.0))
  (tar-unix (>= 2.0))
  yojson

example.windows.spec

@@ -0,0 +1,49 @@
+; This script builds OBuilder itself using a snapshot of the
+; ocaml/opam:window-mingw-20H2-ocaml-4.12 base image.
+;
+; Run it from the top-level of the OBuilder source tree, e.g.
+;
+; dune exec -- obuilder build --docker-backend=../var -f example.windows.spec .
+;
+; The result can then be found in ../var/HASH/rootfs/ (where HASH is
+; displayed at the end of the build).
+
+((build dev
+ ((from ocaml/opam@sha256:875f8962b438f16fd20076730d8db3f328155ab2ce9499b375bda5c4d9da9788)
+ (workdir /project)
+ (env OPAM_HASH "96d0b8dbb64a291108c8bccb5cdbb1936f853207") ; Fix the version of opam-repository we want
+ (shell C:/cygwin64/bin/bash.exe --login -c)
+ (run
+ (network "Default Switch")
+ (shell
+ "cd /home/opam/opam-repository \
+ && (git cat-file -e $OPAM_HASH || git fetch origin master) \
+ && git reset -q --hard $OPAM_HASH \
+ && git log --no-decorate -n1 --oneline \
+ && opam update -u"))
+ (shell C:/Windows/System32/cmd.exe /S /C)
+ ; Copy just the opam file first (helps caching)
+ (copy (src obuilder-spec.opam obuilder.opam) (dst ./))
+ (run (shell "ocaml-env exec --64 -- opam pin add -yn ."))
+ ; Install OS package dependencies
+ (run
+ (cache (opam-archives (target /opam/.opam/download-cache)))
+ (shell "ocaml-env exec --64 -- opam depext -yu obuilder"))
+ ; Install OCaml dependencies
+ (run
+ (cache (opam-archives (target /opam/.opam/download-cache)))
+ (shell "ocaml-env exec --64 -- opam install --deps-only -t obuilder"))
+ (copy ; Copy the rest of the source code
+ (src .)
+ (dst /project/)
+ (exclude .git _build _opam))
+ (run (shell "ocaml-env exec --64 -- dune build @install @runtest")))) ; Build and test
+ ; Now generate a small runtime image with just the resulting binary:
+ (from mcr.microsoft.com/windows/nanoserver:20H2)
+ (copy (from (build dev))
+ (src /cygwin64/usr/x86_64-w64-mingw32/sys-root/mingw/bin/libsqlite3-0.dll)
+ (dst /obuilder/))
+ (copy (from (build dev))
+ (src /project/_build/default/main.exe)
+ (dst /obuilder/obuilder.exe))
+ (run (shell "/obuilder/obuilder.exe --help")))

lib_spec/docker.ml

@@ -10,9 +10,15 @@ let default_ctx = {
 let pp_pair f (k, v) =
  Fmt.pf f "%s=%s" k v
 
-let pp_wrap =
+let pp_escape ~escape =
+ match escape with
+ | '\\' -> Fmt.any " \\@\n "
+ | '`' -> Fmt.any " `@\n "
+ | _ -> assert false
+
+let pp_wrap ~escape =
  Fmt.using (String.split_on_char '\n')
- Fmt.(list ~sep:(unit " \\@\n ") (using String.trim string))
+ Fmt.(list ~sep:(pp_escape ~escape) (using String.trim string))
 
 let pp_cache ~ctx f { Cache.id; target; buildkit_options } =
  let buildkit_options = match ctx.user with
@@ -40,11 +46,11 @@ let pp_mount_secret ~ctx f { Secret.id; target; buildkit_options } =
  in
  Fmt.pf f "%a" Fmt.(list ~sep:(unit ",") pp_pair) buildkit_options
 
-let pp_run ~ctx f { Spec.cache; shell; secrets; network = _ } =
+let pp_run ~escape ~ctx f { Spec.cache; shell; secrets; network = _ } =
  Fmt.pf f "RUN %a%a%a"
  Fmt.(list (pp_mount_secret ~ctx ++ const string " ")) secrets
  Fmt.(list (pp_cache ~ctx ++ const string " ")) cache
- pp_wrap shell
+ (pp_wrap ~escape) shell
 
 let pp_copy ~ctx f { Spec.from; src; dst; exclude = _ } =
  let from = match from with
@@ -79,31 +85,35 @@ let quote ~escape v =
  Buffer.add_substring buf v !j (len - !j);
  Buffer.contents buf
 
-let pp_op ~buildkit ctx f : Spec.op -> ctx = function
+let pp_op ~buildkit ~escape ctx f : Spec.op -> ctx = function
  | `Comment x -> Fmt.pf f "# %s" x; ctx
  | `Workdir x -> Fmt.pf f "WORKDIR %s" x; ctx
  | `Shell xs -> Fmt.pf f "SHELL [ %a ]" Fmt.(list ~sep:comma (quote string)) xs; ctx
- | `Run x when buildkit -> pp_run ~ctx f x; ctx
- | `Run x -> pp_run ~ctx f { x with cache = []; secrets = []}; ctx
+ | `Run x when buildkit -> pp_run ~escape ~ctx f x; ctx
+ | `Run x -> pp_run ~escape ~ctx f { x with cache = []; secrets = []}; ctx
  | `Copy x -> pp_copy ~ctx f x; ctx
  | `User (`Unix { uid; gid } as u) -> Fmt.pf f "USER %d:%d" uid gid; { user = u }
  | `User (`Windows { name } as u) -> Fmt.pf f "USER %s" name; { user = u }
- | `Env (k, v) -> Fmt.pf f "ENV %s=\"%s\"" k (quote ~escape:'\\' v); ctx
+ | `Env (k, v) -> Fmt.pf f "ENV %s=\"%s\"" k (quote ~escape v); ctx
 
-let rec convert ~buildkit f (name, { Spec.child_builds; from; ops }) =
+let rec convert ~buildkit ~escape f (name, { Spec.child_builds; from; ops }) =
  child_builds |> List.iter (fun (name, spec) ->
- convert ~buildkit f (Some name, spec);
+ convert ~buildkit ~escape f (Some name, spec);
  Format.pp_print_newline f ();
  );
  Fmt.pf f "@[<h>FROM %s%a@]@." from Fmt.(option (const string " as " ++ string)) name;
  let (_ : ctx) = List.fold_left (fun ctx op ->
  Format.pp_open_hbox f ();
- let ctx = pp_op ~buildkit ctx f op in
+ let ctx = pp_op ~buildkit ~escape ctx f op in
  Format.pp_close_box f ();
  Format.pp_print_newline f ();
  ctx
  ) default_ctx ops
  in ()
 
-let dockerfile_of_spec ~buildkit t =
- Fmt.strf "%a" (convert ~buildkit) (None, t)
+let dockerfile_of_spec ~buildkit ~escape t =
+ Fmt.str "%a" (fun f ->
+ if escape <> '\\' then
+ (Fmt.pf f "@[<h>#escape=%c@]@." escape;
+ convert ~buildkit ~escape f)
+ else convert ~buildkit ~escape f) (None, t)

lib_spec/docker.mli

@@ -1,5 +1,6 @@
-val dockerfile_of_spec : buildkit:bool -> Spec.t -> string
-(** [dockerfile_of_spec x] produces a Dockerfile that aims to be equivalent to [x].
+val dockerfile_of_spec : buildkit:bool -> escape:char -> Spec.t -> string
+(** [dockerfile_of_spec ~buildkit ~escape x] produces a Dockerfile
+ that aims to be equivalent to [x].
 
  However, note that:
 

main.ml

@@ -83,10 +83,10 @@ let delete () store conf id =
  Builder.delete builder id ~log:(fun id -> Fmt.pr "Removing %s@." id)
  end
 
-let dockerfile () buildkit spec =
+let dockerfile () buildkit escape spec =
  Sexplib.Sexp.load_sexp spec
  |> Obuilder_spec.t_of_sexp
- |> Obuilder_spec.Docker.dockerfile_of_spec ~buildkit
+ |> Obuilder_spec.Docker.dockerfile_of_spec ~buildkit ~escape
  |> print_endline
 
 open Cmdliner
@@ -178,9 +178,16 @@ let buildkit =
  ~doc:"Output extended BuildKit syntax"
  ["buildkit"]
 
+let escape =
+ Arg.value @@
+ Arg.opt Arg.char (if Sys.unix then '\\' else '`') @@
+ Arg.info
+ ~doc:"Dockerfile escape character. The default is '\\' on UNIX and '`' on Windows."
+ ["escape"]
+
 let dockerfile =
  let doc = "Convert a spec to Dockerfile format" in
- Term.(const dockerfile $ setup_log $ buildkit $ spec_file),
+ Term.(const dockerfile $ setup_log $ buildkit $ escape $ spec_file),
  Term.info "dockerfile" ~doc
 
 let healthcheck =

obuilder.opam

@@ -14,7 +14,7 @@ depends: [
  "astring"
  "fmt" {>= "0.8.9"}
  "logs"
- "cmdliner"
+ "cmdliner" {> "1.0.4"}
  "tar" {>= "2.0"}
  "tar-unix" {>= "2.0"}
  "yojson"
@@ -45,6 +45,7 @@ build: [
 ]
 dev-repo: "git+https://github.com/ocurrent/obuilder.git"
 pin-depends: [
+ ["cmdliner.1.0.5" "git+https://github.com/dbuenzli/cmdliner.git#db4d02a9eb47b5c43127a67cb121004b03ea3719"]
  ["lwt.5.4.3" "git+https://github.com/MisterDA/lwt.git#e703de884a798b0c8c45c6f792691cdaf578d35b"]
  ["sha.1.15" "git+https://github.com/djs55/ocaml-sha.git#8b77ab306fc3a5e94219580d6a2c92bc11745d60"]
  ["tar.2.0.0" "git+https://github.com/MisterDA/ocaml-tar.git#47d20548d39337009b6d73309eb2aab346494e76"]

obuilder.opam.template

@@ -1,4 +1,5 @@
 pin-depends: [
+ ["cmdliner.1.0.5" "git+https://github.com/dbuenzli/cmdliner.git#db4d02a9eb47b5c43127a67cb121004b03ea3719"]
  ["lwt.5.4.3" "git+https://github.com/MisterDA/lwt.git#e703de884a798b0c8c45c6f792691cdaf578d35b"]
  ["sha.1.15" "git+https://github.com/djs55/ocaml-sha.git#8b77ab306fc3a5e94219580d6a2c92bc11745d60"]
  ["tar.2.0.0" "git+https://github.com/MisterDA/ocaml-tar.git#47d20548d39337009b6d73309eb2aab346494e76"]

test/test.ml

@@ -460,10 +460,10 @@ let test_sexp () =
  (user (uid 1) (gid 2))
  )|}
 
-let test_docker () =
+let test_docker_unix () =
  let test ~buildkit name expect sexp =
  let spec = Spec.t_of_sexp (Sexplib.Sexp.of_string sexp) in
- let got = Obuilder_spec.Docker.dockerfile_of_spec ~buildkit spec in
+ let got = Obuilder_spec.Docker.dockerfile_of_spec ~buildkit ~escape:'\\' spec in
  let expect = remove_indent expect in
  Alcotest.(check string) name expect got
  in
@@ -555,6 +555,105 @@ let test_docker () =
  (shell "command1"))
  ) |}
 
+let test_docker_windows () =
+ let test ~buildkit name expect sexp =
+ let spec = Spec.t_of_sexp (Sexplib.Sexp.of_string sexp) in
+ let got = Obuilder_spec.Docker.dockerfile_of_spec ~buildkit ~escape:'`' spec in
+ let expect = remove_indent expect in
+ Alcotest.(check string) name expect got
+ in
+ test ~buildkit:false "Dockerfile"
+ {| #escape=`
+ FROM base
+ # A test comment
+ WORKDIR C:/src
+ RUN command1
+ SHELL [ "C:/Windows/System32/cmd.exe", "/c" ]
+ RUN command2 && `
+ command3
+ COPY a b c
+ COPY a b c
+ ENV DEBUG="1"
+ USER Zaphod
+ COPY a b c
+ |} {|
+ ((from base)
+ (comment "A test comment")
+ (workdir C:/src)
+ (run (shell "command1"))
+ (shell C:/Windows/System32/cmd.exe /c)
+ (run
+ (cache (a (target /data))
+ (b (target /srv)))
+ (shell "command2 &&
+ command3"))
+ (copy (src a b) (dst c))
+ (copy (src a b) (dst c) (exclude .git _build))
+ (env DEBUG 1)
+ (user (name Zaphod))
+ (copy (src a b) (dst c))
+ ) |};
+ test ~buildkit:true "BuildKit"
+ {| #escape=`
+ FROM base
+ # A test comment
+ WORKDIR /src
+ RUN command1
+ SHELL [ "C:/Windows/System32/cmd.exe", "/c" ]
+ RUN --mount=type=cache,id=a,target=/data,uid=0 --mount=type=cache,id=b,target=/srv,uid=0 command2
+ COPY a b c
+ COPY a b c
+ ENV DEBUG="1"
+ USER Zaphod
+ COPY a b c
+ |} {|
+ ((from base)
+ (comment "A test comment")
+ (workdir /src)
+ (run (shell "command1"))
+ (shell C:/Windows/System32/cmd.exe /c)
+ (run
+ (cache (a (target /data))
+ (b (target /srv)))
+ (shell "command2"))
+ (copy (src a b) (dst c))
+ (copy (src a b) (dst c) (exclude .git _build))
+ (env DEBUG 1)
+ (user (name Zaphod))
+ (copy (src a b) (dst c))
+ ) |};
+ test ~buildkit:false "Multi-stage"
+ {| #escape=`
+ FROM base as tools
+ RUN make tools
+
+ FROM base
+ COPY --from=tools binary /usr/local/bin/
+ |} {|
+ ((build tools
+ ((from base)
+ (run (shell "make tools"))))
+ (from base)
+ (copy (from (build tools)) (src binary) (dst /usr/local/bin/))
+ ) |};
+ test ~buildkit:true "Secrets"
+ {| #escape=`
+ FROM base as tools
+ RUN make tools
+
+ FROM base
+ RUN --mount=type=secret,id=a,target=/secrets/a,uid=0 --mount=type=secret,id=b,target=/secrets/b,uid=0 command1
+ |} {|
+ ((build tools
+ ((from base)
+ (run (shell "make tools"))))
+ (from base)
+ (run
+ (secrets (a (target /secrets/a))
+ (b (target /secrets/b)))
+ (shell "command1"))
+ ) |}
+
 let manifest =
  Alcotest.result
  (Alcotest.testable
@@ -696,7 +795,8 @@ let main_unix () =
  "spec", [
  test_case_sync "Sexp" `Quick test_sexp;
  test_case_sync "Cache ID" `Quick test_cache_id;
- test_case_sync "Docker" `Quick test_docker;
+ test_case_sync "Docker UNIX" `Quick test_docker_unix;
+ test_case_sync "Docker Windows" `Quick test_docker_windows;
  ];
  "build", [
  test_case "Simple" `Quick test_simple;