diff --git a/dune-project b/dune-project index 5b306e15..3af5d10e 100644 --- a/dune-project +++ b/dune-project @@ -16,7 +16,7 @@ astring (fmt (>= 0.8.9)) logs - cmdliner + (cmdliner (> 1.0.4)) (tar (>= 2.0)) (tar-unix (>= 2.0)) yojson diff --git a/example.windows.spec b/example.windows.spec new file mode 100644 index 00000000..d9e218dc --- /dev/null +++ b/example.windows.spec @@ -0,0 +1,49 @@ +; This script builds OBuilder itself using a snapshot of the +; ocaml/opam:window-mingw-20H2-ocaml-4.12 base image. +; +; Run it from the top-level of the OBuilder source tree, e.g. +; +; dune exec -- obuilder build --docker-backend=../var -f example.windows.spec . +; +; The result can then be found in ../var/HASH/rootfs/ (where HASH is +; displayed at the end of the build). + +((build dev + ((from ocaml/opam@sha256:875f8962b438f16fd20076730d8db3f328155ab2ce9499b375bda5c4d9da9788) + (workdir /project) + (env OPAM_HASH "96d0b8dbb64a291108c8bccb5cdbb1936f853207") ; Fix the version of opam-repository we want + (shell C:/cygwin64/bin/bash.exe --login -c) + (run + (network "Default Switch") + (shell + "cd /home/opam/opam-repository \ + && (git cat-file -e $OPAM_HASH || git fetch origin master) \ + && git reset -q --hard $OPAM_HASH \ + && git log --no-decorate -n1 --oneline \ + && opam update -u")) + (shell C:/Windows/System32/cmd.exe /S /C) + ; Copy just the opam file first (helps caching) + (copy (src obuilder-spec.opam obuilder.opam) (dst ./)) + (run (shell "ocaml-env exec --64 -- opam pin add -yn .")) + ; Install OS package dependencies + (run + (cache (opam-archives (target /opam/.opam/download-cache))) + (shell "ocaml-env exec --64 -- opam depext -yu obuilder")) + ; Install OCaml dependencies + (run + (cache (opam-archives (target /opam/.opam/download-cache))) + (shell "ocaml-env exec --64 -- opam install --deps-only -t obuilder")) + (copy ; Copy the rest of the source code + (src .) + (dst /project/) + (exclude .git _build _opam)) + (run (shell "ocaml-env exec --64 -- dune build @install @runtest")))) ; Build and test + ; Now generate a small runtime image with just the resulting binary: + (from mcr.microsoft.com/windows/nanoserver:20H2) + (copy (from (build dev)) + (src /cygwin64/usr/x86_64-w64-mingw32/sys-root/mingw/bin/libsqlite3-0.dll) + (dst /obuilder/)) + (copy (from (build dev)) + (src /project/_build/default/main.exe) + (dst /obuilder/obuilder.exe)) + (run (shell "/obuilder/obuilder.exe --help"))) diff --git a/lib_spec/docker.ml b/lib_spec/docker.ml index cf6ec1d9..cbf6dcc1 100644 --- a/lib_spec/docker.ml +++ b/lib_spec/docker.ml @@ -10,9 +10,15 @@ let default_ctx = { let pp_pair f (k, v) = Fmt.pf f "%s=%s" k v -let pp_wrap = +let pp_escape ~escape = + match escape with + | '\\' -> Fmt.any " \\@\n " + | '`' -> Fmt.any " `@\n " + | _ -> assert false + +let pp_wrap ~escape = Fmt.using (String.split_on_char '\n') - Fmt.(list ~sep:(unit " \\@\n ") (using String.trim string)) + Fmt.(list ~sep:(pp_escape ~escape) (using String.trim string)) let pp_cache ~ctx f { Cache.id; target; buildkit_options } = let buildkit_options = match ctx.user with @@ -40,11 +46,11 @@ let pp_mount_secret ~ctx f { Secret.id; target; buildkit_options } = in Fmt.pf f "%a" Fmt.(list ~sep:(unit ",") pp_pair) buildkit_options -let pp_run ~ctx f { Spec.cache; shell; secrets; network = _ } = +let pp_run ~escape ~ctx f { Spec.cache; shell; secrets; network = _ } = Fmt.pf f "RUN %a%a%a" Fmt.(list (pp_mount_secret ~ctx ++ const string " ")) secrets Fmt.(list (pp_cache ~ctx ++ const string " ")) cache - pp_wrap shell + (pp_wrap ~escape) shell let pp_copy ~ctx f { Spec.from; src; dst; exclude = _ } = let from = match from with @@ -79,31 +85,35 @@ let quote ~escape v = Buffer.add_substring buf v !j (len - !j); Buffer.contents buf -let pp_op ~buildkit ctx f : Spec.op -> ctx = function +let pp_op ~buildkit ~escape ctx f : Spec.op -> ctx = function | `Comment x -> Fmt.pf f "# %s" x; ctx | `Workdir x -> Fmt.pf f "WORKDIR %s" x; ctx | `Shell xs -> Fmt.pf f "SHELL [ %a ]" Fmt.(list ~sep:comma (quote string)) xs; ctx - | `Run x when buildkit -> pp_run ~ctx f x; ctx - | `Run x -> pp_run ~ctx f { x with cache = []; secrets = []}; ctx + | `Run x when buildkit -> pp_run ~escape ~ctx f x; ctx + | `Run x -> pp_run ~escape ~ctx f { x with cache = []; secrets = []}; ctx | `Copy x -> pp_copy ~ctx f x; ctx | `User (`Unix { uid; gid } as u) -> Fmt.pf f "USER %d:%d" uid gid; { user = u } | `User (`Windows { name } as u) -> Fmt.pf f "USER %s" name; { user = u } - | `Env (k, v) -> Fmt.pf f "ENV %s=\"%s\"" k (quote ~escape:'\\' v); ctx + | `Env (k, v) -> Fmt.pf f "ENV %s=\"%s\"" k (quote ~escape v); ctx -let rec convert ~buildkit f (name, { Spec.child_builds; from; ops }) = +let rec convert ~buildkit ~escape f (name, { Spec.child_builds; from; ops }) = child_builds |> List.iter (fun (name, spec) -> - convert ~buildkit f (Some name, spec); + convert ~buildkit ~escape f (Some name, spec); Format.pp_print_newline f (); ); Fmt.pf f "@[FROM %s%a@]@." from Fmt.(option (const string " as " ++ string)) name; let (_ : ctx) = List.fold_left (fun ctx op -> Format.pp_open_hbox f (); - let ctx = pp_op ~buildkit ctx f op in + let ctx = pp_op ~buildkit ~escape ctx f op in Format.pp_close_box f (); Format.pp_print_newline f (); ctx ) default_ctx ops in () -let dockerfile_of_spec ~buildkit t = - Fmt.strf "%a" (convert ~buildkit) (None, t) +let dockerfile_of_spec ~buildkit ~escape t = + Fmt.str "%a" (fun f -> + if escape <> '\\' then + (Fmt.pf f "@[#escape=%c@]@." escape; + convert ~buildkit ~escape f) + else convert ~buildkit ~escape f) (None, t) diff --git a/lib_spec/docker.mli b/lib_spec/docker.mli index 83fe91e4..0e23fbb1 100644 --- a/lib_spec/docker.mli +++ b/lib_spec/docker.mli @@ -1,5 +1,6 @@ -val dockerfile_of_spec : buildkit:bool -> Spec.t -> string -(** [dockerfile_of_spec x] produces a Dockerfile that aims to be equivalent to [x]. +val dockerfile_of_spec : buildkit:bool -> escape:char -> Spec.t -> string +(** [dockerfile_of_spec ~buildkit ~escape x] produces a Dockerfile + that aims to be equivalent to [x]. However, note that: diff --git a/main.ml b/main.ml index 16383138..f73e1d8d 100644 --- a/main.ml +++ b/main.ml @@ -83,10 +83,10 @@ let delete () store conf id = Builder.delete builder id ~log:(fun id -> Fmt.pr "Removing %s@." id) end -let dockerfile () buildkit spec = +let dockerfile () buildkit escape spec = Sexplib.Sexp.load_sexp spec |> Obuilder_spec.t_of_sexp - |> Obuilder_spec.Docker.dockerfile_of_spec ~buildkit + |> Obuilder_spec.Docker.dockerfile_of_spec ~buildkit ~escape |> print_endline open Cmdliner @@ -178,9 +178,16 @@ let buildkit = ~doc:"Output extended BuildKit syntax" ["buildkit"] +let escape = + Arg.value @@ + Arg.opt Arg.char (if Sys.unix then '\\' else '`') @@ + Arg.info + ~doc:"Dockerfile escape character. The default is '\\' on UNIX and '`' on Windows." + ["escape"] + let dockerfile = let doc = "Convert a spec to Dockerfile format" in - Term.(const dockerfile $ setup_log $ buildkit $ spec_file), + Term.(const dockerfile $ setup_log $ buildkit $ escape $ spec_file), Term.info "dockerfile" ~doc let healthcheck = diff --git a/obuilder.opam b/obuilder.opam index 3b0a19ee..cdfbff1f 100644 --- a/obuilder.opam +++ b/obuilder.opam @@ -14,7 +14,7 @@ depends: [ "astring" "fmt" {>= "0.8.9"} "logs" - "cmdliner" + "cmdliner" {> "1.0.4"} "tar" {>= "2.0"} "tar-unix" {>= "2.0"} "yojson" @@ -45,6 +45,7 @@ build: [ ] dev-repo: "git+https://github.com/ocurrent/obuilder.git" pin-depends: [ + ["cmdliner.1.0.5" "git+https://github.com/dbuenzli/cmdliner.git#db4d02a9eb47b5c43127a67cb121004b03ea3719"] ["lwt.5.4.3" "git+https://github.com/MisterDA/lwt.git#e703de884a798b0c8c45c6f792691cdaf578d35b"] ["sha.1.15" "git+https://github.com/djs55/ocaml-sha.git#8b77ab306fc3a5e94219580d6a2c92bc11745d60"] ["tar.2.0.0" "git+https://github.com/MisterDA/ocaml-tar.git#47d20548d39337009b6d73309eb2aab346494e76"] diff --git a/obuilder.opam.template b/obuilder.opam.template index 0a1d7787..b7343c92 100644 --- a/obuilder.opam.template +++ b/obuilder.opam.template @@ -1,4 +1,5 @@ pin-depends: [ + ["cmdliner.1.0.5" "git+https://github.com/dbuenzli/cmdliner.git#db4d02a9eb47b5c43127a67cb121004b03ea3719"] ["lwt.5.4.3" "git+https://github.com/MisterDA/lwt.git#e703de884a798b0c8c45c6f792691cdaf578d35b"] ["sha.1.15" "git+https://github.com/djs55/ocaml-sha.git#8b77ab306fc3a5e94219580d6a2c92bc11745d60"] ["tar.2.0.0" "git+https://github.com/MisterDA/ocaml-tar.git#47d20548d39337009b6d73309eb2aab346494e76"] diff --git a/test/test.ml b/test/test.ml index f369ac06..e79d0f9f 100644 --- a/test/test.ml +++ b/test/test.ml @@ -460,10 +460,10 @@ let test_sexp () = (user (uid 1) (gid 2)) )|} -let test_docker () = +let test_docker_unix () = let test ~buildkit name expect sexp = let spec = Spec.t_of_sexp (Sexplib.Sexp.of_string sexp) in - let got = Obuilder_spec.Docker.dockerfile_of_spec ~buildkit spec in + let got = Obuilder_spec.Docker.dockerfile_of_spec ~buildkit ~escape:'\\' spec in let expect = remove_indent expect in Alcotest.(check string) name expect got in @@ -555,6 +555,105 @@ let test_docker () = (shell "command1")) ) |} +let test_docker_windows () = + let test ~buildkit name expect sexp = + let spec = Spec.t_of_sexp (Sexplib.Sexp.of_string sexp) in + let got = Obuilder_spec.Docker.dockerfile_of_spec ~buildkit ~escape:'`' spec in + let expect = remove_indent expect in + Alcotest.(check string) name expect got + in + test ~buildkit:false "Dockerfile" + {| #escape=` + FROM base + # A test comment + WORKDIR C:/src + RUN command1 + SHELL [ "C:/Windows/System32/cmd.exe", "/c" ] + RUN command2 && ` + command3 + COPY a b c + COPY a b c + ENV DEBUG="1" + USER Zaphod + COPY a b c + |} {| + ((from base) + (comment "A test comment") + (workdir C:/src) + (run (shell "command1")) + (shell C:/Windows/System32/cmd.exe /c) + (run + (cache (a (target /data)) + (b (target /srv))) + (shell "command2 && + command3")) + (copy (src a b) (dst c)) + (copy (src a b) (dst c) (exclude .git _build)) + (env DEBUG 1) + (user (name Zaphod)) + (copy (src a b) (dst c)) + ) |}; + test ~buildkit:true "BuildKit" + {| #escape=` + FROM base + # A test comment + WORKDIR /src + RUN command1 + SHELL [ "C:/Windows/System32/cmd.exe", "/c" ] + RUN --mount=type=cache,id=a,target=/data,uid=0 --mount=type=cache,id=b,target=/srv,uid=0 command2 + COPY a b c + COPY a b c + ENV DEBUG="1" + USER Zaphod + COPY a b c + |} {| + ((from base) + (comment "A test comment") + (workdir /src) + (run (shell "command1")) + (shell C:/Windows/System32/cmd.exe /c) + (run + (cache (a (target /data)) + (b (target /srv))) + (shell "command2")) + (copy (src a b) (dst c)) + (copy (src a b) (dst c) (exclude .git _build)) + (env DEBUG 1) + (user (name Zaphod)) + (copy (src a b) (dst c)) + ) |}; + test ~buildkit:false "Multi-stage" + {| #escape=` + FROM base as tools + RUN make tools + + FROM base + COPY --from=tools binary /usr/local/bin/ + |} {| + ((build tools + ((from base) + (run (shell "make tools")))) + (from base) + (copy (from (build tools)) (src binary) (dst /usr/local/bin/)) + ) |}; + test ~buildkit:true "Secrets" + {| #escape=` + FROM base as tools + RUN make tools + + FROM base + RUN --mount=type=secret,id=a,target=/secrets/a,uid=0 --mount=type=secret,id=b,target=/secrets/b,uid=0 command1 + |} {| + ((build tools + ((from base) + (run (shell "make tools")))) + (from base) + (run + (secrets (a (target /secrets/a)) + (b (target /secrets/b))) + (shell "command1")) + ) |} + let manifest = Alcotest.result (Alcotest.testable @@ -696,7 +795,8 @@ let main_unix () = "spec", [ test_case_sync "Sexp" `Quick test_sexp; test_case_sync "Cache ID" `Quick test_cache_id; - test_case_sync "Docker" `Quick test_docker; + test_case_sync "Docker UNIX" `Quick test_docker_unix; + test_case_sync "Docker Windows" `Quick test_docker_windows; ]; "build", [ test_case "Simple" `Quick test_simple;