Docker backend

.gitattributes

@@ -1,3 +1,4 @@
 *.cmd   text eol=crlf
 *.bash  text eol=lf
 *.sh    text eol=lf
+test/test.ml    text eol=lf

.github/workflows/main.yml

@@ -93,3 +93,27 @@ jobs:
           sudo wget https://github.com/opencontainers/runc/releases/download/$RUNC_VERSION/runc.amd64 -O /usr/local/bin/runc
 
       - run: ./.run-gha-tests.sh rsync
+
+  windows:
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - windows-latest
+        ocaml-compiler:
+          - 4.14.x
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Use OCaml ${{ matrix.ocaml-compiler }}
+        uses: ocaml/setup-ocaml@v2
+        with:
+          ocaml-compiler: ${{ matrix.ocaml-compiler }}
+
+      - run: opam install . --deps-only --with-test
+
+      - run: opam exec -- dune build

README.md

@@ -7,11 +7,14 @@
 
 OBuilder takes a build script (similar to a Dockerfile) and performs the steps in it in a sandboxed environment.
 
-After each step, OBuild uses the snapshot feature of the filesystem (ZFS or Btrfs) to store the state of the build. There is also an Rsync backend that copies the build state.
+After each step, OBuilder uses the snapshot feature of the filesystem (ZFS or
+Btrfs) to store the state of the build. There is also an Rsync backend that
+copies the build state. On Linux, it uses `runc` to sandbox the build steps, but
+any system that can run a command safely in a chroot could be used.
 Repeating a build will reuse the cached results where possible.
 
-OBuilder aims to be portable, although currently only Linux support is present.
-On Linux, it uses `runc` to sandbox the build steps, but any system that can run a command safely in a chroot could be used.
+OBuilder can also use Docker as a backend (fully replacing of `runc` and the
+snapshotting filesystem) on any system supported by Docker (Linux, Windows, …).
 
 OBuilder stores the log output of each build step.
 This is useful for CI, where you may still want to see the output even if the result was cached from some other build.
@@ -73,9 +76,14 @@ pass the `--fast-sync` option, which installs a seccomp filter that skips all
 sync syscalls. However, if you attempt to use this with an earlier version of
 runc then sync operations will instead fail with `EPERM`.
 
+### Windows
+
+The user running OBuilder must have access to `%PROGRAMDATA%\Docker\volumes`,
+because copying caches and maintaining internal tools is done directly on the host.
+
 ## The build specification language
 
-The spec files are loosly based on the [Dockerfile][] format.
+The spec files are loosely based on the [Dockerfile][] format.
 The main difference is that the format uses S-expressions rather than a custom format,
 which should make it easier to generate and consume it automatically.
 
@@ -104,9 +112,9 @@ The initial filesystem snapshot is `BASE`. `run` and `copy` operations create ne
 The initial context is supplied by the user (see [build.mli](lib/build.mli) for details).
 By default:
 - The environment is taken from the Docker configuration of `BASE`.
-- The user is `(uid 0) (gid 0)`.
-- The workdir is `/`.
-- The shell is `/bin/bash -c`.
+- The user is `(uid 0) (gid 0)` on Linux, `(name ContainerAdministrator)` on Windows.
+- The workdir is `/`, `C:/` on Windows.
+- The shell is `/bin/bash -c`, `C:\Windows\System32\cmd.exe /S /C` on Windows.
 
 ### Multi-stage builds
 
@@ -130,7 +138,6 @@ For example:
 
 At the moment, the `(build ...)` items must appear before the `(from ...)` line.
 
-
 ### workdir
 
 ```sexp
@@ -169,7 +176,6 @@ The command run will be this list of arguments followed by the single argument `
  (network NETWORK...)?
  (secrets SECRET...)?
  (shell COMMAND))
-
 ```
 
 Examples:
@@ -210,9 +216,9 @@ the image. Each `SECRET` entry is under the form `(ID (target PATH))`, where `ID
 `PATH` is the location of the mounted secret file within the container.
 The sandbox context API contains a `secrets` parameter to provide values to the runtime.
 If a requested secret isn't provided with a value, the runtime fails.
-With the command line interface `obuilder`, use the `--secret ID:PATH` option to provide the path of the file
-containing the secret for `ID`.
-When used with Docker, make sure to use the **buildkit** syntax, as only buildkit supports a `--secret` option.
+Use the `--secret ID:PATH` option to provide the path of the file containing the
+secret for `ID`.
+When used with Docker, make sure to use the **BuildKit** syntax, as only BuildKit supports a `--secret` option.
 (See https://docs.docker.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information)
 
 ### copy
@@ -261,13 +267,20 @@ Notes:
 
 - Both `SRC` and `DST` use `/` as the directory separator on all platforms.
 
-- The copy is currently done by running `tar` inside the container to receive the files.
-  Therefore, the filesystem must have a working `tar` binary.
+- The copy is currently done by running `tar` inside the container to receive
+  the files. Therefore, the filesystem must have a working `tar` binary. On
+  Windows when using the Docker backend, OBuilder provides a `tar` binary.
+
+- On Windows, copying from a build step image based on [Nano Server][nanoserver]
+  isn't supported.
+
+[nanoserver]: https://hub.docker.com/_/microsoft-windows-nanoserver
 
 ### user
 
 ```sexp
 (user (uid UID) (gid GID))
+(user (name NAME))      ; on Windows
 ```
 
 Example:

dune-project

@@ -23,13 +23,12 @@
   sexplib
   ppx_deriving
   ppx_sexp_conv
-  sha
+  (sha (>= 1.15.1))
   sqlite3
+  (crunch (and (>= 3.3.1) :build))
   (obuilder-spec (= :version))
-  (ocaml (>= 4.10.0))
-  (alcotest-lwt :with-test))
- (conflicts
-  (result (< "1.5"))))
+  (ocaml (>= 4.14.0))
+  (alcotest-lwt :with-test)))
 (package
  (name obuilder-spec)
  (synopsis "Build specification format")

example.spec

@@ -7,11 +7,11 @@
 ; The result can then be found in /tank/HASH/rootfs/ (where HASH is displayed at the end of the build).
 
 ((build dev
-	((from ocaml/opam@sha256:5b9de826b22c77a0654519d0959536f93a6ffd7020712a8b1c3437445e031e04)
+	((from ocaml/opam@sha256:00f4d3f38bbde3a7a28b1b4b8994eded60fb5ee78822082f425662b7f9463178)
 	 (workdir /src)
 	 (user (uid 1000) (gid 1000))                           ; Build as the "opam" user
 	 (run (shell "sudo chown opam /src"))
-	 (env OPAM_HASH "97da9a1b68b824a65a09e5f7d071fcf2da35bd1b") ; Fix the version of opam-repository we want
+	 (env OPAM_HASH "9adfaed58b31bc1be6e6086f4dda37e891793a7b") ; Fix the version of opam-repository we want
 	 (run
 	  (network host)
 	  (shell "sudo apt-get --allow-releaseinfo-change update"))

example.windows.spec

@@ -0,0 +1,64 @@
+; This script builds OBuilder itself using a snapshot of the
+; ocaml/opam:windows-mingw-ocaml-4.14 base image.
+;
+; Run it from the top-level of the OBuilder source tree, e.g.
+;
+;   root=../var
+;   dune exec -- obuilder build --docker-backend="$root" -f example.windows.spec .
+;
+; The result can then be found in the Docker image "obuilder-ROOTID-image-HASH"
+; (where HASH is displayed at the end of the build).
+; The logs can be found in "$root/logs/HASH.log".
+; ROOTID is computed as follows: $(realpath "$(root)" | sha256sum | cut -b -7)
+
+((build dev
+	((from ocaml/opam@sha256:63f5f8207ea61195988d9d49afcc4044bee3183645c58de6959f0864fabd9383)
+	 (workdir /src)
+	 (env OPAM_HASH "74176d75a60a6ec4d90d4178733b1e09f8becc6f") ; Fix the version of opam-repository-mingw we want
+	 (shell /cygwin64/bin/bash.exe --login -c)
+	 (run
+	  (network "nat")
+	  (shell
+	   "cd /home/opam/opam-repository \
+	    && (git cat-file -e $OPAM_HASH || git fetch origin opam2) \
+	    && git reset -q --hard $OPAM_HASH \
+	    && git --no-pager log --no-decorate -n1 --oneline \
+	    && rsync -ar --update --exclude='.git' ./ /cygdrive/c/opam/.opam/repo/default \
+	    && ocaml-env exec --64 -- opam update -u"))
+         ; opam update -u fails because of patch, so I'm overriding the repo with rsync
+	 (shell cmd /S /C)
+	 ; Copy just the opam file first (helps caching)
+	 (copy (src obuilder-spec.opam obuilder.opam) (dst ./))
+	 (run
+	  (network "nat")
+	  (cache (opam-archives (target /opam/.opam/download-cache)))
+	  (shell "ocaml-env exec --64 -- opam pin add -yn ."))
+	 ; Install OS package dependencies
+	 (run
+	  (network "nat")
+	  (cache (opam-archives (target /opam/.opam/download-cache)))
+	  (shell "ocaml-env exec --64 -- opam depext -yu obuilder"))
+	 ; Install OCaml dependencies
+	 (run
+	  (network "nat")
+	  (cache (opam-archives (target /opam/.opam/download-cache)))
+	  (shell "ocaml-env exec --64 -- opam install --deps-only -t obuilder-spec"))
+	 (run
+	  (network "nat")
+	  (cache (opam-archives (target /opam/.opam/download-cache)))
+	  (shell "ocaml-env exec --64 -- opam install --deps-only -t obuilder"))
+	 (copy                                                  ; Copy the rest of the source code
+	  (src .)
+	  (dst /src/)
+	  (exclude .git _build _opam duniverse))
+	 (run (shell "ocaml-env exec --64 -- dune build @install"))))    ; Build
+ ; Now generate a small runtime image with just the resulting binary:
+ (from mcr.microsoft.com/windows/servercore:ltsc2022)
+ (run (shell "mkdir C:\obuilder"))
+ (copy (from (build dev))
+       (src /cygwin64/usr/x86_64-w64-mingw32/sys-root/mingw/bin/libsqlite3-0.dll)
+       (dst /obuilder/libsqlite3-0.dll))
+ (copy (from (build dev))
+       (src /src/_build/default/main.exe)
+       (dst /obuilder/obuilder.exe))
+ (run (shell "/obuilder/obuilder.exe --help")))

lib/btrfs_store.ml

@@ -101,6 +101,8 @@ let check_kernel_version () =
   | _ ->
       Fmt.failwith "Could not parse output of 'uname -r' (%S)" kver
 
+let root t = t.root
+
 let create root =
   check_kernel_version () >>= fun () ->
   Os.ensure_dir (root / "result");
@@ -140,8 +142,13 @@ let build t ?base ~id fn =
 let result t id =
   let dir = Path.result t id in
   match Os.check_dir dir with
-  | `Present -> Some dir
-  | `Missing -> None
+  | `Present -> Lwt.return_some dir
+  | `Missing -> Lwt.return_none
+
+let log_file t id =
+  result t id >|= function
+  | Some dir -> dir / "log"
+  | None -> (Path.result_tmp t id) / "log"
 
 let get_cache t name =
   match Hashtbl.find_opt t.caches name with
@@ -165,8 +172,11 @@ let cache ~user t name : (string * (unit -> unit Lwt.t)) Lwt.t =
   (* Create writeable clone. *)
   let gen = cache.gen in
   Btrfs.subvolume_snapshot `RW ~src:snapshot tmp >>= fun () ->
-  let { Obuilder_spec.uid; gid } = user in
-  Os.sudo ["chown"; Printf.sprintf "%d:%d" uid gid; tmp] >>= fun () ->
+  begin match user with
+    | `Unix { Obuilder_spec.uid; gid } ->
+      Os.sudo ["chown"; Printf.sprintf "%d:%d" uid gid; tmp]
+    | `Windows _ -> assert false (* btrfs not supported on Windows*)
+  end >>= fun () ->
   let release () =
     Lwt_mutex.with_lock cache.lock @@ fun () ->
     begin