Updates to obuilder

example.spec

@@ -23,8 +23,8 @@
  && git reset -q --hard $OPAM_HASH \
  && git --no-pager log --no-decorate -n1 --oneline \
  && opam update -u"))
- ; Copy just the opam file first (helps caching)
- (copy (src obuilder-spec.opam obuilder.opam) (dst ./))
+ ; Copy just the opam file first (helps caching) + container image whilst vendored
+ (copy (src obuilder-spec.opam obuilder.opam vendor) (dst ./))
  (run (shell "opam pin add -yn ."))
  ; Install OS package dependencies
  (run

lib/container_image_extract.ml

@@ -0,0 +1,114 @@
+let null_auth ?ip:_ ~host:_ _ =
+ Ok None (* Warning: use a real authenticator in your code! *)
+
+let https ~authenticator =
+ let tls_config = Tls.Config.client ~authenticator () in
+ fun uri raw ->
+ let host =
+ Uri.host uri
+ |> Option.map (fun x -> Domain_name.(host_exn (of_string_exn x)))
+ in
+ Tls_eio.client_of_flow ?host tls_config raw
+
+let cache fs =
+ let xdg = Xdg.create ~env:Sys.getenv_opt () in
+ let root = Eio.Path.(fs / Xdg.cache_dir xdg / "image") in
+ let cache = Container_image.Cache.v root in
+ Container_image.Cache.init cache;
+ cache
+
+let fetch ~net ~domain_mgr ?platform cache image no_progress =
+ let client =
+ Cohttp_eio.Client.make
+ ~https:(Some (https ~authenticator:null_auth))
+ net
+ in
+ let show_progress = not no_progress in
+ Container_image.fetch ~show_progress ~client ~cache ~domain_mgr ?platform image
+
+module Make (E : sig
+ val fs : Eio.Fs.dir_ty Eio.Path.t
+ val net : [ `Generic ] Eio.Net.ty Eio.Net.t 
+ val domain_mgr : Eio.Domain_manager.ty Eio.Domain_manager.t
+
+ val progress : bool
+end) = struct
+
+ let get_environment_variables_from_manifest ~cache v i =
+ let module Ci = Container_image in
+ let open Container_image_spec in
+ let of_oci_manifest (docker_manifest : Manifest.OCI.t) =
+ let config = Manifest.OCI.config docker_manifest in
+ let config = 
+ Ci.Cache.Blob.get_string cache (Descriptor.digest config)
+ |> Config.of_string ~media_type:Media_type.(Docker Docker.Image_config)
+ |> Result.get_ok
+ in
+ Config.env config
+ in
+ let of_docker_manifest (docker_manifest : Manifest.Docker.t) =
+ let config = Manifest.Docker.config docker_manifest in
+ let config = 
+ Ci.Cache.Blob.get_string cache (Descriptor.digest config)
+ |> Config.of_string ~media_type:Media_type.(Docker Docker.Image_config)
+ |> Result.get_ok
+ in
+ Config.env config
+ in
+ match (v : Manifest.t) with
+ | `Docker_manifest_list _ -> (
+ match Container_image.Util.guess_manifest v with
+ | None -> failwith "No known manifest for this architecture"
+ | Some descriptor ->
+ let digest = Descriptor.digest descriptor in
+ let manifest = 
+ let img = Ci.Image.v ~digest (Ci.Image.repository i) in
+ Ci.Cache.Manifest.get cache img
+ |> function `Docker_manifest m -> m | _ -> assert false
+ in
+ of_docker_manifest manifest
+ )
+ | `Docker_manifest m -> of_docker_manifest m
+ | `OCI_manifest oci -> of_oci_manifest oci
+ | `OCI_index _ ->
+ (* TODO: Which manifest? *)
+ match Container_image.Util.guess_manifest v with
+ | None -> failwith "No known manifest for this architecture"
+ | Some descriptor ->
+ let digest = Descriptor.digest descriptor in
+ let manifest = 
+ let img = Ci.Image.v ~digest (Ci.Image.repository i) in
+ Ci.Cache.Manifest.get cache img
+ |> function `OCI_manifest m -> m | _ -> assert false
+ in
+ of_oci_manifest manifest
+
+ let fetch ~log:_ ~rootfs base =
+ let module Ci = Container_image in
+ let open Container_image_spec in
+ Logs.info (fun f -> f "Fetching %s" base);
+ let cache = cache E.fs in
+ match Container_image.Image.of_string base with
+ | Error (`Msg m) -> failwith m
+ | Ok img -> 
+ fetch ~net:E.net ~domain_mgr:E.domain_mgr cache img (not E.progress);
+ let root = Eio.Path.(E.fs / rootfs) in
+ let manifest = Container_image.Cache.Manifest.get cache img in
+ match Container_image.Util.guess_manifest manifest with
+ | None -> failwith "No known manifest"
+ | Some descriptor ->
+ let digest = Descriptor.digest descriptor in
+ let single_image = Ci.Image.with_digest digest img in 
+ Ci.checkout ~only_rootfs:true ~cache ~root single_image;
+ Lwt.return @@ get_environment_variables_from_manifest ~cache manifest img
+end
+
+let make_fetcher ?(progress=true) ~fs ~net domain_mgr =
+ let module T = struct
+ let fs = (fs :> Eio.Fs.dir_ty Eio.Path.t)
+ let net = (net :> [`Generic] Eio.Net.ty Eio.Net.t)
+ let domain_mgr = (domain_mgr :> Eio.Domain_manager.ty Eio.Domain_manager.t)
+ let progress = progress
+ end in
+ let module Fetcher = Make (T) in
+ (module Fetcher : S.FETCHER)

lib/container_image_extract.mli

@@ -8,3 +8,12 @@ module Make (_ : sig
 end) : sig
  include S.FETCHER
 end
+
+val make_fetcher :
+ ?progress:bool ->
+ fs:_ Eio.Path.t ->
+ net:_ Eio.Net.ty Eio.Net.t ->
+ _ Eio.Domain_manager.t ->
+ (module S.FETCHER)
+(** A function that returns a fetcher module *)
+

main.ml

@@ -18,13 +18,13 @@ let log tag msg =
  | `Output -> output_string stdout msg; flush stdout
 
 let create_builder env store_spec conf =
- let module T = struct
- let fs = Eio.Stdenv.fs env
- let net = (Eio.Stdenv.net env :> [`Generic] Eio.Net.ty Eio.Net.t)
- let domain_mgr = Eio.Stdenv.domain_mgr env
- let progress = true
- end in
- let module Fetcher = Obuilder.Container_image_extract.Make (T) in
+ let (module Fetcher) = 
+ Obuilder.Container_image_extract.make_fetcher
+ ~progress:true
+ ~fs:(Eio.Stdenv.fs env)
+ ~net:(Eio.Stdenv.net env)
+  (Eio.Stdenv.domain_mgr env)
+ in
  store_spec >>= fun (Store_spec.Store ((module Store), store)) ->
  let module Builder = Obuilder.Builder (Store) (Native_sandbox) (Fetcher) in
  Native_sandbox.create ~state_dir:(Store.state_dir store / "sandbox") conf >|= fun sandbox ->

obuilder.opam

@@ -63,5 +63,5 @@ build: [
 ]
 dev-repo: "git+https://github.com/ocurrent/obuilder.git"
 pin-depends:[
- [ "container-image.dev" "file:./vendor/container-image" ]
+ [ "container-image.dev" "file://./vendor/container-image" ]
 ]

obuilder.opam.template

@@ -1,3 +1,3 @@
 pin-depends:[
- [ "container-image.dev" "file:./vendor/container-image" ]
+ [ "container-image.dev" "file://./vendor/container-image" ]
 ]