Merge pull request #20 from infosiftr/oci-import

Add initial `Builder: oci-import` support

.test/builds.json

@@ -2716,5 +2716,61 @@
         }
       }
     }
+  },
+  "93476ae64659d71f4ee7fac781d6d1890df8926682e2fa6bd647a246b33ad9bf": {
+    "buildId": "93476ae64659d71f4ee7fac781d6d1890df8926682e2fa6bd647a246b33ad9bf",
+    "build": {
+      "img": "oisupport/staging-amd64:93476ae64659d71f4ee7fac781d6d1890df8926682e2fa6bd647a246b33ad9bf",
+      "resolved": null,
+      "sourceId": "a7e4b56dd1e5dde4c9d988092cb8639987559f13c5c92210664a4ffb880f222b",
+      "arch": "amd64",
+      "parents": {},
+      "resolvedParents": {}
+    },
+    "source": {
+      "sourceId": "a7e4b56dd1e5dde4c9d988092cb8639987559f13c5c92210664a4ffb880f222b",
+      "reproducibleGitChecksum": "0ea17ff707666270e1fcb3edd76545906c68714d658ccde88c22d1606b2d7e4f",
+      "allTags": [
+        "ubuntu:22.04",
+        "ubuntu:jammy-20240111",
+        "ubuntu:jammy",
+        "ubuntu:latest"
+      ],
+      "entry": {
+        "GitRepo": "https://git.launchpad.net/cloud-images/+oci/ubuntu-base",
+        "GitFetch": "refs/tags/dist-jammy-amd64-20240111-e6e3490a",
+        "GitCommit": "e6e3490ad3f524ccaa072edafe525f8ca8ac5490",
+        "Directory": "oci",
+        "File": "index.json",
+        "Builder": "oci-import",
+        "SOURCE_DATE_EPOCH": 1704931200
+      },
+      "arches": {
+        "amd64": {
+          "tags": [
+            "ubuntu:22.04",
+            "ubuntu:jammy-20240111",
+            "ubuntu:jammy",
+            "ubuntu:latest"
+          ],
+          "archTags": [],
+          "froms": [
+            "scratch"
+          ],
+          "lastStageFrom": "scratch",
+          "platformString": "linux/amd64",
+          "platform": {
+            "architecture": "amd64",
+            "os": "linux"
+          },
+          "parents": {
+            "scratch": {
+              "sourceId": null,
+              "pin": null
+            }
+          }
+        }
+      }
+    }
   }
 }

.test/example-commands.sh

@@ -79,3 +79,54 @@ SOURCE_DATE_EPOCH=1700741054 \
 # <push>
 docker push 'oisupport/staging-windows-amd64:9b405cfa5b88ba65121aabdb95ae90fd2e1fee7582174de82ae861613ae3072e'
 # </push>
+
+# ubuntu:22.04 [amd64]
+# <pull>
+
+# </pull>
+# <build>
+export BASHBREW_CACHE="${BASHBREW_CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/bashbrew}"
+gitCache="$BASHBREW_CACHE/git"
+git init --bare "$gitCache"
+_git() { git -C "$gitCache" "$@"; }
+_git config gc.auto 0
+_commit() { _git rev-parse 'e6e3490ad3f524ccaa072edafe525f8ca8ac5490^{commit}'; }
+if ! _commit &> /dev/null; then _git fetch 'https://git.launchpad.net/cloud-images/+oci/ubuntu-base' 'e6e3490ad3f524ccaa072edafe525f8ca8ac5490:' || _git fetch 'refs/tags/dist-jammy-amd64-20240111-e6e3490a:'; fi
+_commit
+mkdir temp
+_git archive --format=tar 'e6e3490ad3f524ccaa072edafe525f8ca8ac5490:oci/' | tar -xvC temp
+jq -s '
+	if length != 1 then
+		error("unexpected '\''oci-layout'\'' document count: " + length)
+	else .[0] end
+	| if .imageLayoutVersion != "1.0.0" then
+		error("unsupported imageLayoutVersion: " + .imageLayoutVersion)
+	else . end
+' temp/oci-layout > /dev/null
+jq -s '
+	if length != 1 then
+		error("unexpected '\''index.json'\'' document count: " + length)
+	else .[0] end
+	| if .schemaVersion != 2 then
+		error("unsupported schemaVersion: " + .schemaVersion)
+	else . end
+	| if .manifests | length != 1 then
+		error("expected only one manifests entry, not " + (.manifests | length))
+	else . end
+	| .manifests[0] |= (
+		if .mediaType != "application/vnd.oci.image.manifest.v1+json" then
+			error("unsupported descriptor mediaType: " + .mediaType)
+		else . end
+		| if .size < 0 then
+			error("invalid descriptor size: " + .size)
+		else . end
+		| del(.annotations, .urls)
+		| .annotations = {"org.opencontainers.image.source":"https://git.launchpad.net/cloud-images/+oci/ubuntu-base","org.opencontainers.image.revision":"e6e3490ad3f524ccaa072edafe525f8ca8ac5490","org.opencontainers.image.created":"2024-01-11T00:00:00Z","org.opencontainers.image.version":"22.04","org.opencontainers.image.url":"https://hub.docker.com/_/ubuntu","org.opencontainers.image.base.name":"scratch"}
+	)
+' temp/index.json > temp/index.json.new
+mv temp/index.json.new temp/index.json
+# </build>
+# <push>
+crane push --index temp 'oisupport/staging-amd64:93476ae64659d71f4ee7fac781d6d1890df8926682e2fa6bd647a246b33ad9bf'
+rm -rf temp
+# </push>

.test/library/ubuntu

@@ -0,0 +1,15 @@
+# https://github.com/docker-library/official-images/blob/fe9c059402181390eac083cbdd7229b5d123236e/library/ubuntu but intentionally slimmed down (just "latest" on one architecture, no email addresses)
+
+Maintainers: Tomáš Virtus (@woky), Cristóvão Cordeiro (@cjdcordeiro)
+GitRepo: https://git.launchpad.net/cloud-images/+oci/ubuntu-base
+GitCommit: fa42be9027eccb928a1f0f43d95ffd9a45d36737
+Builder: oci-import
+File: index.json
+
+# 20240111 (jammy)
+Tags: 22.04, jammy-20240111, jammy, latest
+Architectures: amd64
+Directory: oci
+# https://git.launchpad.net/cloud-images/+oci/ubuntu-base/tree/?h=dist-jammy-amd64-20240111-e6e3490a
+amd64-GitFetch: refs/tags/dist-jammy-amd64-20240111-e6e3490a
+amd64-GitCommit: e6e3490ad3f524ccaa072edafe525f8ca8ac5490

.test/sources.json

@@ -757,5 +757,50 @@
         }
       }
     }
+  },
+  "a7e4b56dd1e5dde4c9d988092cb8639987559f13c5c92210664a4ffb880f222b": {
+    "sourceId": "a7e4b56dd1e5dde4c9d988092cb8639987559f13c5c92210664a4ffb880f222b",
+    "reproducibleGitChecksum": "0ea17ff707666270e1fcb3edd76545906c68714d658ccde88c22d1606b2d7e4f",
+    "allTags": [
+      "ubuntu:22.04",
+      "ubuntu:jammy-20240111",
+      "ubuntu:jammy",
+      "ubuntu:latest"
+    ],
+    "entry": {
+      "GitRepo": "https://git.launchpad.net/cloud-images/+oci/ubuntu-base",
+      "GitFetch": "refs/tags/dist-jammy-amd64-20240111-e6e3490a",
+      "GitCommit": "e6e3490ad3f524ccaa072edafe525f8ca8ac5490",
+      "Directory": "oci",
+      "File": "index.json",
+      "Builder": "oci-import",
+      "SOURCE_DATE_EPOCH": 1704931200
+    },
+    "arches": {
+      "amd64": {
+        "tags": [
+          "ubuntu:22.04",
+          "ubuntu:jammy-20240111",
+          "ubuntu:jammy",
+          "ubuntu:latest"
+        ],
+        "archTags": [],
+        "froms": [
+          "scratch"
+        ],
+        "lastStageFrom": "scratch",
+        "platformString": "linux/amd64",
+        "platform": {
+          "architecture": "amd64",
+          "os": "linux"
+        },
+        "parents": {
+          "scratch": {
+            "sourceId": null,
+            "pin": null
+          }
+        }
+      }
+    }
   }
 }

.test/test.sh

@@ -8,7 +8,7 @@ dir="$(dirname "$BASH_SOURCE")"
 dir="$(readlink -ve "$dir")"
 export BASHBREW_LIBRARY="$dir/library"
 
-set -- docker:cli docker:dind docker:windowsservercore notary # a little bit of Windows, a little bit of Linux, a little bit of multi-stage
+set -- docker:cli docker:dind docker:windowsservercore notary ubuntu:latest # a little bit of Windows, a little bit of Linux, a little bit of multi-stage, a little bit of oci-import
 # (see "library/" and ".external-pins/" for where these come from / are hard-coded for consistent testing purposes)
 # NOTE: we are explicitly *not* pinning "golang:1.19-alpine3.16" so that this also tests unpinned parent behavior (that image is deprecated so should stay unchanging)
 

meta.jq

@@ -267,10 +267,76 @@ def build_command:
 		] | join("\n")
 	elif $builder == "oci-import" then
 		[
-			"git init temp", # TODO figure out a good, safe place to temporary "git init"??
-			@sh "git -C temp fetch \(.source.entry.GitRepo) \(.source.entry.GitCommit): || git -C temp fetch \(.source.entry.GitRepo) \(.source.entry.GitFetch):",
-			@sh "git -C temp checkout -q \(.source.entry.GitCommit)",
-			# TODO something clever, especially to deal with "index.json" vs not-"index.json" (possibly using "jq" to either synthesize/normalize to what we actually need it to be for "crane push temp/dir \(.build.img)")
+			# initialize "~/.cache/bashbrew/git"
+			#"gitCache=\"$(bashbrew cat --format '{{ gitCache }}' <(echo 'Maintainers: empty hack (@example)'))\"",
+			# https://github.com/docker-library/bashbrew/blob/5152c0df682515cbe7ac62b68bcea4278856429f/cmd/bashbrew/git.go#L52-L80
+			"export BASHBREW_CACHE=\"${BASHBREW_CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/bashbrew}\"",
+			"gitCache=\"$BASHBREW_CACHE/git\"",
+			"git init --bare \"$gitCache\"",
+			"_git() { git -C \"$gitCache\" \"$@\"; }",
+			"_git config gc.auto 0",
+			# "bashbrew fetch" but in Bash (because we have bashbrew, but not the library file -- we could synthesize a library file instead, but six of one half a dozen of another)
+			@sh "_commit() { _git rev-parse \(.source.entry.GitCommit + "^{commit}"); }",
+			@sh "if ! _commit &> /dev/null; then _git fetch \(.source.entry.GitRepo) \(.source.entry.GitCommit + ":") || _git fetch \(.source.entry.GitFetch + ":"); fi",
+			"_commit",
+
+			# TODO figure out a good, safe place to store our temporary build/push directory (maybe this is fine? we do it for buildx build too)
+			"mkdir temp",
+			# https://github.com/docker-library/bashbrew/blob/5152c0df682515cbe7ac62b68bcea4278856429f/cmd/bashbrew/git.go#L140-L147 (TODO "bashbrew context" ?)
+			@sh "_git archive --format=tar \(.source.entry.GitCommit + ":" + (.source.entry.Directory | if . == "." then "" else . + "/" end)) | tar -xvC temp",
+
+			# validate oci-layout file (https://github.com/docker-library/bashbrew/blob/4e0ea8d8aba49d54daf22bd8415fabba65dc83ee/cmd/bashbrew/oci-builder.go#L104-L112)
+			@sh "jq -s \("
+				if length != 1 then
+					error(\"unexpected 'oci-layout' document count: \" + length)
+				else .[0] end
+				| if .imageLayoutVersion != \"1.0.0\" then
+					error(\"unsupported imageLayoutVersion: \" + .imageLayoutVersion)
+				else . end
+			" | unindent_and_decomment_jq(3)) temp/oci-layout > /dev/null",
+
+			# https://github.com/docker-library/bashbrew/blob/4e0ea8d8aba49d54daf22bd8415fabba65dc83ee/cmd/bashbrew/oci-builder.go#L116
+			if .source.entry.File != "index.json" then
+				@sh "jq -s \("{ schemaVersion: 2, manifests: . }") \("./" + .source.entry.File) > temp/index.json"
+			else empty end,
+
+			@sh "jq -s \("
+				if length != 1 then
+					error(\"unexpected 'index.json' document count: \" + length)
+				else .[0] end
+
+				# https://github.com/docker-library/bashbrew/blob/4e0ea8d8aba49d54daf22bd8415fabba65dc83ee/cmd/bashbrew/oci-builder.go#L117-L127
+				| if .schemaVersion != 2 then
+					error(\"unsupported schemaVersion: \" + .schemaVersion)
+				else . end
+				# TODO check .mediaType ? (technically optional, but does not have to be *and* shouldn't be); https://github.com/moby/buildkit/issues/4595
+				| if .manifests | length != 1 then
+					error(\"expected only one manifests entry, not \" + (.manifests | length))
+				else . end
+
+				| .manifests[0] |= (
+					# https://github.com/docker-library/bashbrew/blob/4e0ea8d8aba49d54daf22bd8415fabba65dc83ee/cmd/bashbrew/oci-builder.go#L135-L144
+					if .mediaType != \"application/vnd.oci.image.manifest.v1+json\" then
+						error(\"unsupported descriptor mediaType: \" + .mediaType)
+					else . end
+					# TODO validate .digest somehow (`crane validate`? see below) - would also be good to validate all descriptors recursively (not sure if `crane push` does that)
+					| if .size < 0 then
+						error(\"invalid descriptor size: \" + .size)
+					else . end
+
+					# purge maintainer-provided URLs / annotations (https://github.com/docker-library/bashbrew/blob/4e0ea8d8aba49d54daf22bd8415fabba65dc83ee/cmd/bashbrew/oci-builder.go#L146-L147)
+					| del(.annotations, .urls)
+
+					# inject our annotations
+					| .annotations = \(build_annotations(.source.entry.GitRepo) | @json)
+				)
+			" | unindent_and_decomment_jq(3)) temp/index.json > temp/index.json.new",
+			"mv temp/index.json.new temp/index.json",
+
+			# TODO consider / check what "crane validate" does and if it would be appropriate here
+
+			# TODO generate SBOM? ... somehow
+
 			empty
 		] | join("\n")
 	else
@@ -291,7 +357,11 @@ def push_command:
 			empty
 		] | join("\n")
 	elif $builder == "oci-import" then
-		"TODO"
+		[
+			@sh "crane push --index temp \(.build.img)",
+			"rm -rf temp",
+			empty
+		] | join("\n")
 	else
 		error("unknown/unimplemented Builder: \($builder)")
 	end

sources.sh

@@ -65,7 +65,12 @@ bashbrew cat --build-order --format '
 							"tags": {{ $.Tags namespace false . | json }},
 							"archTags": {{ if $archNs -}} {{ $.Tags $archNs false . | json }} {{- else -}} [] {{- end }},
 							"froms": {{ $.ArchDockerFroms $a . | json }},
-							"lastStageFrom": {{ $.ArchLastStageFrom $a . | json }},
+							"lastStageFrom": {{ if eq $builder "oci-import" -}}
+								{{- /* TODO remove this special case: https://github.com/docker-library/bashbrew/pull/92 */ -}}
+								"scratch"
+							{{- else -}}
+								{{ $.ArchLastStageFrom $a . | json }}
+							{{- end }},
 							"platformString": {{ (ociPlatform $a).String | json }},
 							"platform": {{ ociPlatform $a | json }},
 							"parents": { }