Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix conversion of Zstd images to non-OCI formats #2151

Merged
merged 4 commits into from
Oct 21, 2023
Merged

Fix conversion of Zstd images to non-OCI formats #2151

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f1a91cd
Rename prepareLayerDecryptEditsIfNecessary to layerEditsOfOCIOnlyFeat…
mtrmac Oct 20, 2023
bc36ea6
Fix a comment referring to a obsolete method
mtrmac Oct 20, 2023
2e05cfc
Split successfulOCI1Conversion from some existing tests
mtrmac Oct 20, 2023
6719f35
Fix conversion of Zstd images to non-OCI formats
mtrmac Oct 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions internal/image/common_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"path/filepath"
"testing"

compressiontypes "github.com/containers/image/v5/pkg/compression/types"
"github.com/containers/image/v5/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -40,3 +41,13 @@ func layerInfosWithCryptoOperation(input []types.BlobInfo, op types.LayerCrypto)
}
return res
}

// layerInfosWithCompressionEdits returns a copy of input where CompressionOperation and CompressionAlgorithm is set to op and algo
func layerInfosWithCompressionEdits(input []types.BlobInfo, op types.LayerCompression, algo *compressiontypes.Algorithm) []types.BlobInfo {
res := slices.Clone(input)
for i := range res {
res[i].CompressionOperation = op
res[i].CompressionAlgorithm = algo
}
return res
}
161 changes: 161 additions & 0 deletions internal/image/fixtures/oci1-all-media-types-config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
{
"architecture": "amd64",
"config": {
"Hostname": "383850eeb47b",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"ExposedPorts": {
"80/tcp": {}
},
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"PATH=/usr/local/apache2/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"HTTPD_PREFIX=/usr/local/apache2",
"HTTPD_VERSION=2.4.23",
"HTTPD_SHA1=5101be34ac4a509b245adb70a56690a84fcc4e7f",
"HTTPD_BZ2_URL=https://www.apache.org/dyn/closer.cgi?action=download\u0026filename=httpd/httpd-2.4.23.tar.bz2",
"HTTPD_ASC_URL=https://www.apache.org/dist/httpd/httpd-2.4.23.tar.bz2.asc"
],
"Cmd": [
"httpd-foreground"
],
"ArgsEscaped": true,
"Image": "sha256:4f83530449c67c1ed8fca72583c5b92fdf446010990028c362a381e55dd84afd",
"Volumes": null,
"WorkingDir": "/usr/local/apache2",
"Entrypoint": null,
"OnBuild": [],
"Labels": {}
},
"container": "8825acde1b009729807e4b70a65a89399dd8da8e53be9216b9aaabaff4339f69",
"container_config": {
"Hostname": "383850eeb47b",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"ExposedPorts": {
"80/tcp": {}
},
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"PATH=/usr/local/apache2/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"HTTPD_PREFIX=/usr/local/apache2",
"HTTPD_VERSION=2.4.23",
"HTTPD_SHA1=5101be34ac4a509b245adb70a56690a84fcc4e7f",
"HTTPD_BZ2_URL=https://www.apache.org/dyn/closer.cgi?action=download\u0026filename=httpd/httpd-2.4.23.tar.bz2",
"HTTPD_ASC_URL=https://www.apache.org/dist/httpd/httpd-2.4.23.tar.bz2.asc"
],
"Cmd": [
"/bin/sh",
"-c",
"#(nop) ",
"CMD [\"httpd-foreground\"]"
],
"ArgsEscaped": true,
"Image": "sha256:4f83530449c67c1ed8fca72583c5b92fdf446010990028c362a381e55dd84afd",
"Volumes": null,
"WorkingDir": "/usr/local/apache2",
"Entrypoint": null,
"OnBuild": [],
"Labels": {}
},
"created": "2016-09-23T23:20:45.78976459Z",
"docker_version": "1.12.1",
"history": [
{
"created": "2016-09-23T18:08:50.537223822Z",
"created_by": "/bin/sh -c #(nop) ADD file:c6c23585ab140b0b320d4e99bc1b0eb544c9e96c24d90fec5e069a6d57d335ca in / "
},
{
"created": "2016-09-23T18:08:51.133779867Z",
"created_by": "/bin/sh -c #(nop) CMD [\"/bin/bash\"]",
"empty_layer": true
},
{
"created": "2016-09-23T19:16:40.725768956Z",
"created_by": "/bin/sh -c #(nop) ENV HTTPD_PREFIX=/usr/local/apache2",
"empty_layer": true
},
{
"created": "2016-09-23T19:16:41.037788416Z",
"created_by": "/bin/sh -c #(nop) ENV PATH=/usr/local/apache2/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"empty_layer": true
},
{
"created": "2016-09-23T19:16:41.990121202Z",
"created_by": "/bin/sh -c mkdir -p \"$HTTPD_PREFIX\" \t\u0026\u0026 chown www-data:www-data \"$HTTPD_PREFIX\""
},
{
"created": "2016-09-23T19:16:42.339911155Z",
"created_by": "/bin/sh -c #(nop) WORKDIR /usr/local/apache2",
"empty_layer": true
},
{
"created": "2016-09-23T19:16:54.948461741Z",
"created_by": "/bin/sh -c apt-get update \t\u0026\u0026 apt-get install -y --no-install-recommends \t\tlibapr1 \t\tlibaprutil1 \t\tlibaprutil1-ldap \t\tlibapr1-dev \t\tlibaprutil1-dev \t\tlibpcre++0 \t\tlibssl1.0.0 \t\u0026\u0026 rm -r /var/lib/apt/lists/*"
},
{
"created": "2016-09-23T19:16:55.321573403Z",
"created_by": "/bin/sh -c #(nop) ENV HTTPD_VERSION=2.4.23",
"empty_layer": true
},
{
"created": "2016-09-23T19:16:55.629947307Z",
"created_by": "/bin/sh -c #(nop) ENV HTTPD_SHA1=5101be34ac4a509b245adb70a56690a84fcc4e7f",
"empty_layer": true
},
{
"created": "2016-09-23T23:19:03.705796801Z",
"created_by": "/bin/sh -c #(nop) ENV HTTPD_BZ2_URL=https://www.apache.org/dyn/closer.cgi?action=download\u0026filename=httpd/httpd-2.4.23.tar.bz2",
"empty_layer": true
},
{
"created": "2016-09-23T23:19:04.009782822Z",
"created_by": "/bin/sh -c #(nop) ENV HTTPD_ASC_URL=https://www.apache.org/dist/httpd/httpd-2.4.23.tar.bz2.asc",
"empty_layer": true
},
{
"created": "2016-09-23T23:20:44.585743332Z",
"created_by": "/bin/sh -c set -x \t\u0026\u0026 buildDeps=' \t\tbzip2 \t\tca-certificates \t\tgcc \t\tlibpcre++-dev \t\tlibssl-dev \t\tmake \t\twget \t' \t\u0026\u0026 apt-get update \t\u0026\u0026 apt-get install -y --no-install-recommends $buildDeps \t\u0026\u0026 rm -r /var/lib/apt/lists/* \t\t\u0026\u0026 wget -O httpd.tar.bz2 \"$HTTPD_BZ2_URL\" \t\u0026\u0026 echo \"$HTTPD_SHA1 *httpd.tar.bz2\" | sha1sum -c - \t\u0026\u0026 wget -O httpd.tar.bz2.asc \"$HTTPD_ASC_URL\" \t\u0026\u0026 export GNUPGHOME=\"$(mktemp -d)\" \t\u0026\u0026 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys A93D62ECC3C8EA12DB220EC934EA76E6791485A8 \t\u0026\u0026 gpg --batch --verify httpd.tar.bz2.asc httpd.tar.bz2 \t\u0026\u0026 rm -r \"$GNUPGHOME\" httpd.tar.bz2.asc \t\t\u0026\u0026 mkdir -p src \t\u0026\u0026 tar -xvf httpd.tar.bz2 -C src --strip-components=1 \t\u0026\u0026 rm httpd.tar.bz2 \t\u0026\u0026 cd src \t\t\u0026\u0026 ./configure \t\t--prefix=\"$HTTPD_PREFIX\" \t\t--enable-mods-shared=reallyall \t\u0026\u0026 make -j\"$(nproc)\" \t\u0026\u0026 make install \t\t\u0026\u0026 cd .. \t\u0026\u0026 rm -r src \t\t\u0026\u0026 sed -ri \t\t-e 's!^(\\s*CustomLog)\\s+\\S+!\\1 /proc/self/fd/1!g' \t\t-e 's!^(\\s*ErrorLog)\\s+\\S+!\\1 /proc/self/fd/2!g' \t\t\"$HTTPD_PREFIX/conf/httpd.conf\" \t\t\u0026\u0026 apt-get purge -y --auto-remove $buildDeps"
},
{
"created": "2016-09-23T23:20:45.127455562Z",
"created_by": "/bin/sh -c #(nop) COPY file:761e313354b918b6cd7ea99975a4f6b53ff5381ba689bab2984aec4dab597215 in /usr/local/bin/ "
},
{
"created": "2016-09-23T23:20:45.453934921Z",
"created_by": "/bin/sh -c #(nop) EXPOSE 80/tcp",
"empty_layer": true
},
{
"created": "2016-09-23T23:20:45.78976459Z",
"created_by": "/bin/sh -c #(nop) CMD [\"httpd-foreground\"]",
"empty_layer": true
},
{
"created": "2023-10-01T02:03:04.56789764Z",
"created_by": "/bin/sh echo something > last"
}
],
"os": "linux",
"rootfs": {
"type": "layers",
"diff_ids": [
"sha256:142a601d97936307e75220c35dde0348971a9584c21e7cb42e1f7004005432ab",
"sha256:90fcc66ad3be9f1757f954b750deb37032f208428aa12599fcb02182b9065a9c",
"sha256:5a8624bb7e76d1e6829f9c64c43185e02bc07f97a2189eb048609a8914e72c56",
"sha256:d349ff6b3afc6a2800054768c82bfbf4289c9aa5da55c1290f802943dcd4d1e9",
"sha256:8c064bb1f60e84fa8cc6079b6d2e76e0423389fd6aeb7e497dfdae5e05b2b25b",
"sha256:1111111111111111111111111111111111111111111111111111111111111111"
]
}
}
161 changes: 161 additions & 0 deletions internal/image/fixtures/oci1-all-media-types-to-schema2-config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
{
"architecture": "amd64",
"config": {
"Hostname": "383850eeb47b",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"ExposedPorts": {
"80/tcp": {}
},
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"PATH=/usr/local/apache2/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"HTTPD_PREFIX=/usr/local/apache2",
"HTTPD_VERSION=2.4.23",
"HTTPD_SHA1=5101be34ac4a509b245adb70a56690a84fcc4e7f",
"HTTPD_BZ2_URL=https://www.apache.org/dyn/closer.cgi?action=download\u0026filename=httpd/httpd-2.4.23.tar.bz2",
"HTTPD_ASC_URL=https://www.apache.org/dist/httpd/httpd-2.4.23.tar.bz2.asc"
],
"Cmd": [
"httpd-foreground"
],
"ArgsEscaped": true,
"Image": "sha256:4f83530449c67c1ed8fca72583c5b92fdf446010990028c362a381e55dd84afd",
"Volumes": null,
"WorkingDir": "/usr/local/apache2",
"Entrypoint": null,
"OnBuild": [],
"Labels": {}
},
"container": "8825acde1b009729807e4b70a65a89399dd8da8e53be9216b9aaabaff4339f69",
"container_config": {
"Hostname": "383850eeb47b",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"ExposedPorts": {
"80/tcp": {}
},
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"PATH=/usr/local/apache2/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"HTTPD_PREFIX=/usr/local/apache2",
"HTTPD_VERSION=2.4.23",
"HTTPD_SHA1=5101be34ac4a509b245adb70a56690a84fcc4e7f",
"HTTPD_BZ2_URL=https://www.apache.org/dyn/closer.cgi?action=download\u0026filename=httpd/httpd-2.4.23.tar.bz2",
"HTTPD_ASC_URL=https://www.apache.org/dist/httpd/httpd-2.4.23.tar.bz2.asc"
],
"Cmd": [
"/bin/sh",
"-c",
"#(nop) ",
"CMD [\"httpd-foreground\"]"
],
"ArgsEscaped": true,
"Image": "sha256:4f83530449c67c1ed8fca72583c5b92fdf446010990028c362a381e55dd84afd",
"Volumes": null,
"WorkingDir": "/usr/local/apache2",
"Entrypoint": null,
"OnBuild": [],
"Labels": {}
},
"created": "2016-09-23T23:20:45.78976459Z",
"docker_version": "1.12.1",
"history": [
{
"created": "2016-09-23T18:08:50.537223822Z",
"created_by": "/bin/sh -c #(nop) ADD file:c6c23585ab140b0b320d4e99bc1b0eb544c9e96c24d90fec5e069a6d57d335ca in / "
},
{
"created": "2016-09-23T18:08:51.133779867Z",
"created_by": "/bin/sh -c #(nop) CMD [\"/bin/bash\"]",
"empty_layer": true
},
{
"created": "2016-09-23T19:16:40.725768956Z",
"created_by": "/bin/sh -c #(nop) ENV HTTPD_PREFIX=/usr/local/apache2",
"empty_layer": true
},
{
"created": "2016-09-23T19:16:41.037788416Z",
"created_by": "/bin/sh -c #(nop) ENV PATH=/usr/local/apache2/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"empty_layer": true
},
{
"created": "2016-09-23T19:16:41.990121202Z",
"created_by": "/bin/sh -c mkdir -p \"$HTTPD_PREFIX\" \t\u0026\u0026 chown www-data:www-data \"$HTTPD_PREFIX\""
},
{
"created": "2016-09-23T19:16:42.339911155Z",
"created_by": "/bin/sh -c #(nop) WORKDIR /usr/local/apache2",
"empty_layer": true
},
{
"created": "2016-09-23T19:16:54.948461741Z",
"created_by": "/bin/sh -c apt-get update \t\u0026\u0026 apt-get install -y --no-install-recommends \t\tlibapr1 \t\tlibaprutil1 \t\tlibaprutil1-ldap \t\tlibapr1-dev \t\tlibaprutil1-dev \t\tlibpcre++0 \t\tlibssl1.0.0 \t\u0026\u0026 rm -r /var/lib/apt/lists/*"
},
{
"created": "2016-09-23T19:16:55.321573403Z",
"created_by": "/bin/sh -c #(nop) ENV HTTPD_VERSION=2.4.23",
"empty_layer": true
},
{
"created": "2016-09-23T19:16:55.629947307Z",
"created_by": "/bin/sh -c #(nop) ENV HTTPD_SHA1=5101be34ac4a509b245adb70a56690a84fcc4e7f",
"empty_layer": true
},
{
"created": "2016-09-23T23:19:03.705796801Z",
"created_by": "/bin/sh -c #(nop) ENV HTTPD_BZ2_URL=https://www.apache.org/dyn/closer.cgi?action=download\u0026filename=httpd/httpd-2.4.23.tar.bz2",
"empty_layer": true
},
{
"created": "2016-09-23T23:19:04.009782822Z",
"created_by": "/bin/sh -c #(nop) ENV HTTPD_ASC_URL=https://www.apache.org/dist/httpd/httpd-2.4.23.tar.bz2.asc",
"empty_layer": true
},
{
"created": "2016-09-23T23:20:44.585743332Z",
"created_by": "/bin/sh -c set -x \t\u0026\u0026 buildDeps=' \t\tbzip2 \t\tca-certificates \t\tgcc \t\tlibpcre++-dev \t\tlibssl-dev \t\tmake \t\twget \t' \t\u0026\u0026 apt-get update \t\u0026\u0026 apt-get install -y --no-install-recommends $buildDeps \t\u0026\u0026 rm -r /var/lib/apt/lists/* \t\t\u0026\u0026 wget -O httpd.tar.bz2 \"$HTTPD_BZ2_URL\" \t\u0026\u0026 echo \"$HTTPD_SHA1 *httpd.tar.bz2\" | sha1sum -c - \t\u0026\u0026 wget -O httpd.tar.bz2.asc \"$HTTPD_ASC_URL\" \t\u0026\u0026 export GNUPGHOME=\"$(mktemp -d)\" \t\u0026\u0026 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys A93D62ECC3C8EA12DB220EC934EA76E6791485A8 \t\u0026\u0026 gpg --batch --verify httpd.tar.bz2.asc httpd.tar.bz2 \t\u0026\u0026 rm -r \"$GNUPGHOME\" httpd.tar.bz2.asc \t\t\u0026\u0026 mkdir -p src \t\u0026\u0026 tar -xvf httpd.tar.bz2 -C src --strip-components=1 \t\u0026\u0026 rm httpd.tar.bz2 \t\u0026\u0026 cd src \t\t\u0026\u0026 ./configure \t\t--prefix=\"$HTTPD_PREFIX\" \t\t--enable-mods-shared=reallyall \t\u0026\u0026 make -j\"$(nproc)\" \t\u0026\u0026 make install \t\t\u0026\u0026 cd .. \t\u0026\u0026 rm -r src \t\t\u0026\u0026 sed -ri \t\t-e 's!^(\\s*CustomLog)\\s+\\S+!\\1 /proc/self/fd/1!g' \t\t-e 's!^(\\s*ErrorLog)\\s+\\S+!\\1 /proc/self/fd/2!g' \t\t\"$HTTPD_PREFIX/conf/httpd.conf\" \t\t\u0026\u0026 apt-get purge -y --auto-remove $buildDeps"
},
{
"created": "2016-09-23T23:20:45.127455562Z",
"created_by": "/bin/sh -c #(nop) COPY file:761e313354b918b6cd7ea99975a4f6b53ff5381ba689bab2984aec4dab597215 in /usr/local/bin/ "
},
{
"created": "2016-09-23T23:20:45.453934921Z",
"created_by": "/bin/sh -c #(nop) EXPOSE 80/tcp",
"empty_layer": true
},
{
"created": "2016-09-23T23:20:45.78976459Z",
"created_by": "/bin/sh -c #(nop) CMD [\"httpd-foreground\"]",
"empty_layer": true
},
{
"created": "2023-10-01T02:03:04.56789764Z",
"created_by": "/bin/sh echo something > last"
}
],
"os": "linux",
"rootfs": {
"type": "layers",
"diff_ids": [
"sha256:142a601d97936307e75220c35dde0348971a9584c21e7cb42e1f7004005432ab",
"sha256:90fcc66ad3be9f1757f954b750deb37032f208428aa12599fcb02182b9065a9c",
"sha256:5a8624bb7e76d1e6829f9c64c43185e02bc07f97a2189eb048609a8914e72c56",
"sha256:d349ff6b3afc6a2800054768c82bfbf4289c9aa5da55c1290f802943dcd4d1e9",
"sha256:8c064bb1f60e84fa8cc6079b6d2e76e0423389fd6aeb7e497dfdae5e05b2b25b",
"sha256:1111111111111111111111111111111111111111111111111111111111111111"
]
}
}
4 changes: 2 additions & 2 deletions internal/image/fixtures/oci1-all-media-types.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"config": {
"mediaType": "application/vnd.oci.image.config.v1+json",
"size": 4651,
"digest": "sha256:a13a0762ab7bed51a1b49adec0a702b1cd99294fd460a025b465bcfb7b152745"
"digest": "sha256:94ac69e4413476d061116c9d05757e46a0afc744e8b9886f75cf7f6f14c78fb3"
},
"layers": [
{
Expand All @@ -28,7 +28,7 @@
"digest": "sha256:8f5dc8a4b12c307ac84de90cdd9a7f3915d1be04c9388868ca118831099c67a9"
},
{
"mediaType": "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip",
"mediaType": "application/vnd.oci.image.layer.nondistributable.v1.tar+zstd",
"size": 8841833,
"digest": "sha256:bbd6b22eb11afce63cc76f6bc41042d99f10d6024c96b655dafba930b8d25909"
},
Expand Down
Loading