diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index b90e0fcd..b5263574 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -47,6 +47,8 @@ jobs: loglevel: info - folder: dockerfile1 loglevel: debug + - folder: dockerfile2 + loglevel: debug steps: - name: Checkout diff --git a/go.mod b/go.mod index ca516e0d..e8946bc0 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/crazy-max/cron/v3 v3.1.1 github.com/crazy-max/gohealthchecks v0.4.1 github.com/crazy-max/gonfig v0.7.1 + github.com/docker/distribution v2.8.2+incompatible github.com/docker/docker v24.0.6+incompatible github.com/docker/go-connections v0.4.0 github.com/docker/go-units v0.5.0 @@ -68,7 +69,6 @@ require ( github.com/containers/ocicrypt v1.1.7 // indirect github.com/containers/storage v1.48.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/docker-credential-helpers v0.7.0 // indirect github.com/emicklei/go-restful/v3 v3.10.1 // indirect github.com/felixge/fgprof v0.9.3 // indirect diff --git a/internal/provider/common_test.go b/internal/provider/common_test.go index 5166c1e5..d6d145e6 100644 --- a/internal/provider/common_test.go +++ b/internal/provider/common_test.go @@ -20,9 +20,8 @@ func TestValidateImage(t *testing.T) { expectedImage model.Image expectedErr interface{} }{ - // Test strip sha { - name: "Test strip sha", + name: "Test strip digest", image: "myimg@sha256:1234567890abcdef", watchByDef: true, expectedImage: model.Image{ @@ -30,7 +29,6 @@ func TestValidateImage(t *testing.T) { }, expectedErr: nil, }, - // Test enable and watch by default { name: "All excluded by default", image: "myimg", diff --git a/pkg/dockerfile/fixtures/valid.Dockerfile b/pkg/dockerfile/fixtures/valid.Dockerfile index a295b329..19cd8d73 100644 --- a/pkg/dockerfile/fixtures/valid.Dockerfile +++ b/pkg/dockerfile/fixtures/valid.Dockerfile @@ -15,3 +15,9 @@ COPY --from=crazymax/yasu / / RUN --mount=type=bind,target=.,rw \ --mount=type=bind,from=crazymax/docker:20.10.6,source=/usr/local/bin/docker,target=/usr/bin/docker \ yasu --version + +# diun.platform=linux/amd64 +# diun.metadata.foo=bar +RUN --mount=type=bind,target=.,rw \ + --mount=type=bind,from=crazymax/ddns-route53:foo@sha256:9cb3af44cdd00615266c87e60bc05cac534297be14c4596800b57322f9313615,source=/usr/local/bin/ddns-route53,target=/usr/local/bin/ddns-route53 \ + ddns-route53 --version diff --git a/pkg/dockerfile/image_test.go b/pkg/dockerfile/image_test.go index 8fb333aa..77f5307a 100644 --- a/pkg/dockerfile/image_test.go +++ b/pkg/dockerfile/image_test.go @@ -17,7 +17,7 @@ func TestFromImages(t *testing.T) { img, err := c.FromImages() require.NoError(t, err) require.NotNil(t, img) - require.Equal(t, 3, len(img)) + require.Equal(t, 4, len(img)) assert.Equal(t, "alpine:3.14", img[0].Name) assert.Equal(t, 5, img[0].Line) @@ -30,4 +30,8 @@ func TestFromImages(t *testing.T) { assert.Equal(t, "crazymax/docker:20.10.6", img[2].Name) assert.Equal(t, 15, img[2].Line) assert.Equal(t, []string{"diun.watch_repo=true", "diun.include_tags=^\\d+\\.\\d+\\.\\d+$", "diun.platform=linux/amd64"}, img[2].Comments) + + assert.Equal(t, "crazymax/ddns-route53:foo@sha256:9cb3af44cdd00615266c87e60bc05cac534297be14c4596800b57322f9313615", img[3].Name) + assert.Equal(t, 21, img[3].Line) + assert.Equal(t, []string{"diun.platform=linux/amd64", "diun.metadata.foo=bar"}, img[3].Comments) } diff --git a/pkg/registry/image_test.go b/pkg/registry/image_test.go index 20f20cbf..3b0618b8 100644 --- a/pkg/registry/image_test.go +++ b/pkg/registry/image_test.go @@ -67,6 +67,18 @@ func TestParseImage(t *testing.T) { Tag: "latest", }, }, + { + desc: "gcr busybox tag/digest", + parseOpts: ParseImageOptions{ + Name: "gcr.io/google-containers/busybox:latest" + sha256digest, + }, + expected: Image{ + Domain: "gcr.io", + Path: "google-containers/busybox", + Tag: "latest", + Digest: sha256digest, + }, + }, { desc: "github ddns-route53", parseOpts: ParseImageOptions{ diff --git a/pkg/registry/manifest.go b/pkg/registry/manifest.go index 7d811d7a..4c6b1b8c 100644 --- a/pkg/registry/manifest.go +++ b/pkg/registry/manifest.go @@ -30,15 +30,20 @@ func (c *Client) Manifest(image Image, dbManifest Manifest) (Manifest, bool, err ctx, cancel := c.timeoutContext() defer cancel() - rmRef, err := ParseReference(image.String()) + rmRef, err := ImageReference(image.String()) if err != nil { return Manifest{}, false, errors.Wrap(err, "cannot parse reference") } - // Retrieve remote digest through HEAD request - rmDigest, err := docker.GetDigest(ctx, c.sysCtx, rmRef) - if err != nil { - return Manifest{}, false, errors.Wrap(err, "cannot get image digest from HEAD request") + // Retrieve remote digest through HEAD request or get one from image reference + var rmDigest digest.Digest + if len(image.Digest) > 0 { + rmDigest = image.Digest + } else { + rmDigest, err = docker.GetDigest(ctx, c.sysCtx, rmRef) + if err != nil { + return Manifest{}, false, errors.Wrap(err, "cannot get image digest from HEAD request") + } } // Digest match, returns db manifest diff --git a/pkg/registry/manifest_test.go b/pkg/registry/manifest_test.go index 7eb8f407..180fbf54 100644 --- a/pkg/registry/manifest_test.go +++ b/pkg/registry/manifest_test.go @@ -10,6 +10,8 @@ func TestCompareDigest(t *testing.T) { t.Parallel() rc, err := New(Options{ CompareDigest: true, + ImageOs: "linux", + ImageArch: "amd64", }) if err != nil { t.Error(err) @@ -22,6 +24,11 @@ func TestCompareDigest(t *testing.T) { t.Error(err) } + // download manifest + _, _, err = rc.Manifest(img, Manifest{}) + assert.NoError(t, err) + + // check manifest manifest, _, err := rc.Manifest(img, Manifest{ Name: "docker.io/crazymax/diun", Tag: "2.5.0", @@ -55,6 +62,11 @@ func TestManifest(t *testing.T) { t.Error(err) } + // download manifest + _, _, err = rc.Manifest(img, Manifest{}) + assert.NoError(t, err) + + // check manifest manifest, updated, err := rc.Manifest(img, Manifest{ Name: "docker.io/portainer/portainer-ce", Tag: "linux-amd64-2.5.1", @@ -116,6 +128,11 @@ func TestManifestMultiUpdatedPlatform(t *testing.T) { t.Error(err) } + // download manifest + _, _, err = rc.Manifest(img, Manifest{}) + assert.NoError(t, err) + + // check manifest manifest, updated, err := rc.Manifest(img, Manifest{ Name: "docker.io/library/mongo", Tag: "3.6.21", @@ -196,6 +213,11 @@ func TestManifestMultiNotUpdatedPlatform(t *testing.T) { t.Error(err) } + // download manifest + _, _, err = rc.Manifest(img, Manifest{}) + assert.NoError(t, err) + + // check manifest manifest, updated, err := rc.Manifest(img, Manifest{ Name: "docker.io/library/mongo", Tag: "3.6.21", @@ -284,3 +306,216 @@ func TestManifestVariant(t *testing.T) { assert.Equal(t, "linux/arm/v7", manifest.Platform) assert.Empty(t, manifest.DockerVersion) } + +func TestManifestTaggedDigest(t *testing.T) { + rc, err := New(Options{ + CompareDigest: true, + ImageOs: "linux", + ImageArch: "amd64", + }) + if err != nil { + t.Error(err) + } + + img, err := ParseImage(ParseImageOptions{ + Name: "crazymax/diun:latest@sha256:3fca3dd86c2710586208b0f92d1ec4ce25382f4cad4ae76a2275db8e8bb24031", + }) + if err != nil { + t.Error(err) + } + + // download manifest + _, _, err = rc.Manifest(img, Manifest{}) + assert.NoError(t, err) + + // check manifest + manifest, updated, err := rc.Manifest(img, manifestCrazymaxDiun4250) + assert.NoError(t, err) + assert.Equal(t, false, updated) + assert.Equal(t, "docker.io/crazymax/diun", manifest.Name) + assert.Equal(t, "latest", manifest.Tag) + assert.Equal(t, "application/vnd.oci.image.index.v1+json", manifest.MIMEType) + assert.Equal(t, "sha256:3fca3dd86c2710586208b0f92d1ec4ce25382f4cad4ae76a2275db8e8bb24031", manifest.Digest.String()) + assert.Equal(t, "linux/amd64", manifest.Platform) +} + +func TestManifestTaggedDigestDummyTag(t *testing.T) { + rc, err := New(Options{ + CompareDigest: true, + ImageOs: "linux", + ImageArch: "amd64", + }) + if err != nil { + t.Error(err) + } + + img, err := ParseImage(ParseImageOptions{ + Name: "crazymax/diun:foo@sha256:3fca3dd86c2710586208b0f92d1ec4ce25382f4cad4ae76a2275db8e8bb24031", + }) + if err != nil { + t.Error(err) + } + + // download manifest + _, _, err = rc.Manifest(img, Manifest{}) + assert.NoError(t, err) + + // check manifest + manifest, updated, err := rc.Manifest(img, manifestCrazymaxDiun4250) + assert.NoError(t, err) + assert.Equal(t, false, updated) + assert.Equal(t, "docker.io/crazymax/diun", manifest.Name) + assert.Equal(t, "latest", manifest.Tag) + assert.Equal(t, "application/vnd.oci.image.index.v1+json", manifest.MIMEType) + assert.Equal(t, "sha256:3fca3dd86c2710586208b0f92d1ec4ce25382f4cad4ae76a2275db8e8bb24031", manifest.Digest.String()) + assert.Equal(t, "linux/amd64", manifest.Platform) +} + +var manifestCrazymaxDiun4250 = Manifest{ + Name: "docker.io/crazymax/diun", + Tag: "latest", + MIMEType: "application/vnd.oci.image.index.v1+json", + Digest: "sha256:3fca3dd86c2710586208b0f92d1ec4ce25382f4cad4ae76a2275db8e8bb24031", + Platform: "linux/amd64", + Raw: []byte(`{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "digest": "sha256:3fca3dd86c2710586208b0f92d1ec4ce25382f4cad4ae76a2275db8e8bb24031", + "size": 4661, + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:bf782d6b2030c2a4c6884abb603ec5c99b5394554f57d56972cea24fb5d545d5", + "size": 866, + "platform": { + "architecture": "386", + "os": "linux" + } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:f44444abd33ee7c088d7527af84e3321f08313d12d9c679327bb8ae228e35f6a", + "size": 866, + "platform": { + "architecture": "amd64", + "os": "linux" + } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:df77b6ef88fbdb6175a2c60a9487a235aa1bdb39f60ee0a277d480d3cbc9f34a", + "size": 866, + "platform": { + "architecture": "arm", + "os": "linux", + "variant": "v6" + } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:73e210387511588b38d16046de4ade809404b746cf6d16cd51ca23a96c8264b7", + "size": 866, + "platform": { + "architecture": "arm", + "os": "linux", + "variant": "v7" + } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:1e070a6b2a3b5bf7c2c296fba6b01c8896514ae62aae6e48f4c28a775e5218dd", + "size": 866, + "platform": { + "architecture": "arm64", + "os": "linux" + } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:b7f984a85faf86839928fef6854f21da7afd2f2405b6043bf2aca562f1e1aa77", + "size": 866, + "platform": { + "architecture": "ppc64le", + "os": "linux" + } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:baa9a5e6de3f155526071eb0e55dcf14c12dca5c4301475e038df88fa5cb7c5a", + "size": 568, + "annotations": { + "vnd.docker.reference.digest": "sha256:bf782d6b2030c2a4c6884abb603ec5c99b5394554f57d56972cea24fb5d545d5", + "vnd.docker.reference.type": "attestation-manifest" + }, + "platform": { + "architecture": "unknown", + "os": "unknown" + } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:422bcf3cad62b4d8b21593387759889bcef02c28d7b0a3f6866b98b6502e8f01", + "size": 568, + "annotations": { + "vnd.docker.reference.digest": "sha256:f44444abd33ee7c088d7527af84e3321f08313d12d9c679327bb8ae228e35f6a", + "vnd.docker.reference.type": "attestation-manifest" + }, + "platform": { + "architecture": "unknown", + "os": "unknown" + } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:8ca5e335824bf17c10143c88f0e6955b5571dd69e06cd1a0ba46681169aa355d", + "size": 568, + "annotations": { + "vnd.docker.reference.digest": "sha256:df77b6ef88fbdb6175a2c60a9487a235aa1bdb39f60ee0a277d480d3cbc9f34a", + "vnd.docker.reference.type": "attestation-manifest" + }, + "platform": { + "architecture": "unknown", + "os": "unknown" + } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:01fdd0609476fe4da74af6bcb5a4fff97b0f9efbbea6b6ab142371ecc0738ffd", + "size": 568, + "annotations": { + "vnd.docker.reference.digest": "sha256:73e210387511588b38d16046de4ade809404b746cf6d16cd51ca23a96c8264b7", + "vnd.docker.reference.type": "attestation-manifest" + }, + "platform": { + "architecture": "unknown", + "os": "unknown" + } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:93178a24195f522195951a2cf16719bbae5358686b3789339c1096a85375117c", + "size": 568, + "annotations": { + "vnd.docker.reference.digest": "sha256:1e070a6b2a3b5bf7c2c296fba6b01c8896514ae62aae6e48f4c28a775e5218dd", + "vnd.docker.reference.type": "attestation-manifest" + }, + "platform": { + "architecture": "unknown", + "os": "unknown" + } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:1f5e5456e6f236c03684fea8070ca4095092a1d07a186acb03b15d160d100043", + "size": 568, + "annotations": { + "vnd.docker.reference.digest": "sha256:b7f984a85faf86839928fef6854f21da7afd2f2405b6043bf2aca562f1e1aa77", + "vnd.docker.reference.type": "attestation-manifest" + }, + "platform": { + "architecture": "unknown", + "os": "unknown" + } + } + ] +}`)} diff --git a/pkg/registry/ref.go b/pkg/registry/ref.go index d7ff9b33..00883bdc 100644 --- a/pkg/registry/ref.go +++ b/pkg/registry/ref.go @@ -6,11 +6,66 @@ import ( "github.com/containers/image/v5/docker" "github.com/containers/image/v5/types" + "github.com/docker/distribution/reference" + "github.com/pkg/errors" ) -func ParseReference(imageStr string) (types.ImageReference, error) { - if !strings.HasPrefix(imageStr, "//") { - imageStr = fmt.Sprintf("//%s", imageStr) +func ImageReference(name string) (types.ImageReference, error) { + ref, err := namedReference(name) + if err != nil { + return nil, errors.Wrap(err, "cannot parse reference") } - return docker.ParseReference(imageStr) + refStr := ref.String() + if !strings.HasPrefix(refStr, "//") { + refStr = fmt.Sprintf("//%s", refStr) + } + return docker.ParseReference(refStr) +} + +func namedReference(name string) (reference.Named, error) { + name = strings.TrimPrefix(name, "//") + + ref, err := reference.ParseNormalizedNamed(name) + if err != nil { + return nil, errors.Wrapf(err, "parsing normalized named %q", name) + } + + if _, ok := ref.(reference.Named); !ok { + return nil, errors.Errorf("%q is not a named reference", name) + } + + if _, hasTag := ref.(reference.NamedTagged); hasTag { + ref, err = normalizeTaggedDigestedNamed(ref) + if err != nil { + return nil, errors.Wrapf(err, "normalizing tagged digested name %q", name) + } + return ref, nil + } + if _, hasDigest := ref.(reference.Digested); hasDigest { + return ref, nil + } + + return reference.TagNameOnly(ref), nil +} + +// normalizeTaggedDigestedNamed strips the tag off the specified named +// reference if it is tagged and digested. Note that the tag is entirely +// ignored. +func normalizeTaggedDigestedNamed(named reference.Named) (reference.Named, error) { + _, isTagged := named.(reference.NamedTagged) + if !isTagged { + return named, nil + } + digested, isDigested := named.(reference.Digested) + if !isDigested { + return named, nil + } + // strip off the tag + newNamed := reference.TrimNamed(named) + // re-add the digest + newNamed, err := reference.WithDigest(newNamed, digested.Digest()) + if err != nil { + return named, err + } + return newNamed, nil } diff --git a/pkg/registry/ref_test.go b/pkg/registry/ref_test.go index 3b4760e8..f70942b9 100644 --- a/pkg/registry/ref_test.go +++ b/pkg/registry/ref_test.go @@ -12,7 +12,7 @@ const ( sha256digest = "@sha256:" + sha256digestHex ) -func TestParseReference(t *testing.T) { +func TestImageReference(t *testing.T) { testCases := []struct { input string expected string @@ -23,28 +23,27 @@ func TestParseReference(t *testing.T) { expected: "docker.io/library/busybox:latest", }, { - input: "//busybox:notlatest", - expected: "docker.io/library/busybox:notlatest", + input: "docker.io/library/busybox", + expected: "docker.io/library/busybox:latest", }, { - input: "//busybox" + sha256digest, - expected: "docker.io/library/busybox" + sha256digest, + input: "docker.io/library/busybox:latest", + expected: "docker.io/library/busybox:latest", }, { - input: "//busybox", - expected: "docker.io/library/busybox:latest", + input: "busybox:notlatest", + expected: "docker.io/library/busybox:notlatest", }, { - input: "//busybox:latest" + sha256digest, - expected: "", - wantErr: true, + input: "busybox" + sha256digest, + expected: "docker.io/library/busybox" + sha256digest, }, { - input: "//docker.io/library/busybox:latest", - expected: "docker.io/library/busybox:latest", + input: "busybox:latest" + sha256digest, + expected: "docker.io/library/busybox" + sha256digest, }, { - input: "//UPPERCASEISINVALID", + input: "UPPERCASEISINVALID", expected: "", wantErr: true, }, @@ -53,7 +52,7 @@ func TestParseReference(t *testing.T) { for _, tt := range testCases { tt := tt t.Run(tt.input, func(t *testing.T) { - ref, err := ParseReference(tt.input) + ref, err := ImageReference(tt.input) if tt.wantErr { require.Error(t, err) return diff --git a/pkg/registry/tags.go b/pkg/registry/tags.go index e8c97668..f8955ee6 100644 --- a/pkg/registry/tags.go +++ b/pkg/registry/tags.go @@ -34,7 +34,7 @@ func (c *Client) Tags(opts TagsOptions) (*Tags, error) { ctx, cancel := c.timeoutContext() defer cancel() - imgRef, err := ParseReference(opts.Image.String()) + imgRef, err := ImageReference(opts.Image.String()) if err != nil { return nil, errors.Wrap(err, "cannot parse reference") } diff --git a/pkg/registry/tags_test.go b/pkg/registry/tags_test.go index 0956d8eb..1038ba85 100644 --- a/pkg/registry/tags_test.go +++ b/pkg/registry/tags_test.go @@ -27,6 +27,27 @@ func TestTags(t *testing.T) { assert.True(t, len(tags.List) > 0) } +func TestTagsWithDigest(t *testing.T) { + assert.NotNil(t, rc) + + image, err := ParseImage(ParseImageOptions{ + Name: "crazymax/diun:latest@sha256:3fca3dd86c2710586208b0f92d1ec4ce25382f4cad4ae76a2275db8e8bb24031", + }) + if err != nil { + t.Error(err) + } + + tags, err := rc.Tags(TagsOptions{ + Image: image, + }) + if err != nil { + t.Error(err) + } + + assert.True(t, tags.Total > 0) + assert.True(t, len(tags.List) > 0) +} + func TestTagsSort(t *testing.T) { testCases := []struct { name string diff --git a/test/dockerfile2/diun.yml b/test/dockerfile2/diun.yml new file mode 100644 index 00000000..d3a1bd46 --- /dev/null +++ b/test/dockerfile2/diun.yml @@ -0,0 +1,15 @@ +watch: + workers: 20 + schedule: "0 */6 * * *" + firstCheckNotif: true + +notif: + script: + cmd: "sh" + args: + - "/mount/notif.sh" + +providers: + dockerfile: + patterns: + - /mount/Dockerfile diff --git a/test/dockerfile2/mount/Dockerfile b/test/dockerfile2/mount/Dockerfile new file mode 100644 index 00000000..0dadae18 --- /dev/null +++ b/test/dockerfile2/mount/Dockerfile @@ -0,0 +1,16 @@ +# syntax=docker/dockerfile:1 + +# diun.platform=linux/amd64 +FROM alpine:latest + +# diun.platform=linux/amd64 +# diun.metadata.foo=bar +RUN --mount=type=bind,target=.,rw \ + --mount=type=bind,from=crazymax/undock:0.5.0@sha256:736fdfde1268b93c2f733c53a7c45ece24e275318628fbb790bee7f89459961f,source=/usr/local/bin/undock,target=/usr/local/bin/undock \ + undock --version + +# diun.platform=linux/amd64 +# diun.metadata.foo=bar +RUN --mount=type=bind,target=.,rw \ + --mount=type=bind,from=crazymax/ddns-route53:foo@sha256:9cb3af44cdd00615266c87e60bc05cac534297be14c4596800b57322f9313615,source=/usr/local/bin/ddns-route53,target=/usr/local/bin/ddns-route53 \ + ddns-route53 --version diff --git a/test/dockerfile2/mount/notif.sh b/test/dockerfile2/mount/notif.sh new file mode 100644 index 00000000..f1ea27b1 --- /dev/null +++ b/test/dockerfile2/mount/notif.sh @@ -0,0 +1 @@ +env|sort