Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

conformance: make references tests part of content discovery workflow #430

Merged
merged 1 commit into from
Jun 27, 2023
Merged

conformance: make references tests part of content discovery workflow #430

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/build-pr.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ jobs:
make docs conformance

set +e
make conformance-ci
make registry-ci conformance-ci
CONFORMANCE_RC="$?"
set -e
if [[ -f report.html ]]; then
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ jobs:
make docs conformance

set +e
make conformance-ci
make registry-ci conformance-ci
CONFORMANCE_RC="$?"
set -e
if [[ -f report.html ]]; then
Expand Down
10 changes: 6 additions & 4 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,16 +91,18 @@ conformance-test:

conformance-binary: $(OUTPUT_DIRNAME)/conformance.test

TEST_REGISTRY_CONTAINER ?= ghcr.io/project-zot/zot-minimal-linux-amd64:v2.0.0-rc3
conformance-ci:
TEST_REGISTRY_CONTAINER ?= ghcr.io/project-zot/zot-minimal-linux-amd64:v2.0.0-rc5@sha256:740c4a4d99bf720761fd6407a227177cfeb3b1c0d4a230e16ceea960dc91dd11
registry-ci:
docker rm -f oci-conformance && \
echo '{"distSpecVersion":"1.1.0-dev","storage":{"rootDirectory":"/tmp/zot","gc":false,"dedupe":false},"http":{"address":"0.0.0.0","port":"5000"},"log":{"level":"debug"}}' > $(shell pwd)/$(OUTPUT_DIRNAME)/zot-config.json
docker run -d \
-v $(shell pwd)/$(OUTPUT_DIRNAME)/zot-config.json:/etc/zot/config.json \
--name=oci-conformance \
-p 5000:5000 \
$(TEST_REGISTRY_CONTAINER) && \
export OCI_ROOT_URL="http://localhost:5000" && \
$(TEST_REGISTRY_CONTAINER)

conformance-ci:
export OCI_ROOT_URL="http://localhost:5000" && \
export OCI_NAMESPACE="myorg/myrepo" && \
export OCI_TEST_PULL=1 && \
export OCI_TEST_PUSH=1 && \
Expand Down
1 change: 0 additions & 1 deletion conformance/00_conformance_suite_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ func TestConformance(t *testing.T) {
test02Push()
test03ContentDiscovery()
test04ContentManagement()
test05Referrers()
})

RegisterFailHandler(g.Fail)
Expand Down
272 changes: 271 additions & 1 deletion conformance/03_discovery_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package conformance

import (
"encoding/json"
"fmt"
"net/http"
"os"
Expand All @@ -10,6 +11,7 @@ import (
"github.com/bloodorangeio/reggie"
g "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
godigest "github.com/opencontainers/go-digest"
)

var test03ContentDiscovery = func() {
Expand Down Expand Up @@ -82,9 +84,151 @@ var test03ContentDiscovery = func() {
RunOnlyIfNot(runContentDiscoverySetup)
tagList = strings.Split(os.Getenv(envVarTagList), ",")
})

g.Specify("References setup", func() {
SkipIfDisabled(contentDiscovery)
RunOnlyIf(runContentDiscoverySetup)

// Populate registry with empty JSON blob
// validate expected empty JSON blob digest
Expect(emptyJSONDescriptor.Digest).To(Equal(godigest.Digest("sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a")))
req := client.NewRequest(reggie.POST, "/v2/<name>/blobs/uploads/")
resp, err := client.Do(req)
Expect(err).To(BeNil())
req = client.NewRequest(reggie.PUT, resp.GetRelativeLocation()).
SetQueryParam("digest", emptyJSONDescriptor.Digest.String()).
SetHeader("Content-Type", "application/octet-stream").
SetHeader("Content-Length", fmt.Sprintf("%d", emptyJSONDescriptor.Size)).
SetBody(emptyJSONBlob)
resp, err = client.Do(req)
Expect(err).To(BeNil())
Expect(resp.StatusCode()).To(SatisfyAll(
BeNumerically(">=", 200),
BeNumerically("<", 300)))

// Populate registry with reference blob before the image manifest is pushed
req = client.NewRequest(reggie.POST, "/v2/<name>/blobs/uploads/")
resp, err = client.Do(req)
Expect(err).To(BeNil())
req = client.NewRequest(reggie.PUT, resp.GetRelativeLocation()).
SetQueryParam("digest", testRefBlobADigest).
SetHeader("Content-Type", "application/octet-stream").
SetHeader("Content-Length", testRefBlobALength).
SetBody(testRefBlobA)
resp, err = client.Do(req)
Expect(err).To(BeNil())
Expect(resp.StatusCode()).To(SatisfyAll(
BeNumerically(">=", 200),
BeNumerically("<", 300)))

// Populate registry with test references manifest (config.MediaType = artifactType)
req = client.NewRequest(reggie.PUT, "/v2/<name>/manifests/<reference>",
reggie.WithReference(refsManifestAConfigArtifactDigest)).
SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
SetBody(refsManifestAConfigArtifactContent)
resp, err = client.Do(req)
Expect(err).To(BeNil())
Expect(resp.StatusCode()).To(SatisfyAll(
BeNumerically(">=", 200),
BeNumerically("<", 300)))
Expect(resp.Header().Get("OCI-Subject")).To(Equal(manifests[4].Digest))

// Populate registry with test references manifest (ArtifactType, config.MediaType = emptyJSON)
req = client.NewRequest(reggie.PUT, "/v2/<name>/manifests/<reference>",
reggie.WithReference(refsManifestALayerArtifactDigest)).
SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
SetBody(refsManifestALayerArtifactContent)
Copy link

@andaaron andaaron Jul 11, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi @jdolitsky

Looking at this test @peusebiu and I noticed refsManifestALayerArtifactContent actually contains a reference to layer testRefBlobB (see https://github.com/opencontainers/distribution-spec/blob/main/conformance/setup.go#L396), which is not yet uploaded to the registry.

My understanding of the push specifications is that layer blobs should be pushed by the client before the manifest.
If the server decides to reject them, it remains fully compliant. So why would the test check the server accepts this upload.

We're not sure if this is by design of the test, or a bug, as testRefBlobB is uploaded later at line 198.
On the other hand the code in setup.go is strange, maybe testRefBlobA should have been reference there in setup.go at line 396

Copy link

@andaaron andaaron Jul 11, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@rchincha looks these the values in setup.go are from your PR #375. This looks like a typo.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I defer to Ram. In this PR I tried to just copy-paste his work

resp, err = client.Do(req)
Expect(err).To(BeNil())
Expect(resp.StatusCode()).To(SatisfyAll(
BeNumerically(">=", 200),
BeNumerically("<", 300)))
Expect(resp.Header().Get("OCI-Subject")).To(Equal(manifests[4].Digest))

// Populate registry with test blob
req = client.NewRequest(reggie.POST, "/v2/<name>/blobs/uploads/")
resp, err = client.Do(req)
Expect(err).To(BeNil())
req = client.NewRequest(reggie.PUT, resp.GetRelativeLocation()).
SetQueryParam("digest", configs[4].Digest).
SetHeader("Content-Type", "application/octet-stream").
SetHeader("Content-Length", configs[4].ContentLength).
SetBody(configs[4].Content)
resp, err = client.Do(req)
Expect(err).To(BeNil())
Expect(resp.StatusCode()).To(SatisfyAll(
BeNumerically(">=", 200),
BeNumerically("<", 300)))

// Populate registry with test layer
req = client.NewRequest(reggie.POST, "/v2/<name>/blobs/uploads/")
resp, err = client.Do(req)
Expect(err).To(BeNil())
req = client.NewRequest(reggie.PUT, resp.GetRelativeLocation()).
SetQueryParam("digest", layerBlobDigest).
SetHeader("Content-Type", "application/octet-stream").
SetHeader("Content-Length", layerBlobContentLength).
SetBody(layerBlobData)
resp, err = client.Do(req)
Expect(err).To(BeNil())
Expect(resp.StatusCode()).To(SatisfyAll(
BeNumerically(">=", 200),
BeNumerically("<", 300)))

// Populate registry with test manifest
tag := testTagName
req = client.NewRequest(reggie.PUT, "/v2/<name>/manifests/<reference>",
reggie.WithReference(tag)).
SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
SetBody(manifests[4].Content)
resp, err = client.Do(req)
Expect(err).To(BeNil())
Expect(resp.StatusCode()).To(SatisfyAll(
BeNumerically(">=", 200),
BeNumerically("<", 300)))

// Populate registry with reference blob after the image manifest is pushed
req = client.NewRequest(reggie.POST, "/v2/<name>/blobs/uploads/")
resp, err = client.Do(req)
Expect(err).To(BeNil())
req = client.NewRequest(reggie.PUT, resp.GetRelativeLocation()).
SetQueryParam("digest", testRefBlobBDigest).
SetHeader("Content-Type", "application/octet-stream").
SetHeader("Content-Length", testRefBlobBLength).
SetBody(testRefBlobB)
resp, err = client.Do(req)
Expect(err).To(BeNil())
Expect(resp.StatusCode()).To(SatisfyAll(
BeNumerically(">=", 200),
BeNumerically("<", 300)))

// Populate registry with test references manifest (config.MediaType = artifactType)
req = client.NewRequest(reggie.PUT, "/v2/<name>/manifests/<reference>",
reggie.WithReference(refsManifestBConfigArtifactDigest)).
SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
SetBody(refsManifestBConfigArtifactContent)
resp, err = client.Do(req)
Expect(err).To(BeNil())
Expect(resp.StatusCode()).To(SatisfyAll(
BeNumerically(">=", 200),
BeNumerically("<", 300)))
Expect(resp.Header().Get("OCI-Subject")).To(Equal(manifests[4].Digest))

// Populate registry with test references manifest (ArtifactType, config.MediaType = emptyJSON)
req = client.NewRequest(reggie.PUT, "/v2/<name>/manifests/<reference>",
reggie.WithReference(refsManifestBLayerArtifactDigest)).
SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
SetBody(refsManifestBLayerArtifactContent)
resp, err = client.Do(req)
Expect(err).To(BeNil())
Expect(resp.StatusCode()).To(SatisfyAll(
BeNumerically(">=", 200),
BeNumerically("<", 300)))
Expect(resp.Header().Get("OCI-Subject")).To(Equal(manifests[4].Digest))
})
})

g.Context("Test content discovery endpoints", func() {
g.Context("Test content discovery endpoints (listing tags)", func() {
g.Specify("GET request to list tags should yield 200 response", func() {
SkipIfDisabled(contentDiscovery)
req := client.NewRequest(reggie.GET, "/v2/<name>/tags/list")
Expand Down Expand Up @@ -127,6 +271,66 @@ var test03ContentDiscovery = func() {
})
})

g.Context("Test content discovery endpoints (listing references)", func() {
g.Specify("GET request to nonexistent blob should result in empty 200 response", func() {
SkipIfDisabled(contentDiscovery)
req := client.NewRequest(reggie.GET, "/v2/<name>/referrers/<digest>",
reggie.WithDigest(dummyDigest))
resp, err := client.Do(req)
Expect(err).To(BeNil())
Expect(resp.StatusCode()).To(Equal(http.StatusOK))

var index index
err = json.Unmarshal(resp.Body(), &index)
Expect(err).To(BeNil())
Expect(len(index.Manifests)).To(BeZero())
})

g.Specify("GET request to existing blob should yield 200", func() {
SkipIfDisabled(contentDiscovery)
req := client.NewRequest(reggie.GET, "/v2/<name>/referrers/<digest>",
reggie.WithDigest(manifests[4].Digest))
resp, err := client.Do(req)
Expect(err).To(BeNil())
Expect(resp.StatusCode()).To(Equal(http.StatusOK))
if h := resp.Header().Get("Docker-Content-Digest"); h != "" {
Expect(h).To(Equal(configs[4].Digest))
}

var index index
err = json.Unmarshal(resp.Body(), &index)
Expect(err).To(BeNil())
Expect(len(index.Manifests)).To(Equal(4))
Expect(index.Manifests[0].Digest).ToNot(Equal(index.Manifests[1].Digest))
})

g.Specify("GET request to existing blob with filter should yield 200", func() {
SkipIfDisabled(contentDiscovery)
req := client.NewRequest(reggie.GET, "/v2/<name>/referrers/<digest>",
reggie.WithDigest(manifests[4].Digest)).
SetQueryParam("artifactType", testRefArtifactTypeA)
resp, err := client.Do(req)
Expect(err).To(BeNil())
Expect(resp.StatusCode()).To(Equal(http.StatusOK))
if h := resp.Header().Get("Docker-Content-Digest"); h != "" {
Expect(h).To(Equal(configs[4].Digest))
}

var index index
err = json.Unmarshal(resp.Body(), &index)
Expect(err).To(BeNil())

// also check resp header "OCI-Filters-Applied: artifactType" denoting that an artifactType filter was applied
if resp.Header().Get("OCI-Filters-Applied") != "" {
Expect(len(index.Manifests)).To(Equal(2))
Expect(resp.Header().Get("OCI-Filters-Applied")).To(Equal(testRefArtifactTypeA))
} else {
Expect(len(index.Manifests)).To(Equal(4))
Warn("filtering by artifact-type is not implemented")
}
})
})

g.Context("Teardown", func() {
if deleteManifestBeforeBlobs {
g.Specify("Delete created manifest & associated tags", func() {
Expand Down Expand Up @@ -191,6 +395,72 @@ var test03ContentDiscovery = func() {
))
})
}

g.Specify("References teardown", func() {
SkipIfDisabled(contentDiscovery)
RunOnlyIf(runContentDiscoverySetup)

deleteReq := func(req *reggie.Request) {
resp, err := client.Do(req)
Expect(err).To(BeNil())
Expect(resp.StatusCode()).To(SatisfyAny(
SatisfyAll(
BeNumerically(">=", 200),
BeNumerically("<", 300),
),
Equal(http.StatusMethodNotAllowed),
))
}

if deleteManifestBeforeBlobs {
req := client.NewRequest(reggie.DELETE, "/v2/<name>/manifests/<reference>",
reggie.WithReference(refsManifestAConfigArtifactDigest))
deleteReq(req)
req = client.NewRequest(reggie.DELETE, "/v2/<name>/manifests/<reference>",
reggie.WithReference(refsManifestALayerArtifactDigest))
deleteReq(req)
req = client.NewRequest(reggie.DELETE, "/v2/<name>/manifests/<digest>", reggie.WithDigest(manifests[4].Digest))
deleteReq(req)
req = client.NewRequest(reggie.DELETE, "/v2/<name>/manifests/<reference>",
reggie.WithReference(refsManifestBConfigArtifactDigest))
deleteReq(req)
req = client.NewRequest(reggie.DELETE, "/v2/<name>/manifests/<reference>",
reggie.WithReference(refsManifestBLayerArtifactDigest))
deleteReq(req)
}

// Delete config blob created in setup
req := client.NewRequest(reggie.DELETE, "/v2/<name>/blobs/<digest>", reggie.WithDigest(configs[4].Digest))
deleteReq(req)

// Delete reference blob created in setup
req = client.NewRequest(reggie.DELETE, "/v2/<name>/blobs/<digest>", reggie.WithDigest(testRefBlobADigest))
deleteReq(req)
req = client.NewRequest(reggie.DELETE, "/v2/<name>/blobs/<digest>", reggie.WithDigest(testRefBlobBDigest))
deleteReq(req)

// Delete empty JSON blob created in setup
req = client.NewRequest(reggie.DELETE, "/v2/<name>/blobs/<digest>", reggie.WithDigest(emptyJSONDescriptor.Digest.String()))
deleteReq(req)

if !deleteManifestBeforeBlobs {
// Delete manifest created in setup
req = client.NewRequest(reggie.DELETE, "/v2/<name>/manifests/<reference>",
reggie.WithReference(refsManifestAConfigArtifactDigest))
deleteReq(req)
req = client.NewRequest(reggie.DELETE, "/v2/<name>/manifests/<reference>",
reggie.WithReference(refsManifestALayerArtifactDigest))
deleteReq(req)
req = client.NewRequest(reggie.DELETE, "/v2/<name>/manifests/<digest>", reggie.WithDigest(manifests[4].Digest))
deleteReq(req)
req = client.NewRequest(reggie.DELETE, "/v2/<name>/manifests/<reference>",
reggie.WithReference(refsManifestBConfigArtifactDigest))
deleteReq(req)
req = client.NewRequest(reggie.DELETE, "/v2/<name>/manifests/<reference>",
reggie.WithReference(refsManifestBLayerArtifactDigest))
deleteReq(req)
}
})
})
})
}
Loading