set digest-only for each mirror

Close: #1407
Add the `digest-only` field to each mirror table, so it will
be configured for each mirror instead of for primary registry.
For the same primary registry, it can support both digest only pull
and tag allowed pull.

The `mirror-by-digest-only` for primary is still allowed configuring,
and it is honored for compatibility

Signed-off-by: Qi Wang <qiwan@redhat.com>

docs/containers-registries.conf.5.md

@@ -108,16 +108,24 @@ Each TOML table in the `mirror` array can contain the following fields, with the
 as if specified in the `[[registry]]` TOML table directly:
 - `location`
 - `insecure`
+- `digest-only`
 
 `mirror-by-digest-only`
 : `true` or `false`.
 If `true`, mirrors will only be used during pulling if the image reference includes a digest.
+Note that if this is `true`, images referenced by a tag will only use the primary
+registry, failing if that registry is not accessible.
+
+`digest-only`: `true` or `false`.
+: if `true` an individual mirror will only be used during pulling if the image reference includes a digest.
+Note that digest-only will be overridden if mirror-by-digest is set for the entire registry.
+Note that if all mirrors in the `mirror` array have `digest-only = true`, images referenced by a tag will only use the primary
+registry, failing if that registry is not accessible.
+
 Referencing an image by digest ensures that the same is always used
 (whereas referencing an image by a tag may cause different registries to return
 different images if the tag mapping is out of sync).
 
-Note that if this is `true`, images referenced by a tag will only use the primary
-registry, failing if that registry is not accessible.
 
 *Note*: Redirection and mirrors are currently processed only when reading images, not when pushing
 to a registry; that may change in the future.

pkg/sysregistriesv2/system_registries_v2.go

@@ -53,6 +53,12 @@ type Endpoint struct {
 	// If true, certs verification will be skipped and HTTP (non-TLS)
 	// connections will be allowed.
 	Insecure bool `toml:"insecure,omitempty"`
+	// If true, the mirror will only be used for digest pulls. Pulling images by
+	// tag can potentially yield different images, depending on which endpoint
+	// we pull from.  Forcing digest-pulls for mirrors avoids that issue.
+	// This only applies to the mirror of the primary registry.
+	// If mirror-by-digest-only is set for the primary registry, mirror-by-digest-only is honored.
+	DigestOnly bool `toml:"digest-only,omitempty"`
 }
 
 // userRegistriesFile is the path to the per user registry configuration file.
@@ -130,16 +136,27 @@ type PullSource struct {
 // reference.
 func (r *Registry) PullSourcesFromReference(ref reference.Named) ([]PullSource, error) {
 	var endpoints []Endpoint
-
+	_, isDigested := ref.(reference.Canonical)
 	if r.MirrorByDigestOnly {
-		// Only use mirrors when the reference is a digest one.
-		if _, isDigested := ref.(reference.Canonical); isDigested {
+		// Only use mirrors when the reference is a digested one.
+		if isDigested {
 			endpoints = append(r.Mirrors, r.Endpoint)
 		} else {
 			endpoints = []Endpoint{r.Endpoint}
 		}
 	} else {
-		endpoints = append(r.Mirrors, r.Endpoint)
+		// check digest-only for each mirror if non mirror-by-digest-only set, since mirror-by-digest-only is honored
+		for _, mirror := range r.Mirrors {
+			if mirror.DigestOnly {
+				if isDigested {
+					// Only use the mirror when the reference is a digested one.
+					endpoints = append(endpoints, mirror)
+				}
+			} else {
+				endpoints = append(endpoints, mirror)
+			}
+		}
+		endpoints = append(endpoints, r.Endpoint)
 	}
 
 	sources := []PullSource{}
@@ -919,15 +936,19 @@ func loadConfigFile(path string, forceV2 bool) (*parsedConfig, error) {
 		res.shortNameMode = types.ShortNameModeInvalid
 	}
 
-	// Valid wildcarded prefixes must be in the format: *.example.com
-	// FIXME: Move to postProcessRegistries
-	// https://github.com/containers/image/pull/1191#discussion_r610623829
 	for i := range res.partialV2.Registries {
+		// Valid wildcarded prefixes must be in the format: *.example.com
+		// FIXME: Move to postProcessRegistries
+		// https://github.com/containers/image/pull/1191#discussion_r610623829
 		prefix := res.partialV2.Registries[i].Prefix
 		if strings.HasPrefix(prefix, "*.") && strings.ContainsAny(prefix, "/@:") {
 			msg := fmt.Sprintf("Wildcarded prefix should be in the format: *.example.com. Current prefix %q is incorrectly formatted", prefix)
 			return nil, &InvalidRegistries{s: msg}
 		}
+		// validate the mirror usage settings does not apply to primary registry
+		if res.partialV2.Registries[i].DigestOnly {
+			return nil, fmt.Errorf("digest-only does not apply to primary registry %s", prefix)
+		}
 	}
 
 	// Parse and validate short-name aliases.

pkg/sysregistriesv2/system_registries_v2_test.go

@@ -675,6 +675,84 @@ func TestPullSourcesFromReference(t *testing.T) {
 	assert.Equal(t, 1, len(pullSources))
 }
 
+func TestPullSourcesMirrorFromReference(t *testing.T) {
+	sys := &types.SystemContext{
+		SystemRegistriesConfPath:    "testdata/pull-sources-mirror-reference.conf",
+		SystemRegistriesConfDirPath: "testdata/this-does-not-exist",
+	}
+	registries, err := GetRegistries(sys)
+	assert.Nil(t, err)
+	assert.Equal(t, 4, len(registries))
+
+	// Registry A has mirrors allow any kind of pull
+	registryA, err := FindRegistry(sys, "registry-a.com/foo/image:latest")
+	assert.Nil(t, err)
+	assert.NotNil(t, registryA)
+	// Digest
+	referenceADigest := toNamedRef(t, "registry-a.com/foo/image@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
+	pullSources, err := registryA.PullSourcesFromReference(referenceADigest)
+	assert.Nil(t, err)
+	assert.Equal(t, 3, len(pullSources))
+	assert.Equal(t, "mirror-1.registry-a.com/image@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", pullSources[0].Reference.String())
+	assert.True(t, pullSources[1].Endpoint.Insecure)
+	// Tag
+	referenceATag := toNamedRef(t, "registry-a.com/foo/image:aaa")
+	pullSources, err = registryA.PullSourcesFromReference(referenceATag)
+	assert.Nil(t, err)
+	assert.Equal(t, 3, len(pullSources))
+	assert.Equal(t, "registry-a.com/bar/image:aaa", pullSources[2].Reference.String())
+
+	// Registry B has mirrors allow digests pull only
+	registryB, err := FindRegistry(sys, "registry-b.com/foo/image:latest")
+	assert.Nil(t, err)
+	assert.NotNil(t, registryB)
+	// Digest
+	referenceBDigest := toNamedRef(t, "registry-b.com/foo/image@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
+	pullSources, err = registryB.PullSourcesFromReference(referenceBDigest)
+	assert.Nil(t, err)
+	assert.Equal(t, 3, len(pullSources))
+	assert.Equal(t, "registry-b.com/bar", pullSources[2].Endpoint.Location)
+	assert.Equal(t, "registry-b.com/bar/image@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", pullSources[2].Reference.String())
+	// Tag
+	referenceBTag := toNamedRef(t, "registry-b.com/foo/image:aaa")
+	pullSources, err = registryB.PullSourcesFromReference(referenceBTag)
+	assert.Nil(t, err)
+	assert.Equal(t, 1, len(pullSources))
+
+	// Registry C has a mirror allows digest pull only and a mirror allows any kind of pull
+	registryC, err := FindRegistry(sys, "registry-c.com/foo/image:latest")
+	assert.Nil(t, err)
+	assert.NotNil(t, registryC)
+	// Digest
+	referenceCDigest := toNamedRef(t, "registry-c.com/foo/image@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
+	pullSources, err = registryC.PullSourcesFromReference(referenceCDigest)
+	assert.Nil(t, err)
+	assert.Equal(t, 3, len(pullSources))
+	assert.Equal(t, "registry-c.com/bar", pullSources[2].Endpoint.Location)
+	assert.Equal(t, "registry-c.com/bar/image@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", pullSources[2].Reference.String())
+	// Tag
+	referenceCTag := toNamedRef(t, "registry-c.com/foo/image:aaa")
+	pullSources, err = registryC.PullSourcesFromReference(referenceCTag)
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(pullSources))
+	// Registry D set mirror-by-digest-only, honor this configuration for registry
+	// Regsitry D has no digest-only set for mirrors table
+	registryD, err := FindRegistry(sys, "registry-d.com/foo/image:latest")
+	assert.Nil(t, err)
+	assert.NotNil(t, registryD)
+	// Digest
+	referenceDDigest := toNamedRef(t, "registry-d.com/foo/image@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
+	pullSources, err = registryD.PullSourcesFromReference(referenceDDigest)
+	assert.Nil(t, err)
+	assert.Equal(t, 3, len(pullSources))
+	assert.Equal(t, "mirror-1.registry-d.com/image@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", pullSources[0].Reference.String())
+	// Tag
+	referenceDTag := toNamedRef(t, "registry-d.com/foo/image:aaa")
+	pullSources, err = registryD.PullSourcesFromReference(referenceDTag)
+	assert.Nil(t, err)
+	assert.Equal(t, 1, len(pullSources))
+}
+
 func TestTryUpdatingCache(t *testing.T) {
 	ctx := &types.SystemContext{
 		SystemRegistriesConfPath:    "testdata/try-update-cache-valid.conf",

pkg/sysregistriesv2/testdata/pull-sources-mirror-reference.conf

@@ -0,0 +1,45 @@
+[[registry]]
+prefix = "registry-a.com/foo"
+location = "registry-a.com/bar"
+
+[[registry.mirror]]
+location = "mirror-1.registry-a.com"
+
+[[registry.mirror]]
+location = "mirror-2.registry-a.com"
+insecure = true
+
+[[registry]]
+prefix = "registry-b.com/foo"
+location = "registry-b.com/bar"
+
+[[registry.mirror]]
+digest-only = true
+location = "mirror-1.registry-b.com"
+
+[[registry.mirror]]
+digest-only = true
+location = "mirror-2.registry-b.com"
+
+[[registry]]
+prefix = "registry-c.com/foo"
+location = "registry-c.com/bar"
+
+[[registry.mirror]]
+location = "mirror-1.registry-c.com"
+
+[[registry.mirror]]
+digest-only = true
+location = "mirror-2.registry-c.com"
+
+[[registry]]
+prefix = "registry-d.com/foo"
+location = "registry-d.com/bar"
+mirror-by-digest-only = true
+
+[[registry.mirror]]
+location = "mirror-1.registry-d.com"
+
+[[registry.mirror]]
+location = "mirror-2.registry-d.com"
+