feat(ovh): add cache based on DNS zone SOA value

OVHcloud customers using 'ovh' provider for large DNS zone will spend a
lot of time waiting for all records to be retrieved from the API. As
records can't change without a modification of the SOA Serial of the DNS
zone, this commit adds a client-side cache that stores the value of the
DNS records for each zone, based of the current SOA Serial value of the
zone. In case the SOA Serial of the zone is identical to the one in
cache, we can serve all the records directly from cache. Otherwise,
fallback on refreshing the cache from the API.

Signed-off-by: Romain Beuque <556072+rbeuque74@users.noreply.github.com>

docs/tutorials/ovh.md

@@ -18,14 +18,15 @@ instructions for creating a zone.
 
 You first need to create an OVH application.
 
-Using the [OVH documentation](https://docs.ovh.com/gb/en/customer/first-steps-with-ovh-api/#creation-of-your-application-keys) you will have your `Application key` and `Application secret`
+Using the [OVH documentation](https://docs.ovh.com/gb/en/api/first-steps-with-ovh-api/#advanced-usage-pair-ovhcloud-apis-with-an-application_2) you will have your `Application key` and `Application secret`
 
 And you will need to generate your consumer key, here the permissions needed :
 - GET on `/domain/zone`
 - GET on `/domain/zone/*/record`
 - GET on `/domain/zone/*/record/*`
 - POST on `/domain/zone/*/record`
 - DELETE on `/domain/zone/*/record/*`
+- GET on `/domain/zone/*/soa`
 - POST on `/domain/zone/*/refresh`
 
 You can use the following `curl` request to generate & validated your `Consumer key`
@@ -37,6 +38,10 @@ curl -XPOST -H "X-Ovh-Application: <ApplicationKey>" -H "Content-type: applicati
       "method": "GET",
       "path": "/domain/zone"
     },
+    {
+      "method": "GET",
+      "path": "/domain/zone/*/soa"
+    },
     {
       "method": "GET",
       "path": "/domain/zone/*/record"

go.mod

@@ -42,7 +42,8 @@ require (
 	github.com/openshift/api v0.0.0-20200605231317-fb2a6ca106ae
 	github.com/openshift/client-go v0.0.0-20200608144219-584632b8fc73
 	github.com/oracle/oci-go-sdk v24.3.0+incompatible
-	github.com/ovh/go-ovh v0.0.0-20181109152953-ba5adb4cf014
+	github.com/ovh/go-ovh v1.3.0
+	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
 	github.com/pluralsh/gqlclient v1.1.6
 	github.com/projectcontour/contour v1.23.2
@@ -151,7 +152,6 @@ require (
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/openshift/gssapi v0.0.0-20161010215902-5fb4217df13b // indirect
 	github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect
-	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 	github.com/peterhellberg/link v1.1.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect

go.sum

@@ -926,8 +926,8 @@ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnh
 github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
 github.com/oracle/oci-go-sdk v24.3.0+incompatible h1:x4mcfb4agelf1O4/1/auGlZ1lr97jXRSSN5MxTgG/zU=
 github.com/oracle/oci-go-sdk v24.3.0+incompatible/go.mod h1:VQb79nF8Z2cwLkLS35ukwStZIg5F66tcBccjip/j888=
-github.com/ovh/go-ovh v0.0.0-20181109152953-ba5adb4cf014 h1:37VE5TYj2m/FLA9SNr4z0+A0JefvTmR60Zwf8XSEV7c=
-github.com/ovh/go-ovh v0.0.0-20181109152953-ba5adb4cf014/go.mod h1:joRatxRJaZBsY3JAOEMcoOp05CnZzsx4scTxi95DHyQ=
+github.com/ovh/go-ovh v1.3.0 h1:mvZaddk4E4kLcXhzb+cxBsMPYp2pHqiQpWYkInsuZPQ=
+github.com/ovh/go-ovh v1.3.0/go.mod h1:AxitLZ5HBRPyUd+Zl60Ajaag+rNTdVXWIkzfrVuTXWA=
 github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2 h1:CXwSGu/LYmbjEab5aMCs5usQRVBGThelUKBNnoSOuso=
 github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2/go.mod h1:L3UMQOThbttwfYRNFOWLLVXMhk5Lkio4GGOtw5UrxS0=
 github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM=
@@ -1804,6 +1804,7 @@ gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ini.v1 v1.66.6 h1:LATuAqN/shcYAOkv3wl2L4rkaKqkcgTBQjOyYDvcPKI=
 gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=

provider/ovh/ovh.go

@@ -21,12 +21,16 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+	"time"
 
+	"github.com/miekg/dns"
 	"github.com/ovh/go-ovh/ovh"
+	"github.com/patrickmn/go-cache"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
 
 	"sigs.k8s.io/external-dns/endpoint"
+	"sigs.k8s.io/external-dns/pkg/apis/externaldns"
 	"sigs.k8s.io/external-dns/plan"
 	"sigs.k8s.io/external-dns/provider"
 
@@ -56,6 +60,17 @@ type OVHProvider struct {
 
 	domainFilter endpoint.DomainFilter
 	DryRun       bool
+
+	// UseCache controls if the OVHProvider will cache records in memory, and serve them
+	// without recontacting the OVHcloud API if the SOA of the domain zone hasn't changed.
+	// Note that, when disabling cache, OVHcloud API has rate-limiting that will hit if
+	// your refresh rate/number of records is too big, which might cause issue with the
+	// provider.
+	// Default value: true
+	UseCache bool
+
+	cacheInstance *cache.Cache
+	dnsClient     dnsClient
 }
 
 type ovhClient interface {
@@ -64,6 +79,10 @@ type ovhClient interface {
 	Delete(string, interface{}) error
 }
 
+type dnsClient interface {
+	ExchangeContext(ctx context.Context, m *dns.Msg, a string) (*dns.Msg, time.Duration, error)
+}
+
 type ovhRecordFields struct {
 	FieldType string `json:"fieldType"`
 	SubDomain string `json:"subDomain"`
@@ -88,6 +107,9 @@ func NewOVHProvider(ctx context.Context, domainFilter endpoint.DomainFilter, end
 	if err != nil {
 		return nil, err
 	}
+
+	client.UserAgent = externaldns.Version
+
 	// TODO: Add Dry Run support
 	if dryRun {
 		return nil, ErrNoDryRun
@@ -97,6 +119,9 @@ func NewOVHProvider(ctx context.Context, domainFilter endpoint.DomainFilter, end
 		domainFilter:   domainFilter,
 		apiRateLimiter: ratelimit.New(apiRateLimit),
 		DryRun:         dryRun,
+		cacheInstance:  cache.New(cache.NoExpiration, cache.NoExpiration),
+		dnsClient:      new(dns.Client),
+		UseCache:       true,
 	}, nil
 }
 
@@ -217,14 +242,48 @@ func (p *OVHProvider) zones() ([]string, error) {
 	return filteredZones, nil
 }
 
+type ovhSoa struct {
+	Server  string `json:"server"`
+	Serial  uint32 `json:"serial"`
+	records []ovhRecord
+}
+
 func (p *OVHProvider) records(ctx *context.Context, zone *string, records chan<- []ovhRecord) error {
 	var recordsIds []uint64
 	ovhRecords := make([]ovhRecord, len(recordsIds))
 	eg, _ := errgroup.WithContext(*ctx)
 
+	if p.UseCache {
+		if cachedSoaItf, ok := p.cacheInstance.Get(*zone + "#soa"); ok {
+			cachedSoa := cachedSoaItf.(ovhSoa)
+
+			m := new(dns.Msg)
+			m.SetQuestion(dns.Fqdn(*zone), dns.TypeSOA)
+			in, _, err := p.dnsClient.ExchangeContext(*ctx, m, strings.TrimSuffix(cachedSoa.Server, ".")+":53")
+			if err == nil {
+				if s, ok := in.Answer[0].(*dns.SOA); ok {
+					// do something with t.Txt
+					if s.Serial == cachedSoa.Serial {
+						records <- cachedSoa.records
+						return nil
+					}
+				}
+			}
+
+			p.cacheInstance.Delete(*zone + "#soa")
+		}
+	}
+
 	log.Debugf("OVH: Getting records for %s", *zone)
 
 	p.apiRateLimiter.Take()
+	var soa ovhSoa
+	if p.UseCache {
+		if err := p.client.Get("/domain/zone/"+*zone+"/soa", &soa); err != nil {
+			return err
+		}
+	}
+
 	if err := p.client.Get(fmt.Sprintf("/domain/zone/%s/record", *zone), &recordsIds); err != nil {
 		return err
 	}
@@ -240,6 +299,12 @@ func (p *OVHProvider) records(ctx *context.Context, zone *string, records chan<-
 	for record := range chRecords {
 		ovhRecords = append(ovhRecords, record)
 	}
+
+	if p.UseCache {
+		soa.records = ovhRecords
+		_ = p.cacheInstance.Add(*zone+"#soa", soa, time.Hour)
+	}
+
 	records <- ovhRecords
 	return nil
 }