diff --git a/README.md b/README.md index 9fabd3c..9325618 100644 --- a/README.md +++ b/README.md @@ -8,16 +8,18 @@ https://prometheus.io/docs/prometheus/latest/http_sd/ ## Usage The `tailscalesd` server is very simple. It serves the SD payload at `/` on its -HTTP server. It respects four configuration parameters, each of which may be -specified as a flag or an environment variable. +HTTP server. It respects the following configuration parameters, each of which +may be specified as a flag or an environment variable. - `-address` / `ADDRESS` is the host:port on which to serve TailscaleSD. Defaults to `0.0.0.0:9242`. +- `-localapi` / `TAILSCALE_USE_LOCAL_API` instructs TailscaleSD to use the + `tailscaled`-exported local API for discovery. +- `-localapi_socket` / `TAILSCALE_LOCAL_API_SOCKET` is the path to the Unix + domain socket over which `tailscaled` serves the local API. - `-poll` / `TAILSCALE_API_POLL_LIMIT` is the limit of how frequently the Tailscale API may be polled. Cached results are served between intervals. Defaults to 5 minutes. Also applies to local API. -- `-localapi` / `TAILSCALE_USE_LOCAL_API` instructs TailscaleSD to use the - `tailscaled`-exported local API for discovery. - `-tailnet` / `TAILNET` is the name of the tailnet to enumerate. Required when using the public API. - `-token` / `TAILSCALE_API_TOKEN` is a Tailscale API token with appropriate diff --git a/cmd/tailscalesd/main.go b/cmd/tailscalesd/main.go index 654abc1..0e33da3 100644 --- a/cmd/tailscalesd/main.go +++ b/cmd/tailscalesd/main.go @@ -10,16 +10,16 @@ import ( "time" "github.com/cfunkhouser/tailscalesd" - "github.com/cfunkhouser/tailscalesd/internal/logwriter" ) var ( - address string = "0.0.0.0:9242" - token string - tailnet string - printVer bool - pollLimit time.Duration = time.Minute * 5 - useLocalAPI bool + address string = "0.0.0.0:9242" + token string + tailnet string + printVer bool + pollLimit time.Duration = time.Minute * 5 + useLocalAPI bool + localAPISocket string = tailscalesd.PublicAPIHost // Version of tailscalesd. Set at build time to something meaningful. Version = "development" @@ -58,11 +58,24 @@ func defineFlags() { flag.DurationVar(&pollLimit, "poll", durationEnvVarWithDefault("TAILSCALE_API_POLL_LIMIT", pollLimit), "Max frequency with which to poll the Tailscale API. Cached results are served between intervals.") flag.BoolVar(&printVer, "version", false, "Print the version and exit.") flag.BoolVar(&useLocalAPI, "localapi", boolEnvVarWithDefault("TAILSCALE_USE_LOCAL_API", false), "Use the Tailscale local API exported by the local node's tailscaled") + flag.StringVar(&localAPISocket, "localapi_socket", envVarWithDefault("TAILSCALE_LOCAL_API_SOCKET", localAPISocket), "Unix Domain Socket to use for communication with the local tailscaled API.") +} + +type logWriter struct { + TZ *time.Location + Format string +} + +func (w *logWriter) Write(data []byte) (int, error) { + return fmt.Printf("%v %v", time.Now().In(w.TZ).Format(w.Format), string(data)) } func main() { log.SetFlags(0) - log.SetOutput(logwriter.Default()) + log.SetOutput(&logWriter{ + TZ: time.UTC, + Format: time.RFC3339, + }) defineFlags() flag.Parse() @@ -73,18 +86,32 @@ func main() { } if !useLocalAPI && (token == "" || tailnet == "") { - fmt.Println("Both -token and -tailnet are required when using the public API") + if _, err := fmt.Fprintln(os.Stderr, "Both -token and -tailnet are required when using the public API"); err != nil { + panic(err) + } flag.Usage() return } - var d tailscalesd.Discoverer + if useLocalAPI && localAPISocket == "" { + if _, err := fmt.Fprintln(os.Stderr, "-localapi_socket must not be empty when using the local API."); err != nil { + panic(err) + } + flag.Usage() + return + } + + var ts tailscalesd.Discoverer if useLocalAPI { - d = tailscalesd.New(tailscalesd.UsingLocalAPI(), tailscalesd.WithRateLimit(pollLimit)) + ts = tailscalesd.LocalAPI(tailscalesd.LocalAPISocket) } else { - d = tailscalesd.New(tailscalesd.UsingPublicAPI(tailnet, token), tailscalesd.WithRateLimit(pollLimit)) + ts = tailscalesd.PublicAPI(tailnet, token) + } + ts = &tailscalesd.RateLimitedDiscoverer{ + Wrap: ts, + Frequency: pollLimit, } - http.Handle("/", tailscalesd.Export(d, time.Minute*5)) + http.Handle("/", tailscalesd.Export(ts)) log.Printf("Serving Tailscale service discovery on %q", address) log.Print(http.ListenAndServe(address, nil)) log.Print("Done") diff --git a/internal/logwriter/logwriter.go b/internal/logwriter/logwriter.go deleted file mode 100644 index d0c42cf..0000000 --- a/internal/logwriter/logwriter.go +++ /dev/null @@ -1,27 +0,0 @@ -package logwriter - -import ( - "fmt" - "io" - "time" -) - -type LogWriter struct { - tz *time.Location - format string -} - -func (w *LogWriter) Write(data []byte) (int, error) { - return fmt.Printf("%v %v", time.Now().In(w.tz).Format(w.format), string(data)) -} - -func New(tz *time.Location, format string) io.Writer { - return &LogWriter{ - tz: tz, - format: format, - } -} - -func Default() io.Writer { - return New(time.UTC, time.RFC3339) -} diff --git a/internal/tailscale/public/public.go b/internal/tailscale/public/public.go deleted file mode 100644 index ee39a4f..0000000 --- a/internal/tailscale/public/public.go +++ /dev/null @@ -1,55 +0,0 @@ -// Package public is a naive, bespoke Tailscale V2 API client. It has only the -// functionality needed for tailscalesd. You should not rely on its API for -// anything else. -package public - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "net/http" - - "github.com/cfunkhouser/tailscalesd/internal/tailscale" -) - -type deviceAPIResponse struct { - Devices []tailscale.Device `json:"devices"` -} - -// API client for the Tailscale public API. -type API struct { - Client *http.Client - APIBase string - Tailnet string - Token string -} - -var ErrFailedRequest = errors.New("failed API call") - -// Devices reported by the Tailscale public API as belonging to the configured -// tailnet. -func (a *API) Devices(ctx context.Context) ([]tailscale.Device, error) { - url := fmt.Sprintf("https://%v@%v/api/v2/tailnet/%v/devices", a.Token, a.APIBase, a.Tailnet) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return nil, err - } - resp, err := a.Client.Do(req) - if err != nil { - return nil, err - } - if (resp.StatusCode / 100) != 2 { - return nil, fmt.Errorf("%w: %v", ErrFailedRequest, resp.Status) - } - defer resp.Body.Close() - var d deviceAPIResponse - if err := json.NewDecoder(resp.Body).Decode(&d); err != nil { - return nil, err - } - for i := range d.Devices { - d.Devices[i].API = a.APIBase - d.Devices[i].Tailnet = a.Tailnet - } - return d.Devices, nil -} diff --git a/internal/tailscale/tailscale.go b/internal/tailscale/tailscale.go deleted file mode 100644 index 19c6242..0000000 --- a/internal/tailscale/tailscale.go +++ /dev/null @@ -1,19 +0,0 @@ -// Package tailscale contains types needed for both API implementations. -package tailscale - -// PublicAPI host for Tailscale. -const PublicAPI = "api.tailscale.com" - -// Device in a Tailnet, as reported by one of the various Tailscale APIs. -type Device struct { - Addresses []string `json:"addresses"` - API string `json:"api"` - Authorized bool `json:"authorized"` - ClientVersion string `json:"clientVersion,omitempty"` - Hostname string `json:"hostname"` - ID string `json:"id"` - Name string `json:"name"` - OS string `json:"os"` - Tailnet string `json:"tailnet"` - Tags []string `json:"tags"` -} diff --git a/internal/tailscale/local/local.go b/localapi.go similarity index 53% rename from internal/tailscale/local/local.go rename to localapi.go index 6a70d23..ea28de0 100644 --- a/internal/tailscale/local/local.go +++ b/localapi.go @@ -1,7 +1,4 @@ -// Package local is a client for the Tailscale local API, which is exported by -// tailscaled. It has only the functionality needed for tailscalesd. You should -// not rely on its API for anything else. -package local +package tailscalesd import ( "context" @@ -10,23 +7,19 @@ import ( "fmt" "net" "net/http" + "time" "inet.af/netaddr" - - "github.com/cfunkhouser/tailscalesd/internal/tailscale" ) -// unixDialer is a DialContext allowing HTTP communication via a unix domain -// socket. -func unixDialer(socket string) func(context.Context, string, string) (net.Conn, error) { - return func(ctx context.Context, _, _ string) (net.Conn, error) { - return net.Dial("unix", socket) - } -} +// LocalAPISocket is the path to the Unix domain socket on which tailscaled +// listens locally. +const LocalAPISocket = "/run/tailscale/tailscaled.sock" // interstingStatusSubset is a json-decodeable subset of the Status struct -// served by the Tailscale local API. This is done to prevent pulling the Tailscale code base and its dependencies into this module. -// The fields were borrowed from version 1.22.2. For field details, see: +// served by the Tailscale local API. This is done to prevent pulling the +// Tailscale code base and its dependencies into this module. The fields were +// borrowed from version 1.22.2. For field details, see: // https://pkg.go.dev/tailscale.com@v1.22.2/ipn/ipnstate?utm_source=gopls#Status type interestingStatusSubset struct { TailscaleIPs []netaddr.IP // Tailscale IP(s) assigned to this node @@ -45,14 +38,13 @@ type interestingPeerStatusSubset struct { Tags []string `json:",omitempty"` } -// API client for the Tailscale local API. -type API struct { +type localAPIClient struct { client *http.Client } -var ErrFailedRequest = errors.New("failed localapi call") +var errFailedLocalAPIRequest = errors.New("failed local API request") -func (a *API) status(ctx context.Context) (interestingStatusSubset, error) { +func (a *localAPIClient) status(ctx context.Context) (interestingStatusSubset, error) { var status interestingStatusSubset req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost/localapi/v0/status", nil) if err != nil { @@ -64,7 +56,7 @@ func (a *API) status(ctx context.Context) (interestingStatusSubset, error) { } if (resp.StatusCode / 100) != 2 { - return status, fmt.Errorf("%w: %v", ErrFailedRequest, resp.Status) + return status, fmt.Errorf("%w: %v", errFailedLocalAPIRequest, resp.Status) } defer resp.Body.Close() @@ -74,27 +66,25 @@ func (a *API) status(ctx context.Context) (interestingStatusSubset, error) { return status, nil } -func translatePeerToDevice(p *interestingPeerStatusSubset, d *tailscale.Device) { +func translatePeerToDevice(p *interestingPeerStatusSubset, d *Device) { for i := range p.TailscaleIPs { d.Addresses = append(d.Addresses, p.TailscaleIPs[i].String()) } - - // Assumes that if the peer is listed in localapi, it is authorized enough. - d.Authorized = true d.API = "localhost" + d.Authorized = true // localapi returned peer; assume it's authorized enough d.Hostname = p.HostName - d.ID = fmt.Sprintf("%v", p.ID) + d.ID = p.ID d.OS = p.OS d.Tags = p.Tags[:] } // Devices reported by the Tailscale local API as peers of the local host. -func (a *API) Devices(ctx context.Context) ([]tailscale.Device, error) { +func (a *localAPIClient) Devices(ctx context.Context) ([]Device, error) { status, err := a.status(ctx) if err != nil { return nil, err } - devices := make([]tailscale.Device, len(status.Peer)) + devices := make([]Device, len(status.Peer)) var i int for _, peer := range status.Peer { translatePeerToDevice(peer, &devices[i]) @@ -103,12 +93,31 @@ func (a *API) Devices(ctx context.Context) ([]tailscale.Device, error) { return devices, nil } -func New(socket string) *API { - return &API{ - client: &http.Client{ - Transport: &http.Transport{ - DialContext: unixDialer(socket), - }, +type dialContext func(context.Context, string, string) (net.Conn, error) + +func unixSocketDialer(socket string) dialContext { + return func(ctx context.Context, _, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "unix", socket) + } +} + +func defaultHTTPClientWithDialer(dc dialContext) *http.Client { + return &http.Client{ + Timeout: time.Second * 10, + Transport: &http.Transport{ + Dial: (&net.Dialer{ + Timeout: 5 * time.Second, + }).Dial, + DialContext: dc, + TLSHandshakeTimeout: 5 * time.Second, }, } } + +// LocalAPI Discoverer interrogates the Tailscale localapi for peer devices. +func LocalAPI(socket string) Discoverer { + return &localAPIClient{ + client: defaultHTTPClientWithDialer(unixSocketDialer(socket)), + } +} diff --git a/localapi_test.go b/localapi_test.go new file mode 100644 index 0000000..4631af5 --- /dev/null +++ b/localapi_test.go @@ -0,0 +1,44 @@ +package tailscalesd + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "inet.af/netaddr" +) + +func TestTranslatePeerToDevice(t *testing.T) { + want := Device{ + Addresses: []string{ + "100.2.3.4", + "fd7a::1234", + }, + API: "localhost", + Authorized: true, + Hostname: "somethingclever", + ID: "id", + OS: "beos", + Tags: []string{ + "tag:foo", + "tag:bar", + }, + } + var got Device + translatePeerToDevice(&interestingPeerStatusSubset{ + ID: "id", + HostName: "somethingclever", + DNSName: "this is currently ignored", + OS: "beos", + TailscaleIPs: []netaddr.IP{ + netaddr.MustParseIP("100.2.3.4"), + netaddr.MustParseIP("fd7a::1234"), + }, + Tags: []string{ + "tag:foo", + "tag:bar", + }, + }, &got) + if diff := cmp.Diff(got, want); diff != "" { + t.Errorf("translatePeerToDevice: mismatch (-got, +want):\n%v", diff) + } +} diff --git a/publicapi.go b/publicapi.go new file mode 100644 index 0000000..4e38758 --- /dev/null +++ b/publicapi.go @@ -0,0 +1,98 @@ +package tailscalesd + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "time" +) + +type deviceAPIResponse struct { + Devices []Device `json:"devices"` +} + +type publicAPIDiscoverer struct { + client *http.Client + apiBase string + tailnet string + token string +} + +var errFailedAPIRequest = errors.New("failed API request") + +func (a *publicAPIDiscoverer) Devices(ctx context.Context) ([]Device, error) { + url := fmt.Sprintf("https://%v@%v/api/v2/tailnet/%v/devices", a.token, a.apiBase, a.tailnet) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := a.client.Do(req) + if err != nil { + return nil, err + } + if (resp.StatusCode / 100) != 2 { + return nil, fmt.Errorf("%w: %v", errFailedAPIRequest, resp.Status) + } + defer resp.Body.Close() + var d deviceAPIResponse + if err := json.NewDecoder(resp.Body).Decode(&d); err != nil { + return nil, fmt.Errorf("%w: bad payload from API: %v", errFailedAPIRequest, err) + } + for i := range d.Devices { + d.Devices[i].API = a.apiBase + d.Devices[i].Tailnet = a.tailnet + } + return d.Devices, nil +} + +type PublicAPIOption func(*publicAPIDiscoverer) + +// WithAPIHost sets the API base against which the PublicAPI Discoverers will +// attempt discovery. If not used, defaults to PublicAPIHost. +func WithAPIHost(host string) PublicAPIOption { + return func(api *publicAPIDiscoverer) { + api.apiBase = host + } +} + +// WithHTTPClient is a PublicAPIOption which allows callers to provide a HTTP +// client to PublicAPI instances. If not used, the defaultHTTPClient is used. +func WithHTTPClient(client *http.Client) PublicAPIOption { + return func(api *publicAPIDiscoverer) { + api.client = client + } +} + +// PublicAPIHost host for Tailscale. +const PublicAPIHost = "api.tailscale.com" + +// defaultHTTPClient is shared across PublicAPI Discoverer instances by default. +// It strives to have sane enough defaults to not shoot users in the foot. +var defaultHTTPClient = &http.Client{ + Timeout: time.Second * 10, + Transport: &http.Transport{ + Dial: (&net.Dialer{ + Timeout: 5 * time.Second, + }).Dial, + TLSHandshakeTimeout: 5 * time.Second, + }, +} + +// PublicAPI Discoverer polls the public Tailscale API for hosts in the tailnet. +func PublicAPI(tailnet, token string, opts ...PublicAPIOption) Discoverer { + api := &publicAPIDiscoverer{ + apiBase: PublicAPIHost, + tailnet: tailnet, + token: token, + } + for _, opt := range opts { + opt(api) + } + if api.client == nil { + api.client = defaultHTTPClient + } + return api +} diff --git a/publicapi_test.go b/publicapi_test.go new file mode 100644 index 0000000..a5d30ba --- /dev/null +++ b/publicapi_test.go @@ -0,0 +1,148 @@ +package tailscalesd + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" +) + +func apiBaseForTest(tb testing.TB, surl string) string { + tb.Helper() + u, err := url.Parse(surl) + if err != nil { + tb.Fatal(err) + } + return u.Host +} + +func TestPublicAPIDiscovererDevices(t *testing.T) { + var wantPath = "/api/v2/tailnet/testTailnet/devices" + for tn, tc := range map[string]struct { + responder func(w http.ResponseWriter) + wantErr error + want []Device + }{ + "returns failed request error when the server responds unsuccessfully": { + responder: func(w http.ResponseWriter) { + w.WriteHeader(http.StatusInternalServerError) + }, + wantErr: errFailedAPIRequest, + }, + "returns failed request error when the server responds with bad payload": { + responder: func(w http.ResponseWriter) { + w.Header().Set("Content-Type", "text/plain") + fmt.Fprintln(w, "This is decidedly not JSON.") + }, + wantErr: errFailedAPIRequest, + }, + "returns devices when the server responds with valid JSON": { + responder: func(w http.ResponseWriter) { + w.Header().Set("Content-Type", "application/json; encoding=utf-8") + _, _ = w.Write([]byte(`{"devices": [{"hostname":"testhostname","os":"beos"}]}`)) + }, + want: []Device{ + { + Hostname: "testhostname", + OS: "beos", + Tailnet: "testTailnet", + }, + }, + }, + } { + t.Run(tn, func(t *testing.T) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got, want := r.URL.Path, wantPath; got != want { + t.Errorf("Devices: request URL path mismatch: got: %q want: %q", got, want) + } + tc.responder(w) + })) + defer server.Close() + + d := PublicAPI("testTailnet", "testToken", WithHTTPClient(server.Client()), WithAPIHost(apiBaseForTest(t, server.URL))) + got, err := d.Devices(context.TODO()) + if got, want := err, tc.wantErr; !errors.Is(got, want) { + t.Errorf("Devices: error mismatch: got: %q want: %q", got, want) + } + // Ignore the API field, which will be set to the arbitrary test + // server's host:port. + if diff := cmp.Diff(got, tc.want, cmpopts.IgnoreFields(Device{}, "API")); diff != "" { + t.Errorf("PublicAPI: mismatch (-got, +want):\n%v", diff) + } + }) + } +} + +func TestWithAPIHost(t *testing.T) { + want := "test.example.com" + var got publicAPIDiscoverer + WithAPIHost(want)(&got) + if got.apiBase != want { + t.Errorf("WithAPIHost: apiBase mismatch: got: %q want: %q", got.apiBase, want) + } +} + +func TestWithHTTPClient(t *testing.T) { + want := &http.Client{} + var got publicAPIDiscoverer + WithHTTPClient(want)(&got) + if got.client != want { + t.Errorf("WithHTTPClient: client mismatch: got: %+v want: %+v", got.client, want) + } +} + +type publicAPIOptTester struct { + called int +} + +func (t *publicAPIOptTester) Opt() PublicAPIOption { + return func(_ *publicAPIDiscoverer) { + t.called++ + } +} + +func publicAPIDiscovererComparer(l, r *publicAPIDiscoverer) bool { + return l.client == r.client && + l.apiBase == r.apiBase && + l.tailnet == r.tailnet && + l.token == r.token +} + +func TestPublicAPISetsDefaults(t *testing.T) { + got, ok := PublicAPI("testTailnet", "testToken").(*publicAPIDiscoverer) + if !ok { + t.Fatalf("PublicAPI: type mismatch: the Discoverer returned by PublicAPI() was not a *publicAPIDiscoverer") + } + want := &publicAPIDiscoverer{ + client: defaultHTTPClient, + apiBase: PublicAPIHost, + tailnet: "testTailnet", + token: "testToken", + } + if diff := cmp.Diff(got, want, cmp.Comparer(publicAPIDiscovererComparer)); diff != "" { + t.Errorf("PublicAPI: mismatch (-got, +want):\n%v", diff) + } +} + +func TestPublicAPIInvokesAllOptionsExactlyOnce(t *testing.T) { + optTesters := make([]publicAPIOptTester, 25) + + opts := make([]PublicAPIOption, len(optTesters)) + for i := range optTesters { + opts[i] = optTesters[i].Opt() + } + + _ = PublicAPI("ignored", "ignored", opts...) + + for i := range optTesters { + if got, want := optTesters[i].called, 1; got != want { + t.Errorf("PublicAPI: option call mismatch: got: %d want: %d", got, want) + } + } +} diff --git a/ratelimited.go b/ratelimited.go new file mode 100644 index 0000000..4e57ec4 --- /dev/null +++ b/ratelimited.go @@ -0,0 +1,49 @@ +package tailscalesd + +import ( + "context" + "errors" + "fmt" + "sync" + "time" +) + +var errStaleResults = errors.New("stale discovery results") + +// RateLimitedDiscoverer wraps a Discoverer and limits calls to it to be no more +// frequent than once per Frequency, returning cached values if more frequent +// calls are made. +type RateLimitedDiscoverer struct { + Wrap Discoverer + Frequency time.Duration + + mu sync.RWMutex // protects following members + earliest time.Time + last []Device +} + +func (c *RateLimitedDiscoverer) refreshDevices(ctx context.Context) ([]Device, error) { + devices, err := c.Wrap.Devices(ctx) + if err != nil { + return devices, fmt.Errorf("%w: %v", errStaleResults, err) + } + + c.mu.Lock() + defer c.mu.Unlock() + c.last = devices + c.earliest = time.Now().Add(c.Frequency) + return devices, nil +} + +func (c *RateLimitedDiscoverer) Devices(ctx context.Context) ([]Device, error) { + c.mu.RLock() + expired := time.Now().After(c.earliest) + last := make([]Device, len(c.last)) + _ = copy(last, c.last) + c.mu.RUnlock() + + if expired { + return c.refreshDevices(ctx) + } + return last, nil +} diff --git a/ratelimited_test.go b/ratelimited_test.go new file mode 100644 index 0000000..c142736 --- /dev/null +++ b/ratelimited_test.go @@ -0,0 +1,118 @@ +package tailscalesd + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/google/go-cmp/cmp" +) + +var devicesForRatelimitedTest = []Device{ + { + Addresses: []string{ + "100.2.3.4", + "fd7a::1234", + }, + API: "foo.example.com", + ClientVersion: "420.69", + Hostname: "somethingclever", + ID: "id", + Name: "somethingclever", + OS: "beos", + Tailnet: "example@gmail.com", + Tags: []string{ + "tag:foo", + "tag:bar", + }, + }, +} + +func discovererForTest(tb testing.TB) *testDiscoverer { + tb.Helper() + return &testDiscoverer{ + discovered: devicesForRatelimitedTest, + } +} + +type rateLimitedDiscovererTestWant struct { + called int + err error + devices []Device +} + +// TestRateLimitedDiscoverer tests the RateLimitedDiscoverer's behavior. It uses +// time math, but limits the chances of flakiness by using increments of 30 +// hours. If this test takes more than 30 hours to run, it may fail erroneously. +// If this test takes more than 30 hours to run, you need a new computer. +func TestRateLimitedDiscoverer(t *testing.T) { + for tn, tc := range map[string]struct { + discoverer *RateLimitedDiscoverer + wrapped *testDiscoverer + want rateLimitedDiscovererTestWant + }{ + "rate limited discoverer which has never been used calls Discover": { + discoverer: &RateLimitedDiscoverer{}, + wrapped: discovererForTest(t), + want: rateLimitedDiscovererTestWant{ + called: 1, + devices: devicesForRatelimitedTest, + }, + }, + "rate limited discoverer which is expired calls Discover": { + discoverer: &RateLimitedDiscoverer{ + earliest: time.Now().Add(-30 * time.Hour), + }, + wrapped: discovererForTest(t), + want: rateLimitedDiscovererTestWant{ + called: 1, + devices: devicesForRatelimitedTest, + }, + }, + "rate limited discoverer which is not expired returns cached results": { + discoverer: &RateLimitedDiscoverer{ + earliest: time.Now().Add(30 * time.Hour), + last: []Device{ + {ID: "ratelimittest"}, + }, + }, + wrapped: discovererForTest(t), + want: rateLimitedDiscovererTestWant{ + devices: []Device{ + {ID: "ratelimittest"}, + }, + }, + }, + "rate limited discoverer which is expired returns cached results on error": { + discoverer: &RateLimitedDiscoverer{ + earliest: time.Now().Add(30 * time.Hour), + last: []Device{ + {ID: "ratelimittest"}, + }, + }, + wrapped: &testDiscoverer{ + err: errors.New("this is a test error"), + }, + want: rateLimitedDiscovererTestWant{ + devices: []Device{ + {ID: "ratelimittest"}, + }, + }, + }, + } { + t.Run(tn, func(t *testing.T) { + tc.discoverer.Wrap = tc.wrapped + got, err := tc.discoverer.Devices(context.TODO()) + if !errors.Is(err, tc.want.err) { + t.Errorf("RateLimitedDiscoverer: unexpected error: %v", err) + } + if got, want := tc.wrapped.Called, tc.want.called; got != want { + t.Errorf("RateLimitedDiscoverer: mismatched Discover call count: got: %d want: %d", got, want) + } + if diff := cmp.Diff(got, tc.want.devices); diff != "" { + t.Errorf("RateLimitedDiscoverer: mismatch (-got, +want):\n%v", diff) + } + }) + } +} diff --git a/tailscalesd.go b/tailscalesd.go index a99914d..c2777c4 100644 --- a/tailscalesd.go +++ b/tailscalesd.go @@ -1,38 +1,27 @@ -// Package tailscalesd provides Prometheus Service Discovery for Tailscale. +// Package tailscalesd provides Prometheus Service Discovery for Tailscale using +// a naive, bespoke Tailscale API client supporting both the public v2 and local +// APIs. It has only the functionality needed for tailscalesd. You should not +// be tempted to use it for anything else. package tailscalesd import ( "bytes" "context" "encoding/json" - "errors" "fmt" "io" "log" "net" "net/http" - "sync" - "time" - - "github.com/cfunkhouser/tailscalesd/internal/tailscale" - "github.com/cfunkhouser/tailscalesd/internal/tailscale/local" - "github.com/cfunkhouser/tailscalesd/internal/tailscale/public" ) -// TargetDescriptor as Prometheus expects it. For more details, see -// https://prometheus.io/docs/prometheus/latest/http_sd/. -type TargetDescriptor struct { - Targets []string `json:"targets"` - Labels map[string]string `json:"labels,omitempty"` -} - const ( // LabelMetaAPI is the host which provided the details about this device. // Will be "localhost" for the local API. LabelMetaAPI = "__meta_tailscale_api" - // LabelMetaDeviceAuthorized is whether the target is currently authorized on the Tailnet. - // Will always be true when using the local API. + // LabelMetaDeviceAuthorized is whether the target is currently authorized + // on the Tailnet. Will always be true when using the local API. LabelMetaDeviceAuthorized = "__meta_tailscale_device_authorized" // LabelMetaDeviceClientVersion is the Tailscale client version in use on @@ -63,20 +52,32 @@ const ( LabelMetaTailnet = "__meta_tailscale_tailnet" ) -// filterEmpty removes entries in a map which have either an empty key or empty -// value. -func filterEmpty(in map[string]string) map[string]string { - if in == nil { - return nil - } - filtered := make(map[string]string) - for k, v := range in { - if k == "" || v == "" { - continue - } - filtered[k] = v - } - return filtered +// Device in a Tailnet, as reported by one of the various Tailscale APIs. +type Device struct { + Addresses []string `json:"addresses"` + API string `json:"api"` + Authorized bool `json:"authorized"` + ClientVersion string `json:"clientVersion,omitempty"` + Hostname string `json:"hostname"` + ID string `json:"id"` + Name string `json:"name"` + OS string `json:"os"` + Tailnet string `json:"tailnet"` + Tags []string `json:"tags"` +} + +// Discoverer of things exposed by the various Tailscale APIs. +type Discoverer interface { + // Devices reported by the Tailscale public API as belonging to the + // configured tailnet. + Devices(context.Context) ([]Device, error) +} + +// TargetDescriptor as Prometheus expects it. For more details, see +// https://prometheus.io/docs/prometheus/latest/http_sd/. +type TargetDescriptor struct { + Targets []string `json:"targets"` + Labels map[string]string `json:"labels,omitempty"` } type filter func(TargetDescriptor) TargetDescriptor @@ -86,7 +87,10 @@ func filterIPv6Addresses(td TargetDescriptor) TargetDescriptor { for _, target := range td.Targets { ip := net.ParseIP(target) if ip == nil { - // target is not a valid IP address of any version. + // target is not a valid IP address of any version, but this filter + // is explicitly for IPv6 addresses, so we leave the garbage in + // place. + targets = append(targets, target) continue } if ipv4 := ip.To4(); ipv4 != nil { @@ -99,15 +103,31 @@ func filterIPv6Addresses(td TargetDescriptor) TargetDescriptor { } } +// excludeEmptyMapEntries removes entries in a map which have either an empty +// key or empty value. +func excludeEmptyMapEntries(in map[string]string) map[string]string { + if in == nil { + return nil + } + filtered := make(map[string]string) + for k, v := range in { + if k == "" || v == "" { + continue + } + filtered[k] = v + } + return filtered +} + func filterEmptyLabels(td TargetDescriptor) TargetDescriptor { return TargetDescriptor{ Targets: td.Targets, - Labels: filterEmpty(td.Labels), + Labels: excludeEmptyMapEntries(td.Labels), } } // translate Devices to Prometheus TargetDescriptor, filtering empty labels. -func translate(devices []tailscale.Device, filters ...filter) (found []TargetDescriptor) { +func translate(devices []Device, filters ...filter) (found []TargetDescriptor) { for _, d := range devices { target := TargetDescriptor{ Targets: d.Addresses, @@ -143,157 +163,56 @@ func translate(devices []tailscale.Device, filters ...filter) (found []TargetDes return } -type tailscaleAPI interface { - Devices(context.Context) ([]tailscale.Device, error) -} - -type discoverer struct { - ts tailscaleAPI -} - -// DiscoverDevices in a tailnet. -func (d *discoverer) DiscoverDevices(ctx context.Context) ([]TargetDescriptor, error) { - devices, err := d.ts.Devices(ctx) - if err != nil { - return nil, err - } - return translate(devices, filterEmptyLabels, filterIPv6Addresses), nil -} - -var ErrStaleResults = errors.New("potentially stale results") - -type rateLimitingDiscoverer struct { - sync.RWMutex - discoverer Discoverer - freq time.Duration - - // protected - lastDevices []TargetDescriptor - earliest time.Time -} - -func (d *rateLimitingDiscoverer) refreshDevices(ctx context.Context) ([]TargetDescriptor, error) { - log.Printf("Refreshing Devices") - devices, err := d.discoverer.DiscoverDevices(ctx) - if err != nil { - return devices, err - } - - d.Lock() - defer d.Unlock() - - d.lastDevices = devices - d.earliest = time.Now().Add(d.freq) - log.Printf("Device refresh successful. Next refresh no sooner than %v", d.earliest.Format(time.RFC3339)) - return devices, nil -} - -func (d *rateLimitingDiscoverer) DiscoverDevices(ctx context.Context) ([]TargetDescriptor, error) { - d.RLock() - expired := time.Now().After(d.earliest) - last := make([]TargetDescriptor, len(d.lastDevices)) - _ = copy(last, d.lastDevices) - d.RUnlock() - - if expired { - devices, err := d.refreshDevices(ctx) - if err != nil { - log.Printf("Failed refreshing: %v", err) - return last, ErrStaleResults - } - return devices, nil - } - return last, nil -} - -func defaultHTTPClient() *http.Client { - return &http.Client{ - Timeout: time.Second * 10, - Transport: &http.Transport{ - Dial: (&net.Dialer{ - Timeout: 5 * time.Second, - }).Dial, - TLSHandshakeTimeout: 5 * time.Second, - }, - } -} - -func UsingLocalAPI() tailscaleAPI { - // TODO(cfunkhouser): Make this configurable. - return local.New("/run/tailscale/tailscaled.sock") -} - -func UsingPublicAPI(tailnet, token string) tailscaleAPI { - return &public.API{ - Client: defaultHTTPClient(), - APIBase: tailscale.PublicAPI, - Tailnet: tailnet, - Token: token, - } -} - -type Option func(Discoverer) Discoverer - -func WithRateLimit(freq time.Duration) Option { - return func(d Discoverer) Discoverer { - return &rateLimitingDiscoverer{ - discoverer: d, - freq: freq, - } - } -} - -// Discoverer of things in a tailnet. -type Discoverer interface { - DiscoverDevices(ctx context.Context) ([]TargetDescriptor, error) -} - -func New(api tailscaleAPI, opts ...Option) Discoverer { - var d Discoverer = &discoverer{ - ts: api, - } - for _, opt := range opts { - d = opt(d) - } - return d +type discoveryHandler struct { + ts Discoverer + filters []filter } -// discoveryHandler is a http.Handler that exposes the SD payload. It caches the -// last valid payload for a fixed period of time to prevent hammering Tailscale's -// API. -type discoveryHandler struct { - d Discoverer +func serveAndLog(w io.Writer, msg string) { + log.Print(msg) + fmt.Fprint(w, msg) } func (h *discoveryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - targets, err := h.d.DiscoverDevices(r.Context()) + if h == nil || h.ts == nil { + w.WriteHeader(http.StatusInternalServerError) + serveAndLog(w, "Attempted to serve with an improperly initialized handler.") + return + } + devices, err := h.ts.Devices(r.Context()) if err != nil { - if err != ErrStaleResults { + if err != errStaleResults { w.WriteHeader(http.StatusInternalServerError) - log.Printf("Failed to discover Tailscale devices: %v", err) - fmt.Fprintf(w, "Failed to discover Tailscale devices: %v", err) + serveAndLog(w, fmt.Sprintf("Failed to discover Tailscale devices: %v", err)) return } + // TODO(cfunkhouser): Investigate whether Prometheus respects cache + // control headers, and implement accordingly here. log.Print("Serving potentially stale results") } + targets := translate(devices, h.filters...) var buf bytes.Buffer if err := json.NewEncoder(&buf).Encode(targets); err != nil { w.WriteHeader(http.StatusInternalServerError) - log.Printf("Failed encoding targets to JSON: %v", err) - fmt.Fprintf(w, "Failed encoding targets to JSON: %v", err) + serveAndLog(w, fmt.Sprintf("Failed encoding targets to JSON: %v", err)) return } - w.Header().Add("Content-Type", "application/json") + w.Header().Add("Content-Type", "application/json; charset=utf-8") if _, err := io.Copy(w, &buf); err != nil { - // The transaction with the client is already started, so there's nothing - // graceful to do here. Log any errors for troubleshooting later. + // The transaction with the client is already started, so there's + // nothing graceful to do here. Log any errors for troubleshooting + // later. log.Printf("Failed sending JSON payload to the client: %v", err) } } -// Export the Discoverer as a http.Handler. -func Export(d Discoverer, pollFrequency time.Duration) http.Handler { - return &discoveryHandler{d} +// Export the Tailscale Discoverer for Service Discovery via HTTP. +func Export(ts Discoverer) http.Handler { + return &discoveryHandler{ + ts: ts, + // TODO(cfunkhouser): Make these filters configurable. + filters: []filter{filterEmptyLabels, filterIPv6Addresses}, + } } diff --git a/tailscalesd_test.go b/tailscalesd_test.go index 9d370ee..568b3ff 100644 --- a/tailscalesd_test.go +++ b/tailscalesd_test.go @@ -1,12 +1,25 @@ package tailscalesd import ( + "context" + "errors" + "io/ioutil" + "log" + "net/http" + "net/http/httptest" + "os" "testing" "github.com/google/go-cmp/cmp" ) -func TestFilterEmpty(t *testing.T) { +func TestMain(m *testing.M) { + // No log output during test runs. + log.SetOutput(ioutil.Discard) + os.Exit(m.Run()) +} + +func TestExcludeEmptyMapEntries(t *testing.T) { for tn, tc := range map[string]struct { in map[string]string want map[string]string @@ -42,9 +55,327 @@ func TestFilterEmpty(t *testing.T) { }, } { t.Run(tn, func(t *testing.T) { - got := filterEmpty(tc.in) - if diff := cmp.Diff(tc.want, got); diff != "" { - t.Errorf("filterEmpty: mismatch (-got +want):\n%v", diff) + got := excludeEmptyMapEntries(tc.in) + if diff := cmp.Diff(got, tc.want); diff != "" { + t.Errorf("excludeEmptyMapEntries: mismatch (-got, +want):\n%v", diff) + } + }) + } +} + +func TestFilterIPv6Addresses(t *testing.T) { + for tn, tc := range map[string]struct { + descriptor TargetDescriptor + want TargetDescriptor + }{ + "zero": {}, + "leaves ipv4 addresses alone": { + descriptor: TargetDescriptor{ + Targets: []string{"100.2.3.4", "100.5.6.7"}, + }, + want: TargetDescriptor{ + Targets: []string{"100.2.3.4", "100.5.6.7"}, + }, + }, + "leaves ipv4 addresses alone while removing ipv6 addresses": { + descriptor: TargetDescriptor{ + Targets: []string{"100.2.3.4", "100.5.6.7", "fd7a::1234", "fd7a::5678"}, + }, + want: TargetDescriptor{ + Targets: []string{"100.2.3.4", "100.5.6.7"}, + }, + }, + "leaves garbage alone without panicking or whatever": { + descriptor: TargetDescriptor{ + Targets: []string{"100.2.3.4", "GARBAGE"}, + }, + want: TargetDescriptor{ + Targets: []string{"100.2.3.4", "GARBAGE"}, + }, + }, + "leaves garbage alone without panicking while removing ipv6 addresses": { + descriptor: TargetDescriptor{ + Targets: []string{"100.2.3.4", "GARBAGE", "fd7a::1234"}, + }, + want: TargetDescriptor{ + Targets: []string{"100.2.3.4", "GARBAGE"}, + }, + }, + } { + t.Run(tn, func(t *testing.T) { + got := filterIPv6Addresses(tc.descriptor) + if diff := cmp.Diff(got, tc.want); diff != "" { + t.Errorf("filterIPv6Addresses: mismatch (-got, +want):\n%v", diff) + } + }) + } +} + +func TestTranslate(t *testing.T) { + for tn, tc := range map[string]struct { + devices []Device + filters []filter + want []TargetDescriptor + }{ + "zero": {}, + "single device without tags expands to single descriptor": { + devices: []Device{ + { + Addresses: []string{ + "100.2.3.4", + "fd7a::1234", + }, + API: "foo.example.com", + ClientVersion: "420.69", + Hostname: "somethingclever", + ID: "id", + Name: "somethingclever", + OS: "beos", + Tailnet: "example@gmail.com", + }, + }, + want: []TargetDescriptor{ + { + Targets: []string{"100.2.3.4", "fd7a::1234"}, + Labels: map[string]string{ + "__meta_tailscale_api": "foo.example.com", + "__meta_tailscale_device_authorized": "false", + "__meta_tailscale_device_client_version": "420.69", + "__meta_tailscale_device_hostname": "somethingclever", + "__meta_tailscale_device_id": "id", + "__meta_tailscale_device_name": "somethingclever", + "__meta_tailscale_device_os": "beos", + "__meta_tailscale_tailnet": "example@gmail.com", + }, + }, + }, + }, + "single device with two tags expands to two descriptors": { + devices: []Device{ + { + Addresses: []string{ + "100.2.3.4", + "fd7a::1234", + }, + API: "foo.example.com", + ClientVersion: "420.69", + Hostname: "somethingclever", + ID: "id", + Name: "somethingclever", + OS: "beos", + Tailnet: "example@gmail.com", + Tags: []string{ + "tag:foo", + "tag:bar", + }, + }, + }, + want: []TargetDescriptor{ + { + Targets: []string{"100.2.3.4", "fd7a::1234"}, + Labels: map[string]string{ + "__meta_tailscale_api": "foo.example.com", + "__meta_tailscale_device_authorized": "false", + "__meta_tailscale_device_client_version": "420.69", + "__meta_tailscale_device_hostname": "somethingclever", + "__meta_tailscale_device_id": "id", + "__meta_tailscale_device_name": "somethingclever", + "__meta_tailscale_device_os": "beos", + "__meta_tailscale_device_tag": "tag:foo", + "__meta_tailscale_tailnet": "example@gmail.com", + }, + }, + { + Targets: []string{"100.2.3.4", "fd7a::1234"}, + Labels: map[string]string{ + "__meta_tailscale_api": "foo.example.com", + "__meta_tailscale_device_authorized": "false", + "__meta_tailscale_device_client_version": "420.69", + "__meta_tailscale_device_hostname": "somethingclever", + "__meta_tailscale_device_id": "id", + "__meta_tailscale_device_name": "somethingclever", + "__meta_tailscale_device_os": "beos", + "__meta_tailscale_device_tag": "tag:bar", + "__meta_tailscale_tailnet": "example@gmail.com", + }, + }, + }, + }, + "filters apply to all descriptors expanded from device": { + devices: []Device{ + { + Addresses: []string{ + "100.2.3.4", + "fd7a::1234", + }, + API: "foo.example.com", + ClientVersion: "420.69", + Hostname: "somethingclever", + ID: "id", + Name: "somethingclever", + OS: "beos", + Tailnet: "example@gmail.com", + Tags: []string{ + "tag:foo", + "tag:bar", + }, + }, + }, + filters: []filter{ + func(in TargetDescriptor) TargetDescriptor { + in.Labels["test_label"] = "IT WORKED" + return in + }, + }, + want: []TargetDescriptor{ + { + Targets: []string{"100.2.3.4", "fd7a::1234"}, + Labels: map[string]string{ + "__meta_tailscale_api": "foo.example.com", + "__meta_tailscale_device_authorized": "false", + "__meta_tailscale_device_client_version": "420.69", + "__meta_tailscale_device_hostname": "somethingclever", + "__meta_tailscale_device_id": "id", + "__meta_tailscale_device_name": "somethingclever", + "__meta_tailscale_device_os": "beos", + "__meta_tailscale_device_tag": "tag:foo", + "__meta_tailscale_tailnet": "example@gmail.com", + "test_label": "IT WORKED", + }, + }, + { + Targets: []string{"100.2.3.4", "fd7a::1234"}, + Labels: map[string]string{ + "__meta_tailscale_api": "foo.example.com", + "__meta_tailscale_device_authorized": "false", + "__meta_tailscale_device_client_version": "420.69", + "__meta_tailscale_device_hostname": "somethingclever", + "__meta_tailscale_device_id": "id", + "__meta_tailscale_device_name": "somethingclever", + "__meta_tailscale_device_os": "beos", + "__meta_tailscale_device_tag": "tag:bar", + "__meta_tailscale_tailnet": "example@gmail.com", + "test_label": "IT WORKED", + }, + }, + }, + }, + } { + t.Run(tn, func(t *testing.T) { + got := translate(tc.devices, tc.filters...) + if diff := cmp.Diff(got, tc.want); diff != "" { + t.Errorf("translate: mismatch (-got, +want):\n%v", diff) + } + }) + } +} + +type testDiscoverer struct { + Called int + discovered []Device + err error +} + +func (t *testDiscoverer) Devices(_ context.Context) ([]Device, error) { + t.Called++ + return t.discovered, t.err +} + +type httpWant struct { + code int + contentType string + body string // string for prettier cmp diff output. +} + +func TestDiscoveryHandler(t *testing.T) { + for tn, tc := range map[string]struct { + discoverer Discoverer + want httpWant + }{ + "nil": { + want: httpWant{ + code: http.StatusInternalServerError, + body: "Attempted to serve with an improperly initialized handler.", + }, + }, + "unspecified API error": { + discoverer: &testDiscoverer{ + err: errors.New("this is a test error"), + }, + want: httpWant{ + code: http.StatusInternalServerError, + body: "Failed to discover Tailscale devices: this is a test error", + }, + }, + "stale results are still served": { + discoverer: &testDiscoverer{ + discovered: []Device{ + { + Addresses: []string{ + "100.2.3.4", + "fd7a::1234", + }, + API: "foo.example.com", + ClientVersion: "420.69", + Hostname: "somethingclever", + ID: "id", + Name: "somethingclever", + OS: "beos", + Tailnet: "example@gmail.com", + Tags: []string{ + "tag:foo", + "tag:bar", + }, + }, + }, + err: errStaleResults, + }, + want: httpWant{ + code: http.StatusOK, + contentType: "application/json; charset=utf-8", + body: `[{"targets":["100.2.3.4"],"labels":{"__meta_tailscale_api":"foo.example.com","__meta_tailscale_device_authorized":"false","__meta_tailscale_device_client_version":"420.69","__meta_tailscale_device_hostname":"somethingclever","__meta_tailscale_device_id":"id","__meta_tailscale_device_name":"somethingclever","__meta_tailscale_device_os":"beos","__meta_tailscale_device_tag":"tag:foo","__meta_tailscale_tailnet":"example@gmail.com"}},{"targets":["100.2.3.4"],"labels":{"__meta_tailscale_api":"foo.example.com","__meta_tailscale_device_authorized":"false","__meta_tailscale_device_client_version":"420.69","__meta_tailscale_device_hostname":"somethingclever","__meta_tailscale_device_id":"id","__meta_tailscale_device_name":"somethingclever","__meta_tailscale_device_os":"beos","__meta_tailscale_device_tag":"tag:bar","__meta_tailscale_tailnet":"example@gmail.com"}}]` + "\n", + }, + }, + "results with no errors are served": { + discoverer: &testDiscoverer{ + discovered: []Device{ + { + Addresses: []string{ + "100.2.3.4", + "fd7a::1234", + }, + API: "foo.example.com", + ClientVersion: "420.69", + Hostname: "somethingclever", + ID: "id", + Name: "somethingclever", + OS: "beos", + Tailnet: "example@gmail.com", + Tags: []string{ + "tag:foo", + "tag:bar", + }, + }, + }, + }, + want: httpWant{ + code: http.StatusOK, + contentType: "application/json; charset=utf-8", + body: `[{"targets":["100.2.3.4"],"labels":{"__meta_tailscale_api":"foo.example.com","__meta_tailscale_device_authorized":"false","__meta_tailscale_device_client_version":"420.69","__meta_tailscale_device_hostname":"somethingclever","__meta_tailscale_device_id":"id","__meta_tailscale_device_name":"somethingclever","__meta_tailscale_device_os":"beos","__meta_tailscale_device_tag":"tag:foo","__meta_tailscale_tailnet":"example@gmail.com"}},{"targets":["100.2.3.4"],"labels":{"__meta_tailscale_api":"foo.example.com","__meta_tailscale_device_authorized":"false","__meta_tailscale_device_client_version":"420.69","__meta_tailscale_device_hostname":"somethingclever","__meta_tailscale_device_id":"id","__meta_tailscale_device_name":"somethingclever","__meta_tailscale_device_os":"beos","__meta_tailscale_device_tag":"tag:bar","__meta_tailscale_tailnet":"example@gmail.com"}}]` + "\n", + }, + }, + } { + t.Run(tn, func(t *testing.T) { + r := httptest.NewRequest(http.MethodGet, "/", nil) + w := httptest.NewRecorder() + + Export(tc.discoverer).ServeHTTP(w, r) + + if w.Code != tc.want.code { + t.Errorf("discoveryHandler: status code mismatch: got: %v want: %v", w.Code, tc.want.code) + } + if diff := cmp.Diff(w.Body.String(), tc.want.body); diff != "" { + t.Errorf("discoveryHandler: content mismatch (-got, +want):\n%v", diff) } }) }