test: add test for file-provider

internal/provider/file/file.go

@@ -12,6 +12,7 @@
 	"os"
 	"path/filepath"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"github.com/fsnotify/fsnotify"
@@ -31,6 +32,9 @@
 	logger         logr.Logger
 	watcher        filewatcher.FileWatcher
 	resourcesStore *resourcesStore
+
+	// ready indicates whether the provider can start watching filesystem events.
+	ready atomic.Bool
 }
 
 func New(svr *config.Server, resources *message.ProviderResources) (*Provider, error) {
@@ -58,7 +62,13 @@
 	}()
 
 	// Start runnable servers.
-	go p.startHealthProbeServer(ctx)
+	var readyzChecker healthz.Checker = func(req *http.Request) error {
+		if !p.ready.Load() {
+			return fmt.Errorf("file provider not ready yet")
+		}
+		return nil
+	}
+	go p.startHealthProbeServer(ctx, readyzChecker)
 
 	initDirs, initFiles := path.ListDirsAndFiles(p.paths)
 	// Initially load resources from paths on host.
@@ -83,7 +93,9 @@
 		}(ch)
 	}
 
+	p.ready.Store(true)
 	curDirs, curFiles := initDirs.Clone(), initFiles.Clone()
+	initFilesParent := path.GetParentDirs(initFiles.UnsortedList())
 	for {
 		select {
 		case <-ctx.Done():
@@ -102,29 +114,35 @@
 			// temporary file when file is saved. So the watcher will only receive:
 			// - Create event, with name "filename~".
 			// - Remove event, with name "filename", but the file actually exist.
-			if initFiles.Has(event.Name) {
+			if initFilesParent.Has(filepath.Dir(event.Name)) {
 				p.logger.Info("file changed", "op", event.Op, "name", event.Name)
 
 				// For Write event, the file definitely exist.
-				if event.Has(fsnotify.Write) {
+				if initFiles.Has(event.Name) && event.Has(fsnotify.Write) {
 					goto handle
 				}
 
-				_, err := os.Lstat(event.Name)
-				if err != nil && os.IsNotExist(err) {
-					curFiles.Delete(event.Name)
-				} else {
-					curFiles.Insert(event.Name)
+				// Iter over the watched files to see the different.
+				for f := range initFiles {
+					_, err := os.Lstat(f)
+					if err != nil {
+						if os.IsNotExist(err) {
+							curFiles.Delete(f)
+						} else {
+							p.logger.Error(err, "stat file error", "name", f)
+						}
+					} else {
+						curFiles.Insert(f)
+					}
 				}
 				goto handle
 			}
 
 			// Ignore the hidden or temporary file related change event under a directory.
-			if _, name := filepath.Split(event.Name); strings.HasPrefix(name, ".") ||
-				strings.HasSuffix(name, "~") {
+			if _, name := filepath.Split(event.Name); strings.HasPrefix(name, ".") || strings.HasSuffix(name, "~") {
 				continue
 			}
-			p.logger.Info("file changed", "op", event.Op, "name", event.Name)
+			p.logger.Info("file changed", "op", event.Op, "name", event.Name, "dir", filepath.Dir(event.Name))
 
 			switch event.Op {
 			case fsnotify.Create, fsnotify.Write, fsnotify.Remove:
@@ -142,7 +160,7 @@
 	}
 }
 
-func (p *Provider) startHealthProbeServer(ctx context.Context) {
+func (p *Provider) startHealthProbeServer(ctx context.Context, readyzChecker healthz.Checker) {
 	const (
 		readyzEndpoint  = "/readyz"
 		healthzEndpoint = "/healthz"
@@ -159,7 +177,7 @@
 
 	readyzHandler := &healthz.Handler{
 		Checks: map[string]healthz.Checker{
-			readyzEndpoint: healthz.Ping,
+			readyzEndpoint: readyzChecker,
 		},
 	}
 	mux.Handle(readyzEndpoint, http.StripPrefix(readyzEndpoint, readyzHandler))

internal/provider/file/file_test.go

@@ -0,0 +1,225 @@
+// Copyright Envoy Gateway Authors
+// SPDX-License-Identifier: Apache-2.0
+// The full text of the Apache license is available in the LICENSE file at
+// the root of the repo.
+
+package file
+
+import (
+	"context"
+	"html/template"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/stretchr/testify/require"
+	"sigs.k8s.io/yaml"
+
+	egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
+	"github.com/envoyproxy/gateway/internal/envoygateway/config"
+	"github.com/envoyproxy/gateway/internal/gatewayapi/resource"
+	"github.com/envoyproxy/gateway/internal/message"
+)
+
+const (
+	resourcesUpdateTimeout = 1 * time.Minute
+	resourcesUpdateTick    = 1 * time.Second
+)
+
+type resourcesParam struct {
+	GatewayClassName    string
+	GatewayName         string
+	GatewayListenerPort string
+	HTTPRouteName       string
+	BackendName         string
+}
+
+func newDefaultResourcesParam() *resourcesParam {
+	return &resourcesParam{
+		GatewayClassName:    "eg",
+		GatewayName:         "eg",
+		GatewayListenerPort: "8888",
+		HTTPRouteName:       "backend",
+		BackendName:         "backend",
+	}
+}
+
+func newFileProviderConfig(paths []string) (*config.Server, error) {
+	cfg, err := config.New()
+	if err != nil {
+		return nil, err
+	}
+
+	cfg.EnvoyGateway.Provider = &egv1a1.EnvoyGatewayProvider{
+		Type: egv1a1.ProviderTypeCustom,
+		Custom: &egv1a1.EnvoyGatewayCustomProvider{
+			Resource: egv1a1.EnvoyGatewayResourceProvider{
+				Type: egv1a1.ResourceProviderTypeFile,
+				File: &egv1a1.EnvoyGatewayFileResourceProvider{
+					Paths: paths,
+				},
+			},
+		},
+	}
+	return cfg, nil
+}
+
+func TestFileProvider(t *testing.T) {
+	watchFileBase, _ := os.MkdirTemp(os.TempDir(), "test-files-*")
+	watchFilePath := filepath.Join(watchFileBase, "test.yaml")
+	watchDirPath, _ := os.MkdirTemp(os.TempDir(), "test-dir-*")
+	// Prepare the watched test file.
+	writeResourcesFile(t, "testdata/resources.tmpl", watchFilePath, newDefaultResourcesParam())
+	require.FileExists(t, watchFilePath)
+	require.DirExists(t, watchDirPath)
+
+	cfg, err := newFileProviderConfig([]string{watchFilePath, watchDirPath})
+	require.NoError(t, err)
+	pResources := new(message.ProviderResources)
+	fp, err := New(cfg, pResources)
+	require.NoError(t, err)
+	// Start file provider.
+	go func() {
+		if err := fp.Start(context.Background()); err != nil {
+			t.Errorf("failed to start file provider: %v", err)
+		}
+	}()
+
+	// Wait for file provider to be ready.
+	waitFileProviderReady(t)
+
+	require.Equal(t, "gateway.envoyproxy.io/gatewayclass-controller", fp.resourcesStore.name)
+
+	t.Run("initial resource load", func(t *testing.T) {
+		require.NotZero(t, pResources.GatewayAPIResources.Len())
+		resources := pResources.GetResourcesByGatewayClass("eg")
+		require.NotNil(t, resources)
+
+		want := &resource.Resources{}
+		mustUnmarshal(t, "testdata/resources.all.yaml", want)
+
+		opts := []cmp.Option{
+			cmpopts.IgnoreFields(resource.Resources{}, "serviceMap"),
+			cmpopts.EquateEmpty(),
+		}
+		require.Empty(t, cmp.Diff(want, resources, opts...))
+	})
+
+	t.Run("rename the watched file then rename it back", func(t *testing.T) {
+		// Rename it
+		renameFilePath := filepath.Join(watchFileBase, "foobar.yaml")
+		err := os.Rename(watchFilePath, renameFilePath)
+		require.NoError(t, err)
+		require.Eventually(t, func() bool {
+			return pResources.GetResourcesByGatewayClass("eg") == nil
+		}, resourcesUpdateTimeout, resourcesUpdateTick)
+
+		// Rename it back
+		err = os.Rename(renameFilePath, watchFilePath)
+		require.NoError(t, err)
+		require.Eventually(t, func() bool {
+			return pResources.GetResourcesByGatewayClass("eg") != nil
+		}, resourcesUpdateTimeout, resourcesUpdateTick)
+
+		resources := pResources.GetResourcesByGatewayClass("eg")
+		want := &resource.Resources{}
+		mustUnmarshal(t, "testdata/resources.all.yaml", want)
+
+		opts := []cmp.Option{
+			cmpopts.IgnoreFields(resource.Resources{}, "serviceMap"),
+			cmpopts.EquateEmpty(),
+		}
+		require.Empty(t, cmp.Diff(want, resources, opts...))
+	})
+
+	t.Run("remove the watched file", func(t *testing.T) {
+		err := os.Remove(watchFilePath)
+		require.NoError(t, err)
+		require.Eventually(t, func() bool {
+			return pResources.GetResourcesByGatewayClass("eg") == nil
+		}, resourcesUpdateTimeout, resourcesUpdateTick)
+	})
+
+	t.Run("add a file in watched dir", func(t *testing.T) {
+		// Write a new file under watched directory.
+		newFilePath := filepath.Join(watchDirPath, "test.yaml")
+		writeResourcesFile(t, "testdata/resources.tmpl", newFilePath, newDefaultResourcesParam())
+
+		require.Eventually(t, func() bool {
+			return pResources.GetResourcesByGatewayClass("eg") != nil
+		}, resourcesUpdateTimeout, resourcesUpdateTick)
+
+		resources := pResources.GetResourcesByGatewayClass("eg")
+		want := &resource.Resources{}
+		mustUnmarshal(t, "testdata/resources.all.yaml", want)
+
+		opts := []cmp.Option{
+			cmpopts.IgnoreFields(resource.Resources{}, "serviceMap"),
+			cmpopts.EquateEmpty(),
+		}
+		require.Empty(t, cmp.Diff(want, resources, opts...))
+	})
+
+	t.Run("remove a file in watched dir", func(t *testing.T) {
+		newFilePath := filepath.Join(watchDirPath, "test.yaml")
+		err := os.Remove(newFilePath)
+		require.NoError(t, err)
+		require.Eventually(t, func() bool {
+			return pResources.GetResourcesByGatewayClass("eg") == nil
+		}, resourcesUpdateTimeout, resourcesUpdateTick)
+	})
+
+	t.Cleanup(func() {
+		_ = os.RemoveAll(watchFileBase)
+		_ = os.RemoveAll(watchDirPath)
+	})
+}
+
+func writeResourcesFile(t *testing.T, tmpl, dst string, params *resourcesParam) {
+	dstFile, err := os.Create(dst)
+	require.NoError(t, err)
+
+	// Write parameters into target file.
+	tmplFile, err := template.ParseFiles(tmpl)
+	require.NoError(t, err)
+
+	err = tmplFile.Execute(dstFile, params)
+	require.NoError(t, err)
+	require.NoError(t, dstFile.Close())
+}
+
+func waitFileProviderReady(t *testing.T) {
+	require.Eventually(t, func() bool {
+		resp, err := http.Get("http://localhost:8081/readyz")
+		if err != nil {
+			t.Logf("failed to get from heathlz server")
+			return false
+		}
+
+		body, err := io.ReadAll(resp.Body)
+		defer resp.Body.Close()
+		if err != nil {
+			t.Logf("failed to get body from response")
+			return false
+		}
+
+		if string(body) != "ok" {
+			t.Logf("the file provider is not ready yet")
+			return false
+		}
+		return true
+	}, 3*resourcesUpdateTimeout, resourcesUpdateTick)
+}
+
+func mustUnmarshal(t *testing.T, path string, out interface{}) {
+	t.Helper()
+
+	content, err := os.ReadFile(path)
+	require.NoError(t, err)
+	require.NoError(t, yaml.UnmarshalStrict(content, out, yaml.DisallowUnknownFields))
+}

internal/provider/file/testdata/resources.all.yaml

@@ -0,0 +1,62 @@
+backends:
+- kind: Backend
+  metadata:
+    creationTimestamp: null
+    name: backend
+    namespace: envoy-gateway-system
+  spec:
+    endpoints:
+    - ip:
+        address: 0.0.0.0
+        port: 3000
+  status: {}
+gatewayClass:
+  kind: GatewayClass
+  metadata:
+    creationTimestamp: null
+    name: eg
+    namespace: envoy-gateway-system
+  spec:
+    controllerName: gateway.envoyproxy.io/gatewayclass-controller
+  status: {}
+gateways:
+- kind: Gateway
+  metadata:
+    creationTimestamp: null
+    name: eg
+    namespace: envoy-gateway-system
+  spec:
+    gatewayClassName: eg
+    listeners:
+    - name: http
+      port: 8888
+      protocol: HTTP
+  status: {}
+httpRoutes:
+- kind: HTTPRoute
+  metadata:
+    creationTimestamp: null
+    name: backend
+    namespace: envoy-gateway-system
+  spec:
+    hostnames:
+    - www.example.com
+    parentRefs:
+    - name: eg
+    rules:
+    - backendRefs:
+      - group: gateway.envoyproxy.io
+        kind: Backend
+        name: backend
+      matches:
+      - path:
+          type: PathPrefix
+          value: /
+  status:
+    parents: null
+namespaces:
+- metadata:
+    creationTimestamp: null
+    name: envoy-gateway-system
+  spec: {}
+  status: {}