From d87c743ec239d2c8fc0d9b1b5642b7836e45547e Mon Sep 17 00:00:00 2001 From: Mike Long Date: Thu, 16 May 2024 14:07:15 -0700 Subject: [PATCH 1/3] Add a filesystem monitoring option for the snapshot paths command --- bin/flyio_env_report.sh | 2 ++ bin/test2 | 0 cmd/kosli/root.go | 1 + cmd/kosli/snapshotPaths.go | 64 ++++++++++++++++++++++++++++++++++++++ go.mod | 2 +- kosli-paths.yaml | 4 +++ 6 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 bin/test2 create mode 100644 kosli-paths.yaml diff --git a/bin/flyio_env_report.sh b/bin/flyio_env_report.sh index 9dcbf319e..09e554aaf 100755 --- a/bin/flyio_env_report.sh +++ b/bin/flyio_env_report.sh @@ -5,6 +5,8 @@ SCRIPT_NAME=flyio_env_report.sh USAGE="$SCRIPT_NAME [options] " + + die() { echo "Error: $1" >&2 diff --git a/bin/test2 b/bin/test2 new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/kosli/root.go b/cmd/kosli/root.go index 9748572db..577d00286 100644 --- a/cmd/kosli/root.go +++ b/cmd/kosli/root.go @@ -207,6 +207,7 @@ The service principal needs to have the following permissions: setTagsFlag = "[optional] The key-value pairs to tag the resource with. The format is: key=value" unsetTagsFlag = "[optional] The list of tag keys to remove from the resource." pathsSpecFileFlag = "The path to a paths file in YAML/JSON/TOML format. Cannot be used together with --path ." + pathsWatchFlag = "[optional] Watch the filesystem for changes and report snapshots of artifacts running in specific filesystem paths to Kosli." snapshotPathPathFlag = "The base path for the artifact to snapshot." snapshotPathExcludeFlag = "[optional] The comma-separated list of literal paths or glob patterns to exclude when fingerprinting the artifact." snapshotPathArtifactNameFlag = "The reported name of the artifact." diff --git a/cmd/kosli/snapshotPaths.go b/cmd/kosli/snapshotPaths.go index 7b76bc0ce..4a982310a 100644 --- a/cmd/kosli/snapshotPaths.go +++ b/cmd/kosli/snapshotPaths.go @@ -3,10 +3,12 @@ package main import ( "fmt" "io" + "log" "net/http" "path/filepath" "strings" + "github.com/fsnotify/fsnotify" "github.com/go-playground/validator/v10" "github.com/kosli-dev/cli/internal/requests" "github.com/kosli-dev/cli/internal/server" @@ -52,6 +54,7 @@ kosli snapshot paths yourEnvironmentName \ type snapshotPathsOptions struct { pathSpecFile string + watch bool } func newSnapshotPathsCmd(out io.Writer) *cobra.Command { @@ -76,6 +79,8 @@ func newSnapshotPathsCmd(out io.Writer) *cobra.Command { } cmd.Flags().StringVar(&o.pathSpecFile, "paths-file", "", pathsSpecFileFlag) + cmd.Flags().BoolVar(&o.watch, "watch", false, pathsWatchFlag) + addDryRunFlag(cmd) if err := RequireFlags(cmd, []string{"paths-file"}); err != nil { @@ -96,10 +101,21 @@ func (o *snapshotPathsOptions) run(args []string) error { return err } + // snapshot the paths + if o.watch { + watch_for_changes(ps, url, envName) + } + + return reportArtifacts(ps, url, envName) +} + +func reportArtifacts(ps *server.PathsSpec, url string, envName string) error { + artifacts, err := server.CreatePathsArtifactsData(ps, logger) if err != nil { return err } + payload := &server.ServerEnvRequest{ Artifacts: artifacts, } @@ -118,6 +134,54 @@ func (o *snapshotPathsOptions) run(args []string) error { return err } +func watch_for_changes(ps *server.PathsSpec, url string, envName string) bool { + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Fatal(err) + } + defer watcher.Close() + + if err := watcher.Add("./bin/"); err != nil { + logger.Error("failed to watch dir: %v", err) + } + + for { + select { + case event := <-watcher.Events: + + logger.Debug("Event: " + event.String()) + + if event.Op&fsnotify.Write == fsnotify.Write { + + logger.Debug("Modified file: " + event.Name) + reportArtifacts(ps, url, envName) + + } else if event.Op&fsnotify.Create == fsnotify.Create { + + logger.Debug("Created file: " + event.Name) + reportArtifacts(ps, url, envName) + + } else if event.Op&fsnotify.Remove == fsnotify.Remove { + + logger.Debug("Removed file: " + event.Name) + reportArtifacts(ps, url, envName) + + } else if event.Op&fsnotify.Rename == fsnotify.Rename { + + logger.Debug("Renamed file: " + event.Name) + reportArtifacts(ps, url, envName) + + } else { + logger.Debug("Other event: " + event.Name) + // ignore + } + + case err := <-watcher.Errors: + logger.Debug("Error: ", err) + } + } +} + func processPathSpecFile(pathsSpecFile string) (*server.PathsSpec, error) { var ps *server.PathsSpec v := viper.New() diff --git a/go.mod b/go.mod index 3df1b5649..1df3dcdec 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/s3 v1.31.0 github.com/aws/smithy-go v1.13.5 github.com/docker/docker v24.0.9+incompatible + github.com/fsnotify/fsnotify v1.7.0 github.com/go-git/go-billy/v5 v5.5.0 github.com/go-git/go-git/v5 v5.11.0 github.com/go-playground/validator/v10 v10.19.0 @@ -91,7 +92,6 @@ require ( github.com/evanphx/json-patch/v5 v5.2.0 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-logr/logr v1.4.1 // indirect diff --git a/kosli-paths.yaml b/kosli-paths.yaml new file mode 100644 index 000000000..5b7495b21 --- /dev/null +++ b/kosli-paths.yaml @@ -0,0 +1,4 @@ +version: 1 +artifacts: + service-a: + path: bin/ From 512907462ff2067a47fb45f548cf99a98bcfb2b6 Mon Sep 17 00:00:00 2001 From: Mike Long Date: Thu, 16 May 2024 15:20:09 -0700 Subject: [PATCH 2/3] Watch every path in the artifacts path file --- cmd/kosli/snapshotPaths.go | 36 ++++++++++-------------------------- kosli-paths.yaml | 2 ++ 2 files changed, 12 insertions(+), 26 deletions(-) diff --git a/cmd/kosli/snapshotPaths.go b/cmd/kosli/snapshotPaths.go index 4a982310a..c1d1efa32 100644 --- a/cmd/kosli/snapshotPaths.go +++ b/cmd/kosli/snapshotPaths.go @@ -141,39 +141,23 @@ func watch_for_changes(ps *server.PathsSpec, url string, envName string) bool { } defer watcher.Close() - if err := watcher.Add("./bin/"); err != nil { - logger.Error("failed to watch dir: %v", err) + // Add every path in path spec to watcher + for _, pathSpec := range ps.Artifacts { + if err := watcher.Add(pathSpec.Path); err != nil { + logger.Error("failed to watch dir: %v", err) + } } for { select { case event := <-watcher.Events: - logger.Debug("Event: " + event.String()) - if event.Op&fsnotify.Write == fsnotify.Write { - - logger.Debug("Modified file: " + event.Name) - reportArtifacts(ps, url, envName) - - } else if event.Op&fsnotify.Create == fsnotify.Create { - - logger.Debug("Created file: " + event.Name) - reportArtifacts(ps, url, envName) - - } else if event.Op&fsnotify.Remove == fsnotify.Remove { - - logger.Debug("Removed file: " + event.Name) - reportArtifacts(ps, url, envName) - - } else if event.Op&fsnotify.Rename == fsnotify.Rename { - - logger.Debug("Renamed file: " + event.Name) - reportArtifacts(ps, url, envName) - - } else { - logger.Debug("Other event: " + event.Name) - // ignore + if event.Op != fsnotify.Chmod { + err := reportArtifacts(ps, url, envName) + if err != nil { + logger.Error("failed to report artifacts: %v", err) + } } case err := <-watcher.Errors: diff --git a/kosli-paths.yaml b/kosli-paths.yaml index 5b7495b21..fd765b113 100644 --- a/kosli-paths.yaml +++ b/kosli-paths.yaml @@ -2,3 +2,5 @@ version: 1 artifacts: service-a: path: bin/ + service-b: + path: hack/ From e59ffb71a12c395f21626f8502a08346d2ca7036 Mon Sep 17 00:00:00 2001 From: Mike Long Date: Thu, 16 May 2024 16:50:11 -0700 Subject: [PATCH 3/3] Perform initial snapshot before watching the filesystem for changes --- cmd/kosli/snapshotPaths.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/kosli/snapshotPaths.go b/cmd/kosli/snapshotPaths.go index c1d1efa32..d23485452 100644 --- a/cmd/kosli/snapshotPaths.go +++ b/cmd/kosli/snapshotPaths.go @@ -103,6 +103,11 @@ func (o *snapshotPathsOptions) run(args []string) error { // snapshot the paths if o.watch { + err := reportArtifacts(ps, url, envName) + if err != nil { + return err + } + watch_for_changes(ps, url, envName) }