From 60cae777b52ab45e2e79a19139dbae077abfe1d9 Mon Sep 17 00:00:00 2001 From: Brian Model Date: Tue, 4 Aug 2020 18:26:39 +0000 Subject: [PATCH] Add cloud logging (#9) * Add client and resource files and respective build files * Add gazelle prefix * Add client interface and stub for GCE client * Add gcp client interface and gce client * Change proto package * Updated dependencies * Create GCP client for GCE * Add files to help with testing * Create outline for unit tests * Update build file for test dependencies * Add regex filtering * Create framework for testing get resources * Add test cases for get resources for GCE client * Add delete resource test, and begin documenting test file * Add documentation, update build files and delete unnecessary test files * Fix tiny indentenation mistake * Add client and resource files and respective build files * Add gazelle prefix * Add client interface and stub for GCE client * Add gcp client interface and gce client * Updated dependencies * Create GCP client for GCE * Add files to help with testing * Create outline for unit tests * Update build file for test dependencies * Add regex filtering * Create framework for testing get resources * Add test cases for get resources for GCE client * Add delete resource test, and begin documenting test file * Add documentation, update build files and delete unnecessary test files * Fix tiny indentenation mistake * Change indententation again * Begin outlining Reaper logic * Continue work on Reaper * Pass context as Auth parameter * Add test for name and skip filtering * Add copyright info to resources test file * Updated go module files * Fixed bug in test file * Clean up test resources file * Fix overengineering of error handling and logging in reaper * Begin work on reaper unit tests * Continue adding to reaper tests * Add test case for checking TTL of watched resource * Add test logic for update reaper * Add test cases for reaper update * Add test logic for reaper run * Add mock clock to WatchedResource for testing * Configure reaper tests to run with mock clock * Add more test cases for reaper * Remove test files and comments * Clean up code, update bazel files, and add documentation * Fix deleted comment * Refactor everything to use pointers * Add logging for when resource is deleted * Add integration test outline * Work on integration test * Update gitignore * Add methods to help build protos * Work on handling duplicate watched resources * Handle duplicate watched resources * Restructure integration test and do some cleaning up * Fix small bugs * Update build files and add documentation * Create reaper manager * Refactor reaper and finish manager logic * Add unit test for running reaper on schedule * Add reaper manager integration test * Refactor reaper to allow for updating watching resources * Add checks to integration test * Refactor update config to not make any web calls, and give that functionality to GetResources * Update tests for reaper refactor * Add documentation * Add delete and list methods to reaper manager * Fix logging bug * Add framework for testing reaper manager * Remove skip filter from reaperconfig proto * Add reaper manager tests * Refactor manager and add test for monitoring reapers * Deleted unnecessary testing files * Add gcs client, and implement client interface * Add unit test for auth and delete for gcs client * Begin adding support for monitoring GCS buckets and objects separately * Finish GCS client and begin work on tests * Add GCS to factory method, and refactor GCE for consistant project structure * Return error instead of log.Fatal * Return error when parsing schedule * Begin working on gRPC server * Continue work on grpc interface * Finish grpc server * Finish gRPC server and client * Add command line arguement for client * Finish client command line tool * Add unit tests, fix concurency bugs * Fix minor bugs, add documentation, clean up code * Add basic logging functions * Fix build files * Modified test * Fix mistake in BUILD file * Add custom logger interface * Refactor existing logging to use custom logger * Implement cloud logging configured with flags * Clean up logger code * Clean up and add license header * Make logging thread safe, and fix tests * Delete unnecessary file * Fix merge mistake in proto * Change command line naming and add message when using cloud logging * add error check when creating cloud logger Co-authored-by: Brian --- WORKSPACE | 14 +++ cmd/start_server/BUILD.bazel | 5 +- cmd/start_server/logs.txt | 0 cmd/start_server/main.go | 25 ++++- go.mod | 1 + go.sum | 7 ++ pkg/logger/BUILD.bazel | 9 ++ pkg/logger/logger.go | 152 +++++++++++++++++++++++++++++ pkg/manager/BUILD.bazel | 1 + pkg/manager/manager_server.go | 14 ++- pkg/manager/reaper_manager.go | 17 ++-- pkg/manager/reaper_manager_test.go | 5 + pkg/reaper/BUILD.bazel | 1 + pkg/reaper/reaper.go | 33 ++++--- pkg/reaper/reaper_test.go | 5 + 15 files changed, 261 insertions(+), 28 deletions(-) create mode 100644 cmd/start_server/logs.txt create mode 100644 pkg/logger/BUILD.bazel create mode 100644 pkg/logger/logger.go diff --git a/WORKSPACE b/WORKSPACE index f4a3642..a582f91 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -537,3 +537,17 @@ go_repository( sum = "h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=", version = "v3.0.1", ) + +go_repository( + name = "com_github_googleapis_google_cloud_go_testing", + importpath = "github.com/googleapis/google-cloud-go-testing", + sum = "h1:YBqybTXA//1pltKcwyntNQdgDw6AnA5oHZCXFOiZhoo=", + version = "v0.0.0-20191008195207-8e1d251e947d", +) + +go_repository( + name = "com_google_cloud_go_logging", + importpath = "cloud.google.com/go/logging", + sum = "h1:kaunpnoEh9L4hu6JUsBa8Y20LBfKnCuDhKUgdZp7oK8=", + version = "v1.0.0", +) diff --git a/cmd/start_server/BUILD.bazel b/cmd/start_server/BUILD.bazel index f301d21..3cddb74 100644 --- a/cmd/start_server/BUILD.bazel +++ b/cmd/start_server/BUILD.bazel @@ -5,7 +5,10 @@ go_library( srcs = ["main.go"], importpath = "github.com/googleinterns/cloudai-gcp-test-resource-reaper/cmd/start_server", visibility = ["//visibility:private"], - deps = ["//pkg/manager:go_default_library"], + deps = [ + "//pkg/logger:go_default_library", + "//pkg/manager:go_default_library", + ], ) go_binary( diff --git a/cmd/start_server/logs.txt b/cmd/start_server/logs.txt new file mode 100644 index 0000000..e69de29 diff --git a/cmd/start_server/main.go b/cmd/start_server/main.go index 6a10d8a..c567e5a 100644 --- a/cmd/start_server/main.go +++ b/cmd/start_server/main.go @@ -15,9 +15,32 @@ package main import ( + "context" + "flag" + "log" + + "github.com/googleinterns/cloudai-gcp-test-resource-reaper/pkg/logger" "github.com/googleinterns/cloudai-gcp-test-resource-reaper/pkg/manager" ) func main() { - manager.StartServer("localhost", "8000") + port := flag.String("port", "8000", "port to run gRPC server on") + projectID := flag.String("project-id", "", "GCP Project ID for where to store logs") + logsName := flag.String("logs-name", "", "name of logs") + + flag.Parse() + + if err := logger.CreateLogger(); err != nil { + log.Fatal(err) + } + defer logger.Close() + if len(*projectID) > 0 && len(*logsName) > 0 { + err := logger.AddCloudLogger(context.Background(), *projectID, *logsName) + if err != nil { + log.Fatal(err) + } + logger.Logf("Logging to %s in project", *logsName, *projectID) + } + + manager.StartServer(*port) } diff --git a/go.mod b/go.mod index 5f53def..c174d9e 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/googleinterns/cloudai-gcp-test-resource-reaper go 1.14 require ( + cloud.google.com/go/logging v1.0.0 cloud.google.com/go/storage v1.6.0 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 github.com/golang/protobuf v1.4.2 diff --git a/go.sum b/go.sum index f0b0066..1068640 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.43.0/go.mod h1:BOSR3VbTLkk6FDC/TcffxP4NF/FFBGA5ku+jvKOP7pg= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= @@ -11,11 +12,14 @@ cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.56.0 h1:WRz29PgAsVEyPSDHyk+0fpEkwEFyfhHn+JbksT6gIL4= cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.60.0 h1:R+tDlceO7Ss+zyvtsdhTxacDyZ1k99xwskQ4FT7ruoM= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/logging v1.0.0 h1:kaunpnoEh9L4hu6JUsBa8Y20LBfKnCuDhKUgdZp7oK8= +cloud.google.com/go/logging v1.0.0/go.mod h1:V1cc3ogwobYzQq5f2R7DS/GvRIrI4FKj01Gs5glwAls= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -173,6 +177,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -260,6 +265,8 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190708153700-3bdd9d9f5532/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= diff --git a/pkg/logger/BUILD.bazel b/pkg/logger/BUILD.bazel new file mode 100644 index 0000000..c50526e --- /dev/null +++ b/pkg/logger/BUILD.bazel @@ -0,0 +1,9 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["logger.go"], + importpath = "github.com/googleinterns/cloudai-gcp-test-resource-reaper/pkg/logger", + visibility = ["//visibility:public"], + deps = ["@com_google_cloud_go_logging//:go_default_library"], +) diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go new file mode 100644 index 0000000..79aa0fa --- /dev/null +++ b/pkg/logger/logger.go @@ -0,0 +1,152 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logger + +import ( + "context" + "fmt" + "log" + "os" + "sync" + + "cloud.google.com/go/logging" +) + +// Logger handles writing local logs to a file and cloud logs to Stackdriver. +type Logger struct { + *log.Logger + *CloudLogger + mux *sync.Mutex +} + +var logger *Logger + +// CreateLogger initializes the logger for the server. The logs will be written to a local +// file called logs.txt. +func CreateLogger() error { + logFile, err := os.OpenFile("logs.txt", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666) + if err != nil { + return err + } + fileLogger := log.New(logFile, "", log.Ldate|log.Ltime) + logger = &Logger{ + Logger: fileLogger, + CloudLogger: nil, + mux: &sync.Mutex{}, + } + return nil +} + +// Log outputs to the necessary logs. Arguments are handled in the manner of fmt.Println. +func Log(v ...interface{}) { + logger.mux.Lock() + logger.log(v...) + logger.mux.Unlock() +} + +// Logf takes a format string and message and writes it to the necessary logs. Arguments are +// handled in the manner of fmt.Printf. +func Logf(format string, v ...interface{}) { + logger.mux.Lock() + logger.logf(format, v...) + logger.mux.Unlock() +} + +// Error outputs an error to the necessary logs. +func Error(v ...interface{}) { + logger.mux.Lock() + logger.error(v...) + logger.mux.Unlock() +} + +// Close closes the logger. +func Close() { + if logger.CloudLogger != nil { + logger.CloudLogger.closeLogger() + } +} + +// AddCloudLogger adds stackdriver logging to the logger in the given project and log name. +func AddCloudLogger(ctx context.Context, projectID, loggerName string) error { + cloudLogger, err := createCloudLogger(ctx, projectID, loggerName) + if err != nil { + return err + } + logger.CloudLogger = cloudLogger + return nil +} + +func (l *Logger) log(v ...interface{}) { + l.Logger.Println(v...) + if l.CloudLogger != nil { + l.CloudLogger.log(v...) + } +} + +func (l *Logger) logf(format string, v ...interface{}) { + l.Logger.Printf(format, v...) + if l.CloudLogger != nil { + l.CloudLogger.logf(format, v...) + } +} + +func (l *Logger) error(v ...interface{}) { + l.Logger.Println(v...) + if l.CloudLogger != nil { + l.CloudLogger.error(v...) + } +} + +// CloudLogger handles writing logs to stackdriver. +type CloudLogger struct { + *logging.Logger + *logging.Client +} + +func createCloudLogger(ctx context.Context, projectID, loggerName string) (*CloudLogger, error) { + logClient, err := logging.NewClient(ctx, projectID) + if err != nil { + return nil, err + } + return &CloudLogger{ + Logger: logClient.Logger(loggerName), + Client: logClient, + }, nil +} + +func (l *CloudLogger) log(v ...interface{}) { + l.Logger.Log( + logging.Entry{Payload: fmt.Sprintln(v...)}, + ) +} + +func (l *CloudLogger) logf(format string, v ...interface{}) { + l.Logger.Log( + logging.Entry{Payload: fmt.Sprintf(format, v...)}, + ) +} + +func (l *CloudLogger) error(v ...interface{}) { + l.Logger.Log( + logging.Entry{ + Payload: fmt.Sprintln(v...), + Severity: logging.Error, + }, + ) +} + +func (l *CloudLogger) closeLogger() { + l.Client.Close() +} diff --git a/pkg/manager/BUILD.bazel b/pkg/manager/BUILD.bazel index 596f4d7..1216eba 100644 --- a/pkg/manager/BUILD.bazel +++ b/pkg/manager/BUILD.bazel @@ -9,6 +9,7 @@ go_library( importpath = "github.com/googleinterns/cloudai-gcp-test-resource-reaper/pkg/manager", visibility = ["//visibility:public"], deps = [ + "//pkg/logger:go_default_library", "//pkg/reaper:go_default_library", "//proto:go_default_library", "@io_bazel_rules_go//proto/wkt:empty_go_proto", diff --git a/pkg/manager/manager_server.go b/pkg/manager/manager_server.go index 2d3879a..35273a1 100644 --- a/pkg/manager/manager_server.go +++ b/pkg/manager/manager_server.go @@ -17,10 +17,11 @@ package manager import ( "context" "fmt" - "log" "net" + "os" "github.com/golang/protobuf/ptypes/empty" + "github.com/googleinterns/cloudai-gcp-test-resource-reaper/pkg/logger" "github.com/googleinterns/cloudai-gcp-test-resource-reaper/reaperconfig" "google.golang.org/api/option" "google.golang.org/grpc" @@ -34,13 +35,16 @@ type reaperManagerServer struct { } // StartServer starts the gRPC server listing on the given address and port. -func StartServer(address, port string, clientOptions ...option.ClientOption) { - lis, err := net.Listen("tcp", fmt.Sprintf("%s:%s", address, port)) +func StartServer(port string, clientOptions ...option.ClientOption) { + lis, err := net.Listen("tcp", fmt.Sprintf(":%s", port)) if err != nil { - log.Fatal(err) + logger.Error(err) + os.Exit(1) } - log.Printf("Starting gRPC Server on %s:%s\n", address, port) + logger.Logf("------------------ Starting gRPC Server on :%s ------------------\n", port) + defer logger.Log("------------------ Shutting down gRPC Server ------------------") + server := grpc.NewServer() reaperconfig.RegisterReaperManagerServer(server, &reaperManagerServer{}) server.Serve(lis) diff --git a/pkg/manager/reaper_manager.go b/pkg/manager/reaper_manager.go index 42e8347..0748755 100644 --- a/pkg/manager/reaper_manager.go +++ b/pkg/manager/reaper_manager.go @@ -17,10 +17,10 @@ package manager import ( "context" "fmt" - "log" "strings" "time" + "github.com/googleinterns/cloudai-gcp-test-resource-reaper/pkg/logger" "github.com/googleinterns/cloudai-gcp-test-resource-reaper/pkg/reaper" "github.com/googleinterns/cloudai-gcp-test-resource-reaper/reaperconfig" "google.golang.org/api/option" @@ -56,11 +56,11 @@ func NewReaperManager(ctx context.Context, clientOptions ...option.ClientOption) // should be stopped. Note that MonitorReapers should be called in a separate // goroutine. func (manager *ReaperManager) MonitorReapers() { - log.Println("Starting Reaper Manager") + logger.Log("Starting Reaper Manager") for { select { case <-manager.quit: - log.Println("Quitting reaper manager") + logger.Log("Quitting reaper manager") return default: manager.sweepReapers() @@ -76,19 +76,20 @@ func (manager *ReaperManager) sweepReapers() { select { case newReaper := <-manager.newReaper: manager.Reapers = append(manager.Reapers, newReaper) - log.Printf("Added new reaper with UUID: %s", newReaper.UUID) + logger.Logf("Added new reaper with UUID: %s", newReaper.UUID) case reaperUUID := <-manager.deleteReaper: deleteSuccess := manager.handleDeleteReaper(reaperUUID) if deleteSuccess { - log.Printf("Reaper with UUID %s successfully deleted", reaperUUID) + logger.Logf("Reaper with UUID %s successfully deleted", reaperUUID) } else { - log.Printf("Reaper with UUID %s does not exist", reaperUUID) + logger.Logf("Reaper with UUID %s does not exist", reaperUUID) } case newReaperConfig := <-manager.updateReaper: err := manager.handleUpdateReaper(newReaperConfig) if err != nil { - log.Println(err) + logger.Error(err) } + logger.Logf("Reaper with UUID %s successfully updated", newReaperConfig.Uuid) default: for _, reaper := range manager.Reapers { reaper.RunOnSchedule(manager.ctx, manager.clientOptions...) @@ -106,7 +107,7 @@ func (manager *ReaperManager) AddReaperFromConfig(newReaperConfig *reaperconfig. newReaper := reaper.NewReaper() err := newReaper.UpdateReaperConfig(newReaperConfig) if err != nil { - log.Printf("Error adding reaper: %v\n", err) + logger.Error(fmt.Errorf("error adding reaper: %v\n", err)) return } manager.newReaper <- newReaper diff --git a/pkg/manager/reaper_manager_test.go b/pkg/manager/reaper_manager_test.go index 0f1a2da..777aaf3 100644 --- a/pkg/manager/reaper_manager_test.go +++ b/pkg/manager/reaper_manager_test.go @@ -22,11 +22,16 @@ import ( "strings" "testing" + "github.com/googleinterns/cloudai-gcp-test-resource-reaper/pkg/logger" "github.com/googleinterns/cloudai-gcp-test-resource-reaper/pkg/reaper" "github.com/googleinterns/cloudai-gcp-test-resource-reaper/reaperconfig" "google.golang.org/api/option" ) +func init() { + logger.CreateLogger() +} + type OperationType int const ( diff --git a/pkg/reaper/BUILD.bazel b/pkg/reaper/BUILD.bazel index a18baf6..9e348a1 100644 --- a/pkg/reaper/BUILD.bazel +++ b/pkg/reaper/BUILD.bazel @@ -7,6 +7,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/clients:go_default_library", + "//pkg/logger:go_default_library", "//pkg/resources:go_default_library", "//proto:go_default_library", "@com_github_robfig_cron_v3//:go_default_library", diff --git a/pkg/reaper/reaper.go b/pkg/reaper/reaper.go index c25fbf1..c7e7c6d 100644 --- a/pkg/reaper/reaper.go +++ b/pkg/reaper/reaper.go @@ -17,10 +17,11 @@ package reaper import ( "context" "fmt" - "log" + "strings" "time" "github.com/googleinterns/cloudai-gcp-test-resource-reaper/pkg/clients" + "github.com/googleinterns/cloudai-gcp-test-resource-reaper/pkg/logger" "github.com/googleinterns/cloudai-gcp-test-resource-reaper/pkg/resources" "github.com/googleinterns/cloudai-gcp-test-resource-reaper/reaperconfig" "github.com/robfig/cron/v3" @@ -68,8 +69,10 @@ func NewReaper() *Reaper { func (reaper *Reaper) RunOnSchedule(ctx context.Context, clientOptions ...option.ClientOption) bool { nextRun := reaper.Schedule.Next(reaper.lastRun) if reaper.lastRun.IsZero() || reaper.Clock.Now().After(nextRun) || reaper.Clock.Now().Equal(nextRun) { - log.Printf("Running reaper with UUID: %s\n", reaper.UUID) + logger.Logf("Running reaper with UUID: %s\n", reaper.UUID) reaper.GetResources(ctx, clientOptions...) + + logger.Logf("Reaper %s sweeping through the following resources: %s", reaper.UUID, reaper.WatchlistString()) reaper.SweepThroughResources(ctx, clientOptions...) reaper.lastRun = reaper.Clock.Now() return true @@ -87,7 +90,7 @@ func (reaper *Reaper) SweepThroughResources(ctx context.Context, clientOptions . if watchedResource.IsReadyForDeletion() { resourceClient, err := getAuthedClient(ctx, reaper, watchedResource.Type, clientOptions...) if err != nil { - log.Println(err) + logger.Error(err) continue } @@ -96,10 +99,10 @@ func (reaper *Reaper) SweepThroughResources(ctx context.Context, clientOptions . "%s client failed to delete resource %s with the following error: %s", watchedResource.Type.String(), watchedResource.Name, err.Error(), ) - log.Println(deleteError) + logger.Error(deleteError) continue } - log.Printf( + logger.Logf( "Deleted %s resource %s in zone %s\n", watchedResource.Type.String(), watchedResource.Name, watchedResource.Zone, ) @@ -135,7 +138,7 @@ func (reaper *Reaper) GetResources(ctx context.Context, clientOptions ...option. resourceClient, err := getAuthedClient(ctx, reaper, resourceType, clientOptions...) if err != nil { - log.Println(err) + logger.Error(err) continue } @@ -145,7 +148,7 @@ func (reaper *Reaper) GetResources(ctx context.Context, clientOptions ...option. "%s client failed to get resources with the following error: %s", resourceType.String(), err.Error(), ) - log.Println(getResourcesError) + logger.Error(getResourcesError) continue } watchedResources := resources.CreateWatchlist(filteredResources, resourceConfig.GetTtl()) @@ -159,7 +162,7 @@ func (reaper *Reaper) GetResources(ctx context.Context, clientOptions ...option. if _, alreadyWatched := newWatchedResources[resource.Zone][resource.Name]; alreadyWatched { newTTL, err := maxTTL(resource, newWatchedResources[resource.Zone][resource.Name]) if err != nil { - log.Println(err) + logger.Error(err) continue } newWatchedResources[resource.Zone][resource.Name].TTL = newTTL @@ -177,13 +180,17 @@ func (reaper *Reaper) GetResources(ctx context.Context, clientOptions ...option. reaper.Watchlist = newWatchlist } -// PrintWatchlist neatly prints the reaper's Watchlist. -func (reaper *Reaper) PrintWatchlist() { - fmt.Print("Watchlist: ") +// WatchlistString returns a near sting of the reaper's Watchlist. +func (reaper *Reaper) WatchlistString() string { + var watchlistBuidler strings.Builder for _, resource := range reaper.Watchlist { - fmt.Printf("%s in %s, ", resource.Name, resource.Zone) + watchlistBuidler.WriteString(fmt.Sprintf("%s in %s, ", resource.Name, resource.Zone)) + } + watchlist := watchlistBuidler.String() + if len(watchlist) > 0 { + watchlist = watchlist[:len(watchlist)-2] } - fmt.Print("\n") + return watchlist } // NewReaperConfig constructs a new ReaperConfig. diff --git a/pkg/reaper/reaper_test.go b/pkg/reaper/reaper_test.go index 36b9321..f57435d 100644 --- a/pkg/reaper/reaper_test.go +++ b/pkg/reaper/reaper_test.go @@ -24,6 +24,7 @@ import ( "testing" "time" + "github.com/googleinterns/cloudai-gcp-test-resource-reaper/pkg/logger" "github.com/googleinterns/cloudai-gcp-test-resource-reaper/pkg/resources" "github.com/googleinterns/cloudai-gcp-test-resource-reaper/reaperconfig" "google.golang.org/api/option" @@ -41,6 +42,10 @@ var ( testData map[string]map[reaperconfig.ResourceType]map[string][]TestData ) +func init() { + logger.CreateLogger() +} + type ReaperRunTestCase struct { Watchlist []*resources.WatchedResource Expected *Reaper