Create gRPC server and client for dynamic reaper configuration (#8)

* Add client and resource files and respective build files

* Add gazelle prefix

* Add client interface and stub for GCE client

* Add gcp client interface and gce client

* Change proto package

* Updated dependencies

* Create GCP client for GCE

* Add files to help with testing

* Create outline for unit tests

* Update build file for test dependencies

* Add regex filtering

* Create framework for testing get resources

* Add test cases for get resources for GCE client

* Add delete resource test, and begin documenting test file

* Add documentation, update build files and delete unnecessary test files

* Fix tiny indentenation mistake

* Add client and resource files and respective build files

* Add gazelle prefix

* Add client interface and stub for GCE client

* Add gcp client interface and gce client

* Updated dependencies

* Create GCP client for GCE

* Add files to help with testing

* Create outline for unit tests

* Update build file for test dependencies

* Add regex filtering

* Create framework for testing get resources

* Add test cases for get resources for GCE client

* Add delete resource test, and begin documenting test file

* Add documentation, update build files and delete unnecessary test files

* Fix tiny indentenation mistake

* Change indententation again

* Begin outlining Reaper logic

* Continue work on Reaper

* Pass context as Auth parameter

* Add test for name and skip filtering

* Add copyright info to resources test file

* Updated go module files

* Fixed bug in test file

* Clean up test resources file

* Fix overengineering of error handling and logging in reaper

* Begin work on reaper unit tests

* Continue adding to reaper tests

* Add test case for checking TTL of watched resource

* Add test logic for update reaper

* Add test cases for reaper update

* Add test logic for reaper run

* Add mock clock to WatchedResource for testing

* Configure reaper tests to run with mock clock

* Add more test cases for reaper

* Remove test files and comments

* Clean up code, update bazel files, and add documentation

* Fix deleted comment

* Refactor everything to use pointers

* Add logging for when resource is deleted

* Add integration test outline

* Work on integration test

* Update gitignore

* Add methods to help build protos

* Work on handling duplicate watched resources

* Handle duplicate watched resources

* Restructure integration test and do some cleaning up

* Fix small bugs

* Update build files and add documentation

* Create reaper manager

* Refactor reaper and finish manager logic

* Add unit test for running reaper on schedule

* Add reaper manager integration test

* Refactor reaper to allow for updating watching resources

* Add checks to integration test

* Refactor update config to not make any web calls, and give that functionality to GetResources

* Update tests for reaper refactor

* Add documentation

* Add delete and list methods to reaper manager

* Fix logging bug

* Add framework for testing reaper manager

* Remove skip filter from reaperconfig proto

* Add reaper manager tests

* Refactor manager and add test for monitoring reapers

* Deleted unnecessary testing files

* Add gcs client, and implement client interface

* Add unit test for auth and delete for gcs client

* Begin adding support for monitoring GCS buckets and objects separately

* Finish GCS client and begin work on tests

* Add GCS to factory method, and refactor GCE for consistant project structure

* Return error instead of log.Fatal

* Return error when parsing schedule

* Begin working on gRPC server

* Continue work on grpc interface

* Finish grpc server

* Finish gRPC server and client

* Add command line arguement for client

* Finish client command line tool

* Add unit tests, fix concurency bugs

* Fix minor bugs, add documentation, clean up code

* Fix build files

* Modified test

* Fix mistake in BUILD file

* Add documentation to client gRPC tools

* Add documentation to proto

* Fix trimming mistake, and syntax error

* Add terminal prompt for uuid when deleting reaper

Co-authored-by: Brian <bmodel@google.com>

client/BUILD.bazel

@@ -0,0 +1,13 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["client.go"],
+    importpath = "github.com/googleinterns/cloudai-gcp-test-resource-reaper/client",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//proto:go_default_library",
+        "@io_bazel_rules_go//proto/wkt:empty_go_proto",
+        "@org_golang_google_grpc//:go_default_library",
+    ],
+)

client/client.go

@@ -0,0 +1,97 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package client
+
+import (
+	"context"
+	"fmt"
+	"log"
+
+	"github.com/golang/protobuf/ptypes/empty"
+	"github.com/googleinterns/cloudai-gcp-test-resource-reaper/reaperconfig"
+	"google.golang.org/grpc"
+)
+
+// ReaperClient is a gRPC client for communicating with the reaper manager server.
+type ReaperClient struct {
+	client reaperconfig.ReaperManagerClient
+	conn   *grpc.ClientConn
+	ctx    context.Context
+}
+
+// StartClient returns a client with a gRPC connection with the server running on address:port.
+func StartClient(ctx context.Context, address, port string) *ReaperClient {
+	conn, err := grpc.Dial(fmt.Sprintf("%s:%s", address, port), grpc.WithInsecure())
+	if err != nil {
+		log.Fatal(err)
+	}
+	reaperManagerClient := reaperconfig.NewReaperManagerClient(conn)
+
+	return &ReaperClient{
+		client: reaperManagerClient,
+		conn:   conn,
+		ctx:    ctx,
+	}
+}
+
+// AddReaper adds a new reaper the reaper manager.
+func (c *ReaperClient) AddReaper(config *reaperconfig.ReaperConfig) (string, error) {
+	res, err := c.client.AddReaper(c.ctx, config)
+	return res.Uuid, err
+}
+
+// UpdateReaper updates the reaper from the given ReaperConfig. If the UUID in the config does not exist,
+// an error will be thrown.
+func (c *ReaperClient) UpdateReaper(config *reaperconfig.ReaperConfig) (string, error) {
+	res, err := c.client.UpdateReaper(c.ctx, config)
+	return res.Uuid, err
+}
+
+// DeleteReaper deletes the reaper with the given UUID.
+func (c *ReaperClient) DeleteReaper(uuid string) error {
+	_, err := c.client.DeleteReaper(c.ctx, &reaperconfig.Reaper{Uuid: uuid})
+	return err
+}
+
+// ListRunningReapers returns a list of the running reapers' UUIDs.
+func (c *ReaperClient) ListRunningReapers() ([]string, error) {
+	res, err := c.client.ListRunningReapers(c.ctx, new(empty.Empty))
+	if err != nil {
+		return nil, err
+	}
+	var runningReapers []string
+	for _, reaper := range res.Reapers {
+		runningReapers = append(runningReapers, reaper.Uuid)
+	}
+	return runningReapers, nil
+}
+
+// StartManager starts running the reaper manager. Note this is different from starting the gRPC
+// server.
+func (c *ReaperClient) StartManager() error {
+	_, err := c.client.StartManager(c.ctx, new(empty.Empty))
+	return err
+}
+
+// ShutdownManager stops the reaper manager process. Note this does not shut down the gRPC server.
+func (c *ReaperClient) ShutdownManager() error {
+	_, err := c.client.ShutdownManager(c.ctx, new(empty.Empty))
+	return err
+}
+
+// Close closes the ReaperClient connection to the gRPC server.
+func (c *ReaperClient) Close() {
+	c.conn.Close()
+}

cmd/reaper/BUILD.bazel

@@ -0,0 +1,19 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    importpath = "github.com/googleinterns/cloudai-gcp-test-resource-reaper/cmd/reaper",
+    visibility = ["//visibility:private"],
+    deps = [
+        "//client:go_default_library",
+        "//pkg/reaper:go_default_library",
+        "//proto:go_default_library",
+    ],
+)
+
+go_binary(
+    name = "reaper",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)

cmd/reaper/main.go

@@ -0,0 +1,211 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bufio"
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/googleinterns/cloudai-gcp-test-resource-reaper/client"
+	"github.com/googleinterns/cloudai-gcp-test-resource-reaper/pkg/reaper"
+	"github.com/googleinterns/cloudai-gcp-test-resource-reaper/reaperconfig"
+)
+
+func main() {
+	deleteCmd := flag.NewFlagSet("delete", flag.ExitOnError)
+	deleteUUID := deleteCmd.String("uuid", "", "UUID of the reaper")
+
+	if len(os.Args) < 2 {
+		fmt.Println("expected 'create', 'update', 'list', 'delete', 'start', or 'shutdown' commands")
+		os.Exit(1)
+	}
+
+	reaperClient := client.StartClient(context.Background(), "localhost", "8000")
+	defer reaperClient.Close()
+
+	switch os.Args[1] {
+	case "create":
+		config, err := createReaperConfigPrompt()
+		if err != nil {
+			fmt.Println("Creating reaper config failed with the following error: ", err.Error())
+			os.Exit(1)
+		}
+		uuid, err := reaperClient.AddReaper(config)
+		if err != nil {
+			fmt.Println("Create reaper failed with following error: ", err.Error())
+			os.Exit(1)
+		}
+		fmt.Printf("Reaper with UUID %s successfully created\n", uuid)
+
+	case "update":
+		config, err := createReaperConfigPrompt()
+		if err != nil {
+			fmt.Println("Creating reaper config failed with the following error: ", err.Error())
+			os.Exit(1)
+		}
+		uuid, err := reaperClient.UpdateReaper(config)
+		if err != nil {
+			fmt.Println("Create reaper failed with following error: ", err.Error())
+			os.Exit(1)
+		}
+		fmt.Printf("Reaper with UUID %s successfully updated\n", uuid)
+
+	case "list":
+		reapers, err := reaperClient.ListRunningReapers()
+		if err != nil {
+			fmt.Println("List reapers failed with following error: ", err.Error())
+			os.Exit(1)
+		}
+		fmt.Println("Running Reaper UUIDs: ", strings.Join(reapers, ", "))
+
+	case "delete":
+		deleteCmd.Parse(os.Args[2:])
+		if len(*deleteUUID) == 0 {
+			var err error
+			reader := bufio.NewReader(os.Stdin)
+			fmt.Print("Reaper UUID: ")
+			*deleteUUID, err = reader.ReadString('\n')
+			if err != nil {
+				log.Fatal(err)
+			}
+			*deleteUUID = strings.TrimSuffix(*deleteUUID, "\n")
+		}
+		err := reaperClient.DeleteReaper(*deleteUUID)
+		if err != nil {
+			fmt.Println("Delete reapers failed with following error: ", err.Error())
+			os.Exit(1)
+		}
+		fmt.Printf("Reaper with UUID %s successfully deleted\n", *deleteUUID)
+
+	case "start":
+		err := reaperClient.StartManager()
+		if err != nil {
+			fmt.Println("Start manager failed with following error: ", err.Error())
+		}
+		fmt.Println("Reaper manager started")
+
+	case "shutdown":
+		err := reaperClient.ShutdownManager()
+		if err != nil {
+			fmt.Println("Shutdown manager failed with following error: ", err.Error())
+		}
+		fmt.Println("Reaper manager shutdown")
+
+	default:
+		fmt.Println("expected 'create', 'update', 'list', 'delete', 'start', or 'shutdown' commands")
+		os.Exit(1)
+	}
+}
+
+// createReaperConfigPrompt is a command line prompt that walks the user through creating
+// a new reaper config.
+func createReaperConfigPrompt() (*reaperconfig.ReaperConfig, error) {
+	reader := bufio.NewReader(os.Stdin)
+	fmt.Println("Reaper Configuration Setup")
+
+	fmt.Print("Reaper UUID: ")
+	uuid, err := reader.ReadString('\n')
+	if err != nil {
+		return nil, err
+	}
+	uuid = strings.TrimSuffix(uuid, "\n")
+
+	fmt.Print("Project ID: ")
+	projectID, err := reader.ReadString('\n')
+	if err != nil {
+		return nil, err
+	}
+	projectID = strings.TrimSuffix(projectID, "\n")
+
+	fmt.Print("Reaper run schedule (in cron time string format): ")
+	schedule, err := reader.ReadString('\n')
+	if err != nil {
+		return nil, err
+	}
+	schedule = strings.TrimSuffix(schedule, "\n")
+
+	var resources []*reaperconfig.ResourceConfig
+	for {
+		fmt.Print("Add another resource? (y/n): ")
+		response, err := reader.ReadString('\n')
+		if err != nil {
+			return nil, err
+		}
+		if len(response) == 1 || response[0] == 'n' {
+			break
+		}
+
+		fmt.Print("Resource type: ")
+		resourceTypeString, err := reader.ReadString('\n')
+		if err != nil {
+			return nil, err
+		}
+		resourceTypeString = strings.TrimSuffix(resourceTypeString, "\n")
+		var resourceType reaperconfig.ResourceType
+
+		switch resourceTypeString {
+		case "GCE_VM":
+			resourceType = reaperconfig.ResourceType_GCE_VM
+		case "GCS_Bucket":
+			resourceType = reaperconfig.ResourceType_GCS_BUCKET
+		case "GCS_Object":
+			resourceType = reaperconfig.ResourceType_GCS_OBJECT
+		default:
+			return nil, fmt.Errorf("Invalid resource type %s", resourceTypeString)
+		}
+
+		fmt.Print("Zones (comma separated list): ")
+		zonesString, err := reader.ReadString('\n')
+		if err != nil {
+			return nil, err
+		}
+		zonesString = strings.TrimSuffix(zonesString, "\n")
+		zones := strings.Split(zonesString, ",")
+
+		fmt.Print("Name filter: ")
+		nameFilter, err := reader.ReadString('\n')
+		if err != nil {
+			return nil, err
+		}
+		nameFilter = strings.TrimSuffix(nameFilter, "\n")
+
+		fmt.Print("Skip filter: ")
+		skipFilter, err := reader.ReadString('\n')
+		if err != nil {
+			return nil, err
+		}
+		skipFilter = strings.TrimSuffix(skipFilter, "\n")
+
+		fmt.Print("TTL (in cron time string format): ")
+		ttl, err := reader.ReadString('\n')
+		if err != nil {
+			return nil, err
+		}
+		ttl = strings.TrimSuffix(ttl, "\n")
+
+		resources = append(
+			resources,
+			reaper.NewResourceConfig(resourceType, zones, nameFilter, skipFilter, ttl),
+		)
+	}
+
+	config := reaper.NewReaperConfig(resources, schedule, projectID, uuid)
+	return config, nil
+}

cmd/start_server/BUILD.bazel

@@ -0,0 +1,15 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    importpath = "github.com/googleinterns/cloudai-gcp-test-resource-reaper/cmd/start_server",
+    visibility = ["//visibility:private"],
+    deps = ["//pkg/manager:go_default_library"],
+)
+
+go_binary(
+    name = "start_server",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)

cmd/start_server/main.go

@@ -0,0 +1,23 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"github.com/googleinterns/cloudai-gcp-test-resource-reaper/pkg/manager"
+)
+
+func main() {
+	manager.StartServer("localhost", "8000")
+}