Add purge configuration and scheduled purger functionality

README.md

@@ -327,8 +327,33 @@ link:
     - backend-taskdef
 ```
 
-See "mirage link" section for details.
+See ["mirage link"](#mirage-link) for details.
 
+#### `purge` section
+
+`purge` section configures purge settings.
+
+```yaml
+purge:
+  schedule: "13 4 * * ? *" # cron expression
+  request:
+    duration: 86400
+    excludes:
+      - foo
+      - bar
+    exclude_tags:
+      - "branch:preview"
+    exclude_regexp: "^(foo|bar)"
+```
+
+The `schedule` is a cron expression to run the purge task.
+
+- mirage-ecs runs the purge task at the specified schedule.
+- The expression is the same as the Amazon EventBridge `cron()`.
+  - See the document of [Cron expressions](https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-scheduled-rule-pattern.html#eb-cron-expressions) for details.
+  - Timezone is depends on the ECS task timezone.
+
+The `request` section is the same as the `/api/purge` API. See [API Documents](#post-apipurge).
 
 #### `auth` section
 
@@ -358,7 +383,7 @@ auth:
     password: "{{ env `MIRAGE_PASSWORD` }}"
 ```
 
-#### `cookie_secret` section
+##### `cookie_secret` sub section
 
 `cookie_secret` section configures secret key for cookie authentication.
 
@@ -367,7 +392,7 @@ When you configure `cookie_secret`, mirage-ecs sets a cookie to the browser afte
 When `/api/*` is accessed, mirage-ecs does not set a cookie to the clients. The `/api/*`
 paths allow authentication by token only.
 
-##### `token` section
+##### `token` sub section
 
 `token` section configures token authentication. The token is passed by specfied HTTP header.
 
@@ -380,7 +405,7 @@ auth:
 
 This configuration requires `x-mirage-token: foobarbaz` HTTP header to access mirage-ecs.
 
-##### `basic` section
+##### `basic` sub section
 
 `basic` section configures HTTP Basic authentication.
 
@@ -393,7 +418,7 @@ auth:
 
 This configuration requires username and password to access mirage-ecs by Basic authentication.
 
-##### `amzn_oidc` section
+##### `amzn_oidc` sub section
 
 `amzn_oidc` section configures OIDC authentication by Application Load Balancer. See also [Authenticate users using an Application Load Balancer](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/listener-authenticate-users.html)
 
@@ -653,6 +678,8 @@ Query parameters:
 
 `/api/purge` terminates tasks that not be accessed in the specified duration.
 
+See also [purge section](#purge-section) of config file.
+
 #### Form parameters
 
 - `excludes`: subdomains of tasks to exclude termination. multiple values are allowed.

config.go

@@ -44,6 +44,7 @@ type Config struct {
 	ECS       ECSCfg     `yaml:"ecs"`
 	Link      Link       `yaml:"link"`
 	Auth      *Auth      `yaml:"auth"`
+	Purge     *Purge     `yaml:"purge"`
 
 	compatV1  bool
 	localMode bool
@@ -234,7 +235,8 @@ func NewConfig(ctx context.Context, p *ConfigParams) (*Config, error) {
 		ECS: ECSCfg{
 			Region: os.Getenv("AWS_REGION"),
 		},
-		Auth: nil,
+		Auth:  nil,
+		Purge: nil,
 
 		localMode: p.LocalMode,
 		compatV1:  p.CompatV1,
@@ -318,6 +320,12 @@ func NewConfig(ctx context.Context, p *ConfigParams) (*Config, error) {
 	if err := cfg.fillECSDefaults(ctx); err != nil {
 		slog.Warn(f("failed to fill ECS defaults: %s", err))
 	}
+
+	if cfg.Purge != nil {
+		if err := cfg.Purge.Validate(); err != nil {
+			return nil, fmt.Errorf("invalid purge config: %w", err)
+		}
+	}
 	return cfg, nil
 }
 

go.mod

@@ -20,12 +20,14 @@ require (
 	github.com/labstack/echo/v4 v4.11.1
 	github.com/methane/rproxy v0.0.0-20130309122237-aafd1c66433b
 	github.com/samber/lo v1.38.1
+	github.com/winebarrel/cronplan v1.10.1
 	golang.org/x/sync v0.3.0
 	gopkg.in/yaml.v2 v2.4.0
 )
 
 require (
 	github.com/BurntSushi/toml v1.3.2 // indirect
+	github.com/alecthomas/participle/v2 v2.1.0 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.13.27 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.5 // indirect

go.sum

@@ -2,6 +2,12 @@ github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8
 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/ReneKroon/ttlcache/v2 v2.11.0 h1:OvlcYFYi941SBN3v9dsDcC2N8vRxyHcCmJb3Vl4QMoM=
 github.com/ReneKroon/ttlcache/v2 v2.11.0/go.mod h1:mBxvsNY+BT8qLLd6CuAJubbKo6r0jh3nb5et22bbfGY=
+github.com/alecthomas/assert/v2 v2.3.0 h1:mAsH2wmvjsuvyBvAmCtm7zFsBlb8mIHx5ySLVdDZXL0=
+github.com/alecthomas/assert/v2 v2.3.0/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ=
+github.com/alecthomas/participle/v2 v2.1.0 h1:z7dElHRrOEEq45F2TG5cbQihMtNTv8vwldytDj7Wrz4=
+github.com/alecthomas/participle/v2 v2.1.0/go.mod h1:Y1+hAs8DHPmc3YUFzqllV+eSQ9ljPTk0ZkPMtEdAx2c=
+github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk=
+github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
 github.com/aws/aws-sdk-go-v2 v1.16.8/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw=
 github.com/aws/aws-sdk-go-v2 v1.19.0 h1:klAT+y3pGFBU/qVf1uzwttpBbiuozJYWzNLHioyDJ+k=
 github.com/aws/aws-sdk-go-v2 v1.19.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
@@ -82,6 +88,8 @@ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
+github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
+github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
@@ -127,13 +135,15 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
-github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
 github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
 github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
+github.com/winebarrel/cronplan v1.10.1 h1:sTnmKWpGjXr3tDpgSSTCohltaimpBNXW/LgFf0SEMwo=
+github.com/winebarrel/cronplan v1.10.1/go.mod h1:FXpmoZVzj9eZoyHe1lpUezcFL3Tk6p5OBSovWeHq4qY=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0=
 go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=

mirage.go

@@ -74,9 +74,10 @@ func (m *Mirage) Run(ctx context.Context) error {
 		}(v.ListenPort)
 	}
 
-	wg.Add(2)
+	wg.Add(3)
 	go m.syncECSToMirage(ctx, &wg)
 	go m.RunAccessCountCollector(ctx, &wg)
+	go m.RunScheduledPurger(ctx, &wg)
 	wg.Wait()
 	slog.Info("shutdown mirage-ecs")
 	select {
@@ -147,6 +148,32 @@ func (m *Mirage) RunAccessCountCollector(ctx context.Context, wg *sync.WaitGroup
 	}
 }
 
+func (m *Mirage) RunScheduledPurger(ctx context.Context, wg *sync.WaitGroup) {
+	defer wg.Done()
+	p := m.Config.Purge
+	if p == nil {
+		slog.Debug("Purge is not configured")
+		return
+	}
+	slog.Info(f("starting up RunScheduledPurger() schedule: %s", p.Cron.String()))
+	for {
+		now := time.Now().Add(time.Minute)
+		next := p.Cron.Next(now)
+		slog.Info(f("next purge invocation at: %s", next))
+		duration := time.Until(next)
+		select {
+		case <-ctx.Done():
+			slog.Info("RunScheduledPurger() is done")
+			return
+		case <-time.After(duration):
+			slog.Info("scheduled purge invoked")
+			if err := m.WebApi.purge(ctx, p.PurgeParams); err != nil {
+				slog.Warn(err.Error())
+			}
+		}
+	}
+}
+
 const (
 	CloudWatchMetricNameSpace = "mirage-ecs"
 	CloudWatchMetricName      = "RequestCount"
@@ -184,7 +211,7 @@ SYNC:
 		})
 		available := make(map[string]bool)
 		for _, info := range running {
-			slog.Debug(f("ruuning task %s", info.ID))
+			slog.Debug(f("running task %s", info.ID))
 			if info.IPAddress != "" {
 				available[info.SubDomain] = true
 				for name, port := range info.PortMap {

purge.go

@@ -0,0 +1,34 @@
+package mirageecs
+
+import (
+	"fmt"
+
+	"github.com/winebarrel/cronplan"
+)
+
+type Purge struct {
+	Schedule string           `json:"schedule" yaml:"schedule"`
+	Request  *APIPurgeRequest `json:"request" yaml:"request"`
+
+	PurgeParams *PurgeParams         `json:"-" yaml:"-"`
+	Cron        *cronplan.Expression `json:"-" yaml:"-"`
+}
+
+func (p *Purge) Validate() error {
+	cron, err := cronplan.Parse(p.Schedule)
+	if err != nil {
+		return fmt.Errorf("invalid schedule expression %s: %w", p.Schedule, err)
+	}
+	p.Cron = cron
+
+	if p.Request == nil {
+		return fmt.Errorf("purge request is required")
+	}
+	purgeParams, err := p.Request.Validate()
+	if err != nil {
+		return fmt.Errorf("invalid purge request: %w", err)
+	}
+	p.PurgeParams = purgeParams
+
+	return nil
+}

purge_test.go

@@ -0,0 +1,48 @@
+package mirageecs_test
+
+import (
+	"testing"
+	"time"
+
+	mirageecs "github.com/acidlemon/mirage-ecs/v2"
+	"github.com/kayac/go-config"
+)
+
+func TestPurgeConfig(t *testing.T) {
+	cfg := mirageecs.Config{}
+	err := config.LoadWithEnvBytes(&cfg, []byte(`
+purge:
+  schedule: "*/3 * * * ? *" # every 3 minutes
+  request:
+    duration: "300" # 5 minutes
+    excludes:
+      - "test"
+      - "test2"
+    exclude_tags:
+      - "DontPurge:true"
+    exclude_regexp: "te.t"
+`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := cfg.Purge.Validate(); err != nil {
+		t.Fatal(err)
+	}
+	now := time.Date(2024, 11, 7, 11, 22, 33, 0, time.UTC)
+	next := cfg.Purge.Cron.Next(now)
+	if next != time.Date(2024, 11, 7, 11, 24, 0, 0, time.UTC) {
+		t.Errorf("unexpected next time: %s", next)
+	}
+	if cfg.Purge.PurgeParams.Duration != time.Second * 300 {
+		t.Errorf("unexpected duration: %d", cfg.Purge.PurgeParams.Duration)
+	}
+	if len(cfg.Purge.PurgeParams.Excludes) != 2 {
+		t.Errorf("unexpected excludes: %v", cfg.Purge.PurgeParams.Excludes)
+	}
+	if len(cfg.Purge.PurgeParams.ExcludeTags) != 1 {
+		t.Errorf("unexpected exclude_tags: %v", cfg.Purge.PurgeParams.ExcludeTags)
+	}
+	if !cfg.Purge.PurgeParams.ExcludeRegexp.MatchString("test") {
+		t.Errorf("unexpected exclude_regexp: %v", cfg.Purge.PurgeParams.ExcludeRegexp)
+	}
+}

types.go

@@ -59,10 +59,10 @@ func (r *APILaunchRequest) MergeForm(form url.Values) {
 }
 
 type APIPurgeRequest struct {
-	Duration      json.Number `json:"duration" form:"duration"`
-	Excludes      []string    `json:"excludes" form:"excludes"`
-	ExcludeTags   []string    `json:"exclude_tags" form:"exclude_tags"`
-	ExcludeRegexp string      `json:"exclude_regexp" form:"exclude_regexp"`
+	Duration      json.Number `json:"duration" form:"duration" yaml:"duration"`
+	Excludes      []string    `json:"excludes" form:"excludes" yaml:"excludes"`
+	ExcludeTags   []string    `json:"exclude_tags" form:"exclude_tags" yaml:"exclude_tags"`
+	ExcludeRegexp string      `json:"exclude_regexp" form:"exclude_regexp" yaml:"exclude_regexp"`
 }
 
 type PurgeParams struct {

webapi.go

@@ -240,11 +240,22 @@ func (api *WebApi) ApiAccess(c echo.Context) error {
 }
 
 func (api *WebApi) ApiPurge(c echo.Context) error {
-	code, err := api.purge(c)
+	r := APIPurgeRequest{}
+	if err := c.Bind(&r); err != nil {
+		return c.JSON(http.StatusBadRequest, APICommonResponse{Result: err.Error()})
+	}
+
+	params, err := r.Validate()
 	if err != nil {
-		return c.JSON(code, APICommonResponse{Result: err.Error()})
+		slog.Error(f("purge failed: %s", err))
+		return c.JSON(http.StatusBadRequest, APICommonResponse{Result: err.Error()})
 	}
-	return c.JSON(code, APICommonResponse{Result: "accepted"})
+
+	ctx := c.Request().Context()
+	if err := api.purge(ctx, params); err != nil {
+		return c.JSON(http.StatusInternalServerError, APICommonResponse{Result: err.Error()})
+	}
+	return c.JSON(http.StatusOK, APICommonResponse{Result: "accepted"})
 }
 
 func (api *WebApi) logs(c echo.Context) (int, []string, error) {
@@ -371,22 +382,11 @@ func validateSubdomain(s string) error {
 	return nil
 }
 
-func (api *WebApi) purge(c echo.Context) (int, error) {
-	r := APIPurgeRequest{}
-	if err := c.Bind(&r); err != nil {
-		return http.StatusBadRequest, err
-	}
-
-	p, err := r.Validate()
-	if err != nil {
-		slog.Error(f("purge failed: %s", err))
-		return http.StatusBadRequest, err
-	}
-
-	infos, err := api.runner.List(c.Request().Context(), statusRunning)
+func (api *WebApi) purge(ctx context.Context, p *PurgeParams) error {
+	infos, err := api.runner.List(ctx, statusRunning)
 	if err != nil {
 		slog.Error(f("list ecs failed: %s", err))
-		return http.StatusInternalServerError, err
+		return fmt.Errorf("list tasks failed: %w", err)
 	}
 	slog.Info("purge subdomains",
 		"duration", p.Duration,
@@ -408,7 +408,7 @@ func (api *WebApi) purge(c echo.Context) (int, error) {
 	}
 
 	slog.Info("no subdomains to purge")
-	return http.StatusOK, nil
+	return nil
 }
 
 func (api *WebApi) purgeSubdomains(ctx context.Context, subdomains []string, duration time.Duration) {