ref: cleanup binary for self-hosted

.dockerignore

@@ -1,6 +1,7 @@
 **
 !/.git
 !/cmd/vroom
+!/cmd/cleanup
 !/internal
 !/pkg
 !/vendor

.gitignore

@@ -7,6 +7,7 @@
 /vroom
 /downloader
 /issuedetection
+/cleanup
 
 # Test binary, built with `go test -c`
 *.test

Dockerfile

@@ -6,6 +6,8 @@ COPY . .
 
 RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o . -ldflags="-s -w -X main.release=$(git rev-parse HEAD)" ./cmd/vroom
 
+RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o . -ldflags="-s -w" ./cmd/cleanup
+
 FROM alpine
 
 EXPOSE 8080
@@ -18,6 +20,8 @@ ENV SENTRY_BUCKET_PROFILES=file://localhost/$PROFILES_DIR
 
 COPY --from=builder /src/vroom /bin/vroom
 
+COPY --from=builder /src/cleanup /bin/cleanup
+
 WORKDIR /var/vroom
 
 ENTRYPOINT ["/bin/vroom"]

cmd/cleanup/README.md

@@ -0,0 +1,7 @@
+# Cleanup
+
+Simple script to cleanup profiles that's stored on local filesystem daily, used by self-hosted repository.
+
+It requires 2 environment variables:
+- `SENTRY_BUCKET_PROFILES` - path to profiles directory
+- `SENTRY_EVENT_RETENTION_DAYS` - retention days for profiles, in plain numbers (sample: 90, not 90d). A common environment variable on self-hosted (also used by Sentry and Snuba service)

cmd/cleanup/cleanup.go

@@ -0,0 +1,91 @@
+package main
+
+import (
+	"errors"
+	"os"
+	"os/signal"
+	"path"
+	"strconv"
+	"time"
+
+	"github.com/getsentry/sentry-go"
+	"github.com/getsentry/vroom/internal/logutil"
+	"github.com/robfig/cron/v3"
+	"github.com/rs/zerolog/log"
+)
+
+func cleanup(profilesPath string, timeLimit time.Time) error {
+	dirEntries, err := os.ReadDir(profilesPath)
+	if err != nil {
+		return err
+	}
+
+	for _, entry := range dirEntries {
+		if entry.IsDir() {
+			return cleanup(path.Join(profilesPath, entry.Name()), timeLimit)
+		}
+
+		fileInfo, err := entry.Info()
+		if err != nil && !errors.Is(err, os.ErrNotExist) {
+			return err
+		}
+
+		if timeLimit.After(fileInfo.ModTime()) {
+			err = os.Remove(path.Join(profilesPath, entry.Name()))
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func main() {
+	sentryBucketProfiles, ok := os.LookupEnv("SENTRY_BUCKET_PROFILES")
+	if !ok {
+		sentryBucketProfiles = "/var/lib/sentry-profiles"
+	}
+
+	sentryEventRetentionDays, ok := os.LookupEnv("SENTRY_EVENT_RETENTION_DAYS")
+	if !ok {
+		sentryEventRetentionDays = "90"
+	}
+
+	logutil.ConfigureLogger()
+
+	err := sentry.Init(sentry.ClientOptions{})
+	if err != nil {
+		log.Fatal().Err(err).Msg("can't initialize sentry")
+	}
+
+	retentionDays, err := strconv.ParseInt(sentryEventRetentionDays, 10, 64)
+	if err != nil {
+		log.Fatal().Err(err).Msg("can't parse retention days")
+	}
+
+	timeLimit := time.Now().Add(time.Hour * 24 * -1 * time.Duration(retentionDays))
+
+	c := cron.New()
+	_, err = c.AddFunc("@daily", func() {
+		err := cleanup(sentryBucketProfiles, timeLimit)
+		if err != nil {
+			sentry.CaptureException(err)
+			log.Error().Err(err).Msg("error cleaning up directories")
+		}
+	})
+	if err != nil {
+		log.Fatal().Err(err).Msg("can't set up cron function")
+	}
+
+	exitSignal := make(chan os.Signal, 1)
+	signal.Notify(exitSignal, os.Interrupt)
+
+	go func() {
+		<-exitSignal
+
+		c.Stop()
+	}()
+
+	c.Run()
+}

go.mod

@@ -19,6 +19,7 @@ require (
 	github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
 	github.com/pierrec/lz4 v2.6.1+incompatible
 	github.com/pierrec/lz4/v4 v4.1.15
+	github.com/robfig/cron/v3 v3.0.1
 	github.com/rs/zerolog v1.26.1
 	github.com/segmentio/kafka-go v0.4.38
 	gocloud.dev v0.29.0

go.sum

@@ -1681,6 +1681,8 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T
 github.com/rakyll/embedmd v0.0.0-20171029212350-c8060a0752a2/go.mod h1:7jOTMgqac46PZcF54q6l2hkLEG8op93fZu61KmxWDV4=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
+github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=