Skip to content

Commit

Permalink
Merge pull request #1 from tumblr/gabe/label-predicates
Browse files Browse the repository at this point in the history
Adding Label based predicates
  • Loading branch information
byxorna authored Jul 11, 2019
2 parents 39e6999 + e4c025b commit ec3cdec
Show file tree
Hide file tree
Showing 26 changed files with 1,143 additions and 567 deletions.
1 change: 1 addition & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ COPY . .
RUN make && rm -rf vendor/

FROM alpine:latest
LABEL maintainer="Tumblr"
RUN apk --no-cache add ca-certificates
COPY --from=0 /app/bin/docker-registry-pruner /bin/docker-registry-pruner
COPY ./entrypoint.sh /bin/entrypoint.sh
Expand Down
7 changes: 4 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ GO2XUNIT = $(BIN)/go2xunit
$(BIN)/go2xunit: | ; $(info $(M) building go2xunit…)
$Q go get github.com/tebeka/go2xunit


# Tests
#

Expand All @@ -63,10 +64,10 @@ test-race: ARGS=-race ## Run tests with race detector
$(TEST_TARGETS): NAME=$(MAKECMDGOALS:test-%=%)
$(TEST_TARGETS): test
check test tests: fmt lint vendor | ; $(info $(M) running $(NAME:%=% )tests…) @ ## Run tests
$Q $(GO) test -timeout $(TIMEOUT)s $(ARGS) $(TESTPKGS)
$Q $(GO) test -count=1 -timeout $(TIMEOUT)s $(ARGS) $(TESTPKGS)

test-xml: fmt lint vendor | $(GO2XUNIT) ; $(info $(M) running $(NAME:%=% )tests…) @ ## Run tests with xUnit output
$Q 2>&1 $(GO) test -timeout 20s -v $(TESTPKGS) | tee test/tests.output
$Q 2>&1 $(GO) test -count=1 -timeout 20s -v $(TESTPKGS) | tee test/tests.output
$(GO2XUNIT) -fail -input test/tests.output -output test/tests.xml

COVERAGE_MODE = atomic
Expand All @@ -79,7 +80,7 @@ test-coverage: COVERAGE_DIR := $(CURDIR)/test/coverage.$(shell date -u +"%Y-%m-%
test-coverage: fmt lint vendor test-coverage-tools | ; $(info $(M) running coverage tests…) @ ## Run coverage tests
$Q mkdir -p $(COVERAGE_DIR)/coverage
$Q for pkg in $(TESTPKGS); do \
$(GO) test \
$(GO) test -count=1 \
-coverpkg=$$($(GO) list -f '{{ join .Deps "\n" }}' $$pkg | \
grep '^$(PACKAGE)/' | grep -v '^$(PACKAGE)/vendor/' | \
tr '\n' ',')$$pkg \
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# docker-registry-pruner 🌱✂️
# docker-registry-pruner 🐳✂️


`docker-registry-pruner` is a rules-based tool that applies business logic to docker images in a Docker Registry storage system for retention.
Expand Down
34 changes: 21 additions & 13 deletions cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"text/tabwriter"
"time"

"github.com/tumblr/docker-registry-pruner/pkg/client"
"github.com/tumblr/docker-registry-pruner/pkg/config"
"github.com/tumblr/docker-registry-pruner/pkg/registry"
"github.com/tumblr/docker-registry-pruner/pkg/rules"
Expand Down Expand Up @@ -43,7 +44,7 @@ func main() {
log.Fatal(err)
}

hub, err := registry.New(cfg)
hub, err := client.New(cfg)
if err != nil {
log.Fatal(err)
}
Expand All @@ -62,7 +63,7 @@ func main() {
repos = append(repos, repo)
}

for _, rule := range hub.Rules {
for _, rule := range hub.Config.Rules {
log.Infof("Loaded rule: %s", rule.String())
}

Expand Down Expand Up @@ -94,40 +95,47 @@ func PrintTableManifests(matches map[string][]*registry.Manifest) {
w.Flush()
}

func FetchImagesAndApplyRules(hub *registry.Client, repos []string) map[string][]*registry.Manifest {
func FetchImagesAndApplyRules(hub *client.Client, repos []string) map[string][]*registry.Manifest {
repoTags, err := hub.RepoTags(repos)
if err != nil {
log.Fatal(err)
}

selectors := rules.RulesToSelectors(hub.Rules)
filteredRepoTags := rules.FilterRepoTags(repoTags, selectors)
for repo, tags := range filteredRepoTags {
log.Debugf("Repo %s has %d tag matching ruless\n", repo, len(tags))
}

allManifests, err := hub.Manifests(filteredRepoTags)
selectors := rules.RulesToSelectors(hub.Config.Rules)
allManifests, err := hub.Manifests(repoTags)
if err != nil {
log.Fatal(err)
}

keep, delete := registry.ApplyRules(hub.Rules, allManifests)
filteredManifestsByRepo := rules.FilterManifests(allManifests, selectors)
filteredManifests := []*registry.Manifest{}
filteredCount := 0
for n, manifests := range filteredManifestsByRepo {
filteredCount += len(manifests)
log.Debugf("%s: filtered to %d manifests", n, len(manifests))
for _, m := range manifests {
filteredManifests = append(filteredManifests, m)
}
}
log.Debugf("Selector filtering %d manifests to %d manifests", len(allManifests), len(filteredManifests))

keep, delete := rules.ApplyRules(hub.Config.Rules, filteredManifests)
matches := map[string][]*registry.Manifest{
"keep": keep,
"delete": delete,
}
return matches
}

func ShowMatchingRepos(hub *registry.Client, repos []string) {
func ShowMatchingRepos(hub *client.Client, repos []string) {
log.Infof("Querying for manifests. This may take a while...")
matches := FetchImagesAndApplyRules(hub, repos)
deletes, keeps := len(matches["delete"]), len(matches["keep"])
PrintTableManifests(matches)
fmt.Fprintf(os.Stderr, "deleting %d images, keeping %d images\n", deletes, keeps)
}

func DeleteMatchingImages(hub *registry.Client, repos []string) bool {
func DeleteMatchingImages(hub *client.Client, repos []string) bool {
log.Infof("Querying for manifests. This may take a while...")
matches := FetchImagesAndApplyRules(hub, repos)
log.Infof("Beginning deletion of %d images", len(matches["delete"]))
Expand Down
17 changes: 17 additions & 0 deletions docs/config.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,14 @@ A Rule is made up of a Selector, and an Action. See below for more details.
A selector is a predicate that images must satisfy to be considered by the `Action` for deletion.

* `repos` is a list of repositories to apply this rule to. This is literal string matching, _not_ regex. (i.e. `tumblr/plumbus`)
* `labels` is a map of Docker labels that must be present on the Manifest. You can set these in your Dockerfiles with `LABEL foo=bar`. This is useful to create blanket rules for image retention that allow image owners to opt in to cleanups on their own.
* `match_tags` is a list of regexp. Any matching image will have the rule action evaluated against it (i.e. `^v\d+`)
* `ignore_tags` is a list of regexp. Any matching image will explicitly not be evaluated, even if it would have matched `match_tags`

NOTE: the `^latest$` tag is always implicitly inherited into `ignore_tags`.

At least one of the predicates `repos`, `labels` must be present. You may combine `repos` and `labels`, as described in the examples below.

## Actions

You must provide one action, either `keep_versions`, `keep_recent`, or `keep_days`. Images that match the selector and fail the action predicate will be marked for deletion.
Expand Down Expand Up @@ -87,5 +90,19 @@ rules:
- repos:
- web/devtools
keep_recent: 5
# for any image that has the labels {prune=true,environment=development}, expire images after 15 days
- labels:
prune: "true"
environment: "development"
keep_days: 15
# for any repo matching some/image||another/image, if they have the environment=production label, keep the last 5 versions
- repos:
- some/image
- another/image
labels:
environment: production
keep_versions: 5
```

1 change: 0 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,5 @@ require (
go.uber.org/atomic v1.3.2 // indirect
go.uber.org/multierr v1.1.0 // indirect
go.uber.org/zap v1.10.0
gonum.org/v1/gonum v0.0.0-20190430173231-ac0c935b54d8
gopkg.in/yaml.v2 v2.2.2
)
96 changes: 96 additions & 0 deletions internal/pkg/rules/rules2_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
package rules

import (
"reflect"
"sort"
"testing"
"time"

_ "github.com/tumblr/docker-registry-pruner/internal/pkg/testing"
"github.com/tumblr/docker-registry-pruner/pkg/config"
"github.com/tumblr/docker-registry-pruner/pkg/registry"
"github.com/tumblr/docker-registry-pruner/pkg/rules"
)

// helper function to make a test fixture manifest
func mkmanifest(r, t string, daysOld int64, labels map[string]string) *registry.Manifest {
if labels == nil {
labels = map[string]string{}
}
return must(registry.NewManifest(r, t, tNow.Add(time.Duration(-daysOld*24)*time.Hour), labels))
}

func must(m *registry.Manifest, err error) *registry.Manifest {
if err != nil {
panic(err)
}
return m
}

var (
tNow = time.Now()
)

type testpayload struct {
rulesFile string
input []*registry.Manifest
keepImages []string
deleteImages []string
}

func TestApplyRules(t *testing.T) {

tc, err := loadTestConfig("test/fixtures/manifest_tests/apply-rules.yaml")
if err != nil {
t.Error(err)
t.FailNow()
}

for _, test := range tc.Tests {
// sort any expected tag sets
for _, tags := range test.Expected.Keep {
sort.Strings(tags)
}
for _, tags := range test.Expected.Keep {
sort.Strings(tags)
}
t.Logf("%s: loading rules from %s", tc.SourceFile, test.Config)

cfg, err := config.LoadFromFile(test.Config)
if err != nil {
t.Error(err)
t.FailNow()
}

keep, delete := rules.ApplyRules(cfg.Rules, tc.Manifests)
keep_tags := manifestsAsImageMap(keep)
delete_tags := manifestsAsImageMap(delete)
if test.Config == "test/fixtures/rules/labels-devel-3-versions.yaml" {
t.Logf("%s: expected: %+v", test.Config, test)
t.Logf("%s: kept: %+v", test.Config, keep_tags)
t.Logf("%s: deleted: %+v", test.Config, delete_tags)
}

if !reflect.DeepEqual(test.Expected.Keep, keep_tags) {
t.Errorf("%s: expected keep images to be %v but was actually %v", test.Config, test.Expected.Keep, keep_tags)
t.FailNow()
}
if !reflect.DeepEqual(test.Expected.Delete, delete_tags) {
t.Errorf("%s: expected delete images tags to be %v but was actually %v", test.Config, test.Expected.Delete, delete_tags)
t.FailNow()
}
}
}

// turn a list of Manifest into a map of repo->list of tags
func manifestsAsImageMap(ms []*registry.Manifest) map[string][]string {
res := map[string][]string{}
for _, m := range ms {
if _, ok := res[m.Name]; !ok {
res[m.Name] = []string{}
}
res[m.Name] = append(res[m.Name], m.Tag)
sort.Strings(res[m.Name])
}
return res
}
Loading

0 comments on commit ec3cdec

Please sign in to comment.