Merge pull request #52 from thepwagner/wip

RepoUpdater + dockerurl support

.golangci.yml

@@ -37,7 +37,6 @@ linters:
     - govet
     - ineffassign
     - interfacer
-    - lll
     - misspell
     - nakedret
     - noctx

docker/dependencies.go

@@ -0,0 +1,54 @@
+package docker
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/dependabot/gomodules-extracted/cmd/go/_internal_/semver"
+	"github.com/moby/buildkit/frontend/dockerfile/command"
+	"github.com/moby/buildkit/frontend/dockerfile/parser"
+	"github.com/thepwagner/action-update-go/updater"
+)
+
+func (u *Updater) Dependencies(_ context.Context) ([]updater.Dependency, error) {
+	return ExtractDockerfileDependencies(u.root, u.extractDockerfile)
+}
+
+var _ updater.Updater = (*Updater)(nil)
+
+func (u *Updater) extractDockerfile(parsed *parser.Result) ([]updater.Dependency, error) {
+	vars := NewInterpolation(parsed)
+
+	deps := make([]updater.Dependency, 0)
+	for _, instruction := range parsed.AST.Children {
+		// Ignore everything but FROM instructions
+		if instruction.Value != command.From {
+			continue
+		}
+
+		// Parse the image name:
+		image := instruction.Next.Value
+		imageSplit := strings.SplitN(image, ":", 2)
+		if len(imageSplit) == 1 {
+			// No tag provided, default to ":latest"
+			deps = append(deps, updater.Dependency{Path: image, Version: "latest"})
+			continue
+		}
+
+		if strings.Contains(imageSplit[1], "$") {
+			// Version contains a variable, attempt interpolation:
+			vers := vars.Interpolate(imageSplit[1])
+			if !strings.Contains(vers, "$") {
+				deps = append(deps, updater.Dependency{Path: imageSplit[0], Version: vers})
+			}
+		} else if semver.IsValid(imageSplit[1]) {
+			// Image tag is valid semver:
+			deps = append(deps, updater.Dependency{Path: imageSplit[0], Version: imageSplit[1]})
+		} else if s := fmt.Sprintf("v%s", imageSplit[1]); semver.IsValid(s) {
+			// Image tag is close-enough to valid semver:
+			deps = append(deps, updater.Dependency{Path: imageSplit[0], Version: imageSplit[1]})
+		}
+	}
+	return deps, nil
+}

docker/dependencies_test.go

@@ -0,0 +1,22 @@
+package docker_test
+
+import (
+	"testing"
+
+	"github.com/thepwagner/action-update-go/updater"
+	"github.com/thepwagner/action-update-go/updatertest"
+)
+
+func TestUpdater_Dependencies(t *testing.T) {
+	cases := map[string][]updater.Dependency{
+		"simple": {
+			{Path: "alpine", Version: "3.11.0"},
+		},
+		"buildarg": {
+			{Path: "redis", Version: "6.0.0-alpine"},
+			{Path: "redis", Version: "6.0.0-alpine"},
+			{Path: "alpine", Version: "3.11.0"},
+		},
+	}
+	updatertest.DependenciesFixtures(t, updaterFromFixture, cases)
+}

docker/interpolate.go

@@ -0,0 +1,62 @@
+package docker
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/moby/buildkit/frontend/dockerfile/command"
+	"github.com/moby/buildkit/frontend/dockerfile/parser"
+)
+
+// Interpolation attempts to interpolate a variable Dockerfile string
+// Easily fooled by duplicate ARGs
+type Interpolation struct {
+	Vars      map[string]string
+	varsByLen []string
+}
+
+func NewInterpolation(parsed *parser.Result) *Interpolation {
+	i := &Interpolation{
+		Vars: map[string]string{},
+	}
+	for _, instruction := range parsed.AST.Children {
+		switch instruction.Value {
+		case command.Arg:
+			varSplit := strings.SplitN(instruction.Next.Value, "=", 2)
+			if len(varSplit) == 2 {
+				i.Vars[varSplit[0]] = varSplit[1]
+			}
+
+		case command.Env:
+			iter := instruction
+			for iter.Next != nil {
+				i.Vars[iter.Next.Value] = iter.Next.Next.Value
+				iter = iter.Next.Next
+			}
+		}
+	}
+
+	i.varsByLen = make([]string, 0, len(i.Vars))
+	for k := range i.Vars {
+		i.varsByLen = append(i.varsByLen, k)
+	}
+	sort.SliceStable(i.varsByLen, func(x, y int) bool {
+		return len(i.varsByLen[x]) > len(i.varsByLen[y])
+	})
+
+	return i
+}
+
+func (i *Interpolation) Interpolate(s string) string {
+	pre := s
+	for _, k := range i.varsByLen {
+		v := i.Vars[k]
+		s = strings.ReplaceAll(s, fmt.Sprintf("${%s}", k), v)
+		s = strings.ReplaceAll(s, fmt.Sprintf("$%s", k), v)
+	}
+	if pre != s {
+		return i.Interpolate(s)
+	}
+	return s
+}

docker/interpolate_test.go

@@ -0,0 +1,38 @@
+package docker_test
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/moby/buildkit/frontend/dockerfile/parser"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/thepwagner/action-update-go/docker"
+)
+
+func TestInterpolation_Interpolate(t *testing.T) {
+	parsed, err := parser.Parse(strings.NewReader(`FROM scratch
+ENV FOO=bar FOZ=baz
+ENV FOOBAR 123
+ARG NESTED=SUPER${FOO}AND$FOZTOO
+`))
+	require.NoError(t, err)
+
+	i := docker.NewInterpolation(parsed)
+	cases := map[string]string{
+		"$FOO":       "bar",
+		"$FOOBAR":    "123",
+		"${FOO}":     "bar",
+		"${FOO}BAR":  "barBAR",
+		"$FOO$FOZ":   "barbaz",
+		"$FOZZYBEAR": "bazZYBEAR",
+		"$FOX":       "$FOX", // what is your sound?
+		"$NESTED":    "SUPERbarANDbazTOO",
+	}
+
+	for s, expected := range cases {
+		t.Run(s, func(t *testing.T) {
+			assert.Equal(t, expected, i.Interpolate(s))
+		})
+	}
+}

docker/testdata/buildarg/Dockerfile

@@ -0,0 +1,6 @@
+ARG REDIS_VERSION=6.0.0
+ARG ALPINE_VERSION=3.11.0
+
+FROM redis:${REDIS_VERSION}-alpine
+FROM redis:$REDIS_VERSION-alpine
+FROM alpine:$ALPINE_VERSION

docker/testdata/simple/Dockerfile

@@ -0,0 +1 @@
+FROM alpine:3.11.0

docker/updater.go

@@ -0,0 +1,25 @@
+package docker
+
+import (
+	"context"
+
+	"github.com/thepwagner/action-update-go/updater"
+)
+
+type Updater struct {
+	root string
+}
+
+var _ updater.Updater = (*Updater)(nil)
+
+func NewUpdater(root string) *Updater {
+	return &Updater{root: root}
+}
+
+func (u *Updater) Check(ctx context.Context, dependency updater.Dependency) (*updater.Update, error) {
+	panic("implement me")
+}
+
+func (u *Updater) ApplyUpdate(ctx context.Context, update updater.Update) error {
+	panic("implement me")
+}

docker/updater_test.go

@@ -0,0 +1,23 @@
+package docker_test
+
+import (
+	"fmt"
+	"testing"
+
+	deepcopy "github.com/otiai10/copy"
+	"github.com/stretchr/testify/require"
+	"github.com/thepwagner/action-update-go/docker"
+	"github.com/thepwagner/action-update-go/updater"
+)
+
+func updaterFromFixture(t *testing.T, fixture string) updater.Updater {
+	tempDir := tempDirFromFixture(t, fixture)
+	return docker.NewUpdater(tempDir)
+}
+
+func tempDirFromFixture(t *testing.T, fixture string) string {
+	tempDir := t.TempDir()
+	err := deepcopy.Copy(fmt.Sprintf("testdata/%s", fixture), tempDir)
+	require.NoError(t, err)
+	return tempDir
+}

docker/walk.go

@@ -0,0 +1,67 @@
+package docker
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/moby/buildkit/frontend/dockerfile/parser"
+	"github.com/thepwagner/action-update-go/updater"
+)
+
+func WalkDockerfiles(root string, walkFunc func(path string, parsed *parser.Result) error) error {
+	err := filepath.Walk(root, func(path string, fi os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if fi.IsDir() || !strings.HasPrefix(filepath.Base(path), "Dockerfile") {
+			return nil
+		}
+
+		parsed, err := parseDockerfile(path)
+		if err != nil {
+			return fmt.Errorf("parsing %q: %w", path, err)
+		}
+		if err := walkFunc(path, parsed); err != nil {
+			return fmt.Errorf("walking %q: %w", path, err)
+		}
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("walking filesystem: %w", err)
+	}
+	return nil
+}
+
+func ExtractDockerfileDependencies(root string, extractor func(parsed *parser.Result) ([]updater.Dependency, error)) ([]updater.Dependency, error) {
+	deps := make([]updater.Dependency, 0)
+
+	err := WalkDockerfiles(root, func(path string, parsed *parser.Result) error {
+		fileDeps, err := extractor(parsed)
+		if err != nil {
+			return fmt.Errorf("extracting dependencies: %w", err)
+		}
+		deps = append(deps, fileDeps...)
+		return nil
+	})
+
+	if err != nil {
+		return nil, fmt.Errorf("walking filesystem: %w", err)
+	}
+
+	return deps, nil
+}
+
+func parseDockerfile(path string) (*parser.Result, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("opening dockerfile: %w", err)
+	}
+	defer f.Close()
+	parsed, err := parser.Parse(f)
+	if err != nil {
+		return nil, fmt.Errorf("parsing dockerfile: %w", err)
+	}
+	return parsed, nil
+}

docker/walk_test.go

@@ -0,0 +1,27 @@
+package docker_test
+
+import (
+	"fmt"
+	"sync/atomic"
+	"testing"
+
+	"github.com/moby/buildkit/frontend/dockerfile/parser"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/thepwagner/action-update-go/docker"
+	"github.com/thepwagner/action-update-go/updater"
+)
+
+const fixtureCount = 2
+
+func TestWalkDockerfiles(t *testing.T) {
+	var cnt int64
+	deps, err := docker.ExtractDockerfileDependencies("testdata/", func(_ *parser.Result) ([]updater.Dependency, error) {
+		cur := atomic.AddInt64(&cnt, 1)
+		return []updater.Dependency{{Path: "test", Version: fmt.Sprintf("v%d", cur)}}, nil
+	})
+	require.NoError(t, err)
+
+	assert.Equal(t, int64(fixtureCount), cnt, "function not invoked N times")
+	assert.Len(t, deps, fixtureCount, "walk did not collect N results")
+}