Merge pull request #1 from thepwagner/import

import

.github/workflows/ci.yaml

@@ -0,0 +1,12 @@
+name: PR
+on: push
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-go@v2
+      with:
+        go-version: '1.15.0'
+    - run: script/test
+    - run: script/lint

.github/workflows/update.yaml

@@ -0,0 +1,22 @@
+name: Periodic Dependency Update
+on:
+  schedule:
+    - cron: '0 8 * * *'
+  workflow_dispatch:
+  pull_request:
+    types: [reopened]
+
+jobs:
+  update:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          token: ${{ secrets.MY_GITHUB_PAT }}
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.15.0'
+      - uses: thepwagner/action-update-go@main
+        with:
+          log_level: debug
+          token: ${{ secrets.MY_GITHUB_PAT }}

action.yml

@@ -0,0 +1,50 @@
+name: Update Docker URLs
+description: Update Docker URL dependencies
+author: 'thepwagner'
+inputs:
+  branches:
+    description: 'Branches to update'
+    required: false
+  token:
+    description: >
+      Personal access token (PAT) used to fetch the repository. The PAT is configured
+      with the local git config, which enables your scripts to run authenticated git
+      commands. The post-job step removes the PAT.
+      We recommend using a service account with the least permissions necessary.
+      Also when generating a new PAT, select the least scopes necessary.
+      [Learn more about creating and using encrypted secrets](https://help.github.com/en/actions/automating-your-workflow-with-github-actions/creating-and-using-encrypted-secrets)
+    default: ${{ github.token }}
+    required: true
+  signing_key:
+    default: "i deserve this"
+    description: >
+      Unique key to use for maintaining trusted metadata in PR body.
+    required: false
+  log_level:
+    description: 'Control debug/info/warn/error output'
+    required: false
+  batches:
+    description: >
+      Configuration for grouping updates together, as a nested YAML of lists:
+      e.g.
+        internal: [github.com/thepwagner/]
+        aws:
+          - github.com/aws
+    required: false
+runs:
+  using: "composite"
+  steps:
+    - name: Verify Go SDK
+      run: which go || echo "Go required, please use actions/setup-go before me"
+      shell: bash
+    - name: Compile action-update-dockerurl
+      run: cd "${{github.action_path}}" && go build -o "${{github.action_path}}/action-update-dockerurl" .
+      shell: bash
+    - name: Run action-update-dockerurl
+      run: ${{github.action_path}}/action-update-dockerurl
+      shell: bash
+      env:
+        INPUT_BRANCHES: ${{ inputs.branches }}
+        INPUT_BATCHES: ${{ inputs.batches }}
+        INPUT_TOKEN: ${{ inputs.token }}
+        INPUT_LOG_LEVEL: ${{ inputs.log_level }}

dockerurl/applyupdate.go

@@ -0,0 +1,311 @@
+package dockerurl
+
+import (
+	"context"
+	"crypto/sha256"
+	"crypto/sha512"
+	"fmt"
+	"hash"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/google/go-github/v32/github"
+	"github.com/moby/buildkit/frontend/dockerfile/parser"
+	"github.com/sirupsen/logrus"
+	"github.com/thepwagner/action-update-docker/docker"
+	"github.com/thepwagner/action-update/updater"
+)
+
+func (u *Updater) ApplyUpdate(ctx context.Context, update updater.Update) error {
+	_, name := parseGitHubRelease(update.Path)
+	nameUpper := strings.ToUpper(name)
+
+	// Potential keys the version/release hash might be stored under:
+	versionKeys := []string{
+		fmt.Sprintf("%s_VERSION", nameUpper),
+		fmt.Sprintf("%s_RELEASE", nameUpper),
+	}
+	hashKeys := []string{
+		fmt.Sprintf("%s_SHASUM", nameUpper),
+		fmt.Sprintf("%s_SHA256SUM", nameUpper),
+		fmt.Sprintf("%s_SHA512SUM", nameUpper),
+		fmt.Sprintf("%s_CHECKSUM", nameUpper),
+	}
+
+	return docker.WalkDockerfiles(u.root, func(path string, parsed *parser.Result) error {
+		patterns := u.collectPatterns(ctx, parsed, versionKeys, hashKeys, update)
+		logrus.WithFields(logrus.Fields{
+			"path":     path,
+			"patterns": len(patterns),
+		}).Debug("collected patterns")
+		return updateDockerfile(path, patterns)
+	})
+}
+
+func (u *Updater) collectPatterns(ctx context.Context, parsed *parser.Result, versionKeys, hashKeys []string, update updater.Update) map[string]string {
+	i := docker.NewInterpolation(parsed)
+	patterns := map[string]string{}
+	var versionHit bool
+	for _, k := range versionKeys {
+		prev, ok := i.Vars[k]
+		if !ok {
+			continue
+		}
+		switch prev {
+		case update.Previous:
+			logrus.WithFields(logrus.Fields{
+				"key":    k,
+				"prefix": true,
+			}).Debug("identified version key")
+			patterns[fmt.Sprintf("%s=%s", k, prev)] = fmt.Sprintf("%s=%s", k, update.Next)
+			versionHit = true
+		case update.Previous[1:]:
+			logrus.WithFields(logrus.Fields{
+				"key":    k,
+				"prefix": false,
+			}).Debug("identified version key")
+			patterns[fmt.Sprintf("%s=%s", k, update.Previous[1:])] = fmt.Sprintf("%s=%s", k, update.Next[1:])
+			versionHit = true
+		}
+	}
+	if !versionHit {
+		return patterns
+	}
+
+	for _, k := range hashKeys {
+		prev, ok := i.Vars[k]
+		if !ok {
+			continue
+		}
+		log := logrus.WithField("key", k)
+		log.Debug("identified hash key")
+
+		newHash, err := u.updatedHash(ctx, update, prev)
+		if err != nil {
+			log.WithError(err).Warn("fetching updated hash")
+		} else if newHash != "" {
+			log.Debug("updated hash key")
+			patterns[fmt.Sprintf("%s=%s", k, prev)] = fmt.Sprintf("%s=%s", k, newHash)
+		}
+	}
+	return patterns
+}
+
+func (u *Updater) updatedHash(ctx context.Context, update updater.Update, oldHash string) (string, error) {
+	// Fetch the previous release:
+	owner, repoName := parseGitHubRelease(update.Path)
+	prevRelease, _, err := u.ghRepos.GetReleaseByTag(ctx, owner, repoName, update.Previous)
+	if err != nil {
+		return "", fmt.Errorf("fetching previous release: %w", err)
+	}
+
+	// First pass, does the project release a SHASUMS etc file we can grab?
+	for _, prevAsset := range prevRelease.Assets {
+		log := logrus.WithField("name", prevAsset.GetName())
+		oldAsset, err := u.isShasumAsset(ctx, prevAsset, oldHash)
+		if err != nil {
+			log.WithError(err).Warn("inspecting potential hash asset")
+			continue
+		} else if len(oldAsset) == 0 {
+			log.Debug("old shasum asset not found")
+			continue
+		}
+		log.Debug("identified shasum asset in previous release")
+
+		// The previous release contained a shasum file that contained the previous hash
+		// Does the new release have the same file?
+		newHash, err := u.updatedHashFromShasumAsset(ctx, prevAsset, oldAsset, oldHash, update)
+		if err != nil {
+			log.WithError(err).Warn("fetching updated hash asset")
+			continue
+		}
+		if newHash != "" {
+			log.Debug("fetched corresponding shasum asset from new release")
+			return newHash, nil
+		}
+	}
+
+	// There are no shasum files - get downloading
+	logrus.Debug("shasum file not found, searching files from previous release")
+	for _, prevAsset := range prevRelease.Assets {
+		log := logrus.WithField("name", prevAsset.GetName())
+		h, err := u.isHashAsset(ctx, prevAsset, oldHash)
+		if err != nil {
+			log.WithError(err).Warn("checking hash of previous assets")
+			continue
+		} else if !h {
+			continue
+		}
+		log.Debug("identified hashed asset in previous release")
+
+		// This asset from a previous release matched the previous hash
+		// Does the new release have the same file?
+		newHash, err := u.updatedHashFromAsset(ctx, prevAsset, update, oldHash)
+		if err != nil {
+			return "", err
+		}
+		if newHash != "" {
+			log.Debug("fetched corresponding asset from new release")
+			return newHash, nil
+		}
+	}
+
+	return "", nil
+}
+
+// isShasumAsset returns true if the release asset is a SHASUMS file containing the previous hash
+func (u *Updater) isShasumAsset(ctx context.Context, asset *github.ReleaseAsset, oldHash string) ([]string, error) {
+	if asset.GetSize() > 1024 {
+		return nil, nil
+	}
+
+	req, err := http.NewRequest("GET", asset.GetBrowserDownloadURL(), nil)
+	if err != nil {
+		return nil, err
+	}
+	req = req.WithContext(ctx)
+
+	res, err := u.http.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+	b, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return nil, err
+	}
+	s := string(b)
+	if !strings.Contains(s, oldHash) {
+		return nil, nil
+	}
+	return strings.Split(s, "\n"), nil
+}
+
+func (u *Updater) updatedHashFromShasumAsset(ctx context.Context, asset *github.ReleaseAsset, oldContents []string, oldHash string, update updater.Update) (string, error) {
+	res, err := u.getUpdatedAsset(ctx, asset, update)
+	if err != nil {
+		return "", err
+	}
+	defer res.Body.Close()
+	b, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return "", err
+	}
+	s := string(b)
+
+	// If there's one line, extract the checksum and return it:
+	if len(oldContents) == 1 {
+		return strings.SplitN(s, " ", 2)[0], nil
+	}
+
+	// If there's multiple lines, find the file corresponding the old hash:
+	var hashedFile string
+	for _, oldLine := range oldContents {
+		split := strings.SplitN(oldLine, " ", 2)
+		if split[0] == oldHash {
+			hashedFile = split[1]
+		}
+	}
+	if hashedFile == "" {
+		return "", nil
+	}
+
+	logrus.WithField("fn", hashedFile).Debug("identified hashed file in shasum asset")
+	for _, newLine := range strings.Split(s, "\n") {
+		split := strings.SplitN(newLine, " ", 2)
+		if len(split) == 1 {
+			continue
+		}
+		if split[1] == hashedFile {
+			return split[0], nil
+		}
+	}
+
+	return "", nil
+}
+
+func (u *Updater) getUpdatedAsset(ctx context.Context, asset *github.ReleaseAsset, update updater.Update) (*http.Response, error) {
+	newURL := asset.GetBrowserDownloadURL()
+	newURL = strings.ReplaceAll(newURL, update.Previous, update.Next)
+	newURL = strings.ReplaceAll(newURL, update.Previous[1:], update.Next[1:])
+	req, err := http.NewRequest("GET", newURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	req = req.WithContext(ctx)
+	return u.http.Do(req)
+}
+
+func (u *Updater) isHashAsset(ctx context.Context, asset *github.ReleaseAsset, oldHash string) (bool, error) {
+	h, ok := hasher(oldHash)
+	if !ok {
+		return false, nil
+	}
+
+	req, err := http.NewRequest("GET", asset.GetBrowserDownloadURL(), nil)
+	if err != nil {
+		return false, err
+	}
+	req = req.WithContext(ctx)
+
+	res, err := u.http.Do(req)
+	if err != nil {
+		return false, err
+	}
+	defer res.Body.Close()
+	if _, err := io.Copy(h, res.Body); err != nil {
+		return false, err
+	}
+	sum := h.Sum(nil)
+	return fmt.Sprintf("%x", sum) == oldHash, nil
+}
+
+func hasher(oldHash string) (hash.Hash, bool) {
+	switch len(oldHash) {
+	case 64:
+		return sha256.New(), true
+	case 128:
+		return sha512.New(), true
+	default:
+		return nil, false
+	}
+}
+
+func (u *Updater) updatedHashFromAsset(ctx context.Context, asset *github.ReleaseAsset, update updater.Update, oldHash string) (string, error) {
+	res, err := u.getUpdatedAsset(ctx, asset, update)
+	if err != nil {
+		return "", err
+	}
+	defer res.Body.Close()
+	h, _ := hasher(oldHash)
+	if _, err := io.Copy(h, res.Body); err != nil {
+		return "", err
+	}
+	sum := h.Sum(nil)
+	return fmt.Sprintf("%x", sum), nil
+}
+
+func updateDockerfile(path string, patterns map[string]string) error {
+	// Buffer contents as a string
+	b, err := ioutil.ReadFile(path)
+	if err != nil {
+		return fmt.Errorf("reading file: %w", err)
+	}
+	s := string(b)
+
+	for old, replace := range patterns {
+		s = strings.ReplaceAll(s, old, replace)
+	}
+
+	stat, err := os.Stat(path)
+	if err != nil {
+		return fmt.Errorf("stating file: %w", err)
+	}
+	if err := ioutil.WriteFile(path, []byte(s), stat.Mode()); err != nil {
+		return fmt.Errorf("writing updated file: %w", err)
+	}
+	return nil
+}