Create assetdata

We create some yaml files that record the hashes for well-known file assets,
and then we will be able to look them up by their canonical URL.

This is not yet actually used, that can be done in a future commit.

pkg/assets/assetdata/README.md

@@ -0,0 +1,11 @@
+This directory contains hashes for well-known dependencies of kubernetes / a kOps cluster.
+
+We store the hashes here, rather than downloading them every time - it is a little more secure,
+and it is more efficient.
+
+The yaml file structure is intended to mirror the asset structure used by the k8s project,
+e.g. by [kpromo](https://github.com/kubernetes-sigs/promo-tools/blob/main/docs/file-promotion.md).
+However, this should be treated as an implementation detail.
+
+Currently many hash files are manually curated.  Some of them can be automatically generated,
+and we have scripts named `generate-<foo>.sh` to generate them.

pkg/assets/assetdata/cni.yaml

@@ -0,0 +1,14 @@
+filestores:
+- base: https://storage.googleapis.com/k8s-artifacts-cni/release/
+
+files:
+- name: v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz
+  sha256: 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7
+- name: v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz
+  sha256: ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0
+
+- name: v1.2.0/cni-plugins-linux-amd64-v1.2.0.tgz
+  sha256: f3a841324845ca6bf0d4091b4fc7f97e18a623172158b72fc3fdcdb9d42d2d37
+- name: v1.2.0/cni-plugins-linux-arm64-v1.2.0.tgz
+  sha256: 525e2b62ba92a1b6f3dc9612449a84aa61652e680f7ebf4eff579795fe464b57
+

pkg/assets/assetdata/containerd.yaml

@@ -0,0 +1,16 @@
+filestores:
+- base: https://github.com/containerd/containerd/releases/download/
+
+# Manually constructed; there is a .sha256sum file available
+
+files:
+- name: v1.6.20/containerd-1.6.20-linux-amd64.tar.gz
+  sha256: bb9a9ccd6517e2a54da748a9f60dc9aa9d79d19d4724663f2386812f083968e2
+- name: v1.6.20/containerd-1.6.20-linux-arm64.tar.gz
+  sha256: c3e6a054b18b20fce06c7c3ed53f0989bb4b255c849bede446ebca955f07a9ce
+
+- name: v1.7.13/containerd-1.7.13-linux-amd64.tar.gz
+  sha256: c2371c009dd8b7738663333d91e5ab50d204f8bcae24201f45d59060d12c3a23
+- name: v1.7.13/containerd-1.7.13-linux-arm64.tar.gz
+  sha256: 118759e398f35337109592b4d237538872dc12a207d38832b9d04515d0acbc4d
+

pkg/assets/assetdata/crictl.yaml

@@ -0,0 +1,9 @@
+filestores:
+- base: https://github.com/kubernetes-sigs/cri-tools/releases/download/
+
+# Manually constructed, though we could quickly get the hash by downloading the .sha256 file for each asset
+files:
+- name: v1.29.0/crictl-v1.29.0-linux-amd64.tar.gz
+  sha256: d16a1ffb3938f5a19d5c8f45d363bd091ef89c0bc4d44ad16b933eede32fdcbb
+- name: v1.29.0/crictl-v1.29.0-linux-arm64.tar.gz
+  sha256: 0b615cfa00c331fb9c4524f3d4058a61cc487b33a3436d1269e7832cf283f925

pkg/assets/assetdata/data.go

@@ -0,0 +1,102 @@
+package assetdata
+
+import (
+	"embed"
+	"fmt"
+	"io/fs"
+	"net/url"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/kops/util/pkg/hashing"
+	"sigs.k8s.io/yaml"
+)
+
+//go:embed *.yaml
+var embeddedDataFS embed.FS
+
+// GetHash returns the stored hash for the well-known asset, looking it up by the canonicalURL.
+// If found, it returns (hash, true, nil)
+// If not found, it returns (nil, false, nil)
+func GetHash(canonicalURL *url.URL) (*hashing.Hash, bool, error) {
+	var allMatches []*file
+
+	if err := fs.WalkDir(embeddedDataFS, ".", func(p string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			return nil
+		}
+		b, err := fs.ReadFile(embeddedDataFS, p)
+		if err != nil {
+			return fmt.Errorf("reading embedded file %q: %w", p, err)
+		}
+
+		manifest, err := parseManifestFile(b)
+		if err != nil {
+			return fmt.Errorf("parsing embedded file %q: %w", p, err)
+		}
+
+		matches := manifest.Matches(canonicalURL.String())
+		allMatches = append(allMatches, matches...)
+		return nil
+	}); err != nil {
+		return nil, false, fmt.Errorf("walking embedded data: %w", err)
+	}
+
+	hashes := sets.New[string]()
+	for _, match := range allMatches {
+		hashes.Insert(match.SHA256)
+	}
+	if len(hashes) == 0 {
+		return nil, false, nil
+	}
+	if len(hashes) > 1 {
+		return nil, false, fmt.Errorf("found multiple matches for asset %q", canonicalURL)
+	}
+	h, err := hashing.FromString(hashes.UnsortedList()[0])
+	if err != nil {
+		return nil, false, err
+	}
+	return h, true, nil
+}
+
+type file struct {
+	Name   string `json:"name,omitempty"`
+	SHA256 string `json:"sha256,omitempty"`
+}
+
+type fileStore struct {
+	Base string `json:"base,omitempty"`
+}
+
+type manifest struct {
+	FileStores []fileStore `json:"filestores,omitempty"`
+	Files      []file      `json:"files,omitempty"`
+}
+
+func parseManifestFile(b []byte) (*manifest, error) {
+	m := &manifest{}
+	if err := yaml.Unmarshal(b, m); err != nil {
+		return nil, fmt.Errorf("parsing yaml: %w", err)
+	}
+	return m, nil
+}
+
+func (m *manifest) Matches(canonicalURL string) []*file {
+	var matches []*file
+	for _, fileStore := range m.FileStores {
+		if !strings.HasPrefix(canonicalURL, fileStore.Base) {
+			continue
+		}
+		relativePath := strings.TrimPrefix(canonicalURL, fileStore.Base)
+		for i := range m.Files {
+			f := &m.Files[i]
+			if f.Name == relativePath {
+				matches = append(matches, f)
+			}
+		}
+	}
+	return matches
+}

pkg/assets/assetdata/data_test.go

@@ -0,0 +1,34 @@
+package assetdata
+
+import (
+	"net/url"
+	"testing"
+)
+
+func TestGetHash(t *testing.T) {
+	grid := []struct {
+		Name string
+		Hash string
+	}{
+		{Name: "https://dl.k8s.io/release/v1.26.0/bin/linux/amd64/kubelet", Hash: "sha256:b64949fe696c77565edbe4100a315b6bf8f0e2325daeb762f7e865f16a6e54b5"},
+	}
+
+	for _, g := range grid {
+		u, err := url.Parse(g.Name)
+		if err != nil {
+			t.Fatalf("parsing url %q: %v", g.Name, err)
+		}
+		h, found, err := GetHash(u)
+		if err != nil {
+			t.Fatalf("getting hash for %q: %v", g.Name, err)
+		}
+		if !found {
+			t.Fatalf("hash for %q was not found", g.Name)
+		}
+		got := h.String()
+		want := g.Hash
+		if got != g.Hash {
+			t.Errorf("unexpected hash for %q; got %q, want %q", g.Name, got, want)
+		}
+	}
+}

pkg/assets/assetdata/ecr.yaml

@@ -0,0 +1,10 @@
+filestores:
+- base: https://artifacts.k8s.io/binaries/cloud-provider-aws/
+
+# Manually constructed; there is a .sha256 file available for each asset
+
+files:
+- name: v1.27.1/linux/amd64/ecr-credential-provider-linux-amd64
+  sha256: 5035d7814c95cd3cedbc5efb447ef25a4942ef05caab2159746d55ce1698c74a
+- name: v1.27.1/linux/arm64/ecr-credential-provider-linux-arm64
+  sha256: b3d567bda9e2996fc1fbd9d13506bd16763d3865b5c7b0b3c4b48c6088c04481

pkg/assets/assetdata/generate-k8s.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+for version in 1.24.0 1.24.1 1.24.2 1.24.3 1.25.0 1.25.1 1.25.2 1.25.3 1.25.4 1.25.5 1.26.0 1.26.1 1.26.2 1.27.0 1.27.1 1.27.2 1.28.0 1.28.1 1.28.2 1.28.3 1.29.0 1.29.1; do
+cat > k8s-${version}.yaml <<EOF
+filestores:
+- base: https://dl.k8s.io/release/
+
+EOF
+
+go run ./tools/cmd/generatefileassets --base https://dl.k8s.io/release/ --prefix v${version}/ >> k8s-${version}.yaml
+done
+

pkg/assets/assetdata/generate-runc.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+for version in 1.1.0 1.1.1 1.1.2 1.1.3 1.1.4 1.1.5 1.1.6 1.1.7 1.1.8 1.1.9 1.1.10 1.1.11 1.1.12; do
+cat > runc-${version}.yaml <<EOF
+filestores:
+- base:  https://github.com/opencontainers/runc/releases/download/
+
+EOF
+
+go run ./tools/cmd/generatefileassets --base https://github.com/opencontainers/runc/releases/download/ \
+ --prefix v${version}/ \
+ --sums https://github.com/opencontainers/runc/releases/download/v${version}/runc.sha256sum \
+  >> runc-${version}.yaml
+done
+