From bb904739a285e856c1d828933fcb8ce57c702a09 Mon Sep 17 00:00:00 2001 From: nicholasSSUSE Date: Wed, 13 Nov 2024 19:00:53 -0300 Subject: [PATCH 1/6] refactoring constant paths --- pkg/path/path.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/pkg/path/path.go b/pkg/path/path.go index bc84f84..2ca2e6a 100644 --- a/pkg/path/path.go +++ b/pkg/path/path.go @@ -6,41 +6,54 @@ const ( // RepositoryHelmIndexFile is the file on your Staging/Live branch that contains your Helm repository index RepositoryHelmIndexFile = "index.yaml" + // RepositoryPackagesDir is a directory on your Staging branch that contains the files necessary to generate your package RepositoryPackagesDir = "packages" + // RepositoryAssetsDir is a directory on your Staging/Live branch that contains chart archives for each version of your package RepositoryAssetsDir = "assets" + // RepositoryChartsDir is a directory on your Staging/Live branch that contains unarchived charts for each version of your package RepositoryChartsDir = "charts" // PackageOptionsFile is the name of a file that contains information about how to prepare your package // The expected structure of this file is one that can be marshalled into a PackageOptions struct PackageOptionsFile = "package.yaml" + // PackageTemplatesDir is a directory containing templates used as additional chart options PackageTemplatesDir = "templates" // GeneratedChangesDir is a directory that contains GeneratedChanges GeneratedChangesDir = "generated-changes" + // GeneratedChangesAdditionalChartDir is a directory that contains additionalCharts GeneratedChangesAdditionalChartDir = "additional-charts" + // GeneratedChangesDependenciesDir is a directory that contains dependencies within GeneratedChangesDir GeneratedChangesDependenciesDir = "dependencies" + // GeneratedChangesExcludeDir is a directory that contains excludes within GeneratedChangesDir GeneratedChangesExcludeDir = "exclude" + // GeneratedChangesOverlayDir is a directory that contains overlays within GeneratedChangesDir GeneratedChangesOverlayDir = "overlay" + // GeneratedChangesPatchDir is a directory that contains patches within GeneratedChangesDir GeneratedChangesPatchDir = "patch" + // DependencyOptionsFile is a file that contains information about how to prepare your dependency // The expected structure of this file is one that can be marshalled into a ChartOptions struct DependencyOptionsFile = "dependency.yaml" // ChartCRDDir represents the directory that we expect to contain CRDs within the chart ChartCRDDir = "crds" + // ChartExtraFileDir represents the directory that contains non-YAML files ChartExtraFileDir = "files" + // ChartCRDTgzFilename represents the filename of the crd's tgz file ChartCRDTgzFilename = "crd-manifest.tgz" + // ChartValidateInstallCRDFile is the path to the file pushed to upstream that validates the existence of CRDs in the chart ChartValidateInstallCRDFile = "templates/validate-install-crd.yaml" @@ -50,7 +63,7 @@ const ( // RepositoryLogosDir is a directory on your Staging/Live branch that contains the files with the logos of each chart RepositoryLogosDir = "assets/logos" - // RepositoryStAte file is a file to hold the current status of the released and developed assets versions + // RepositoryStateFile file is a file to hold the current status of the released and developed assets versions RepositoryStateFile = "state.json" // RepositoryReleaseYaml is the file on your Staging/Live branch that contains the release information From 07cc62dedd1ac322935c5e32b4d3b5ba57c3fd18 Mon Sep 17 00:00:00 2001 From: nicholasSSUSE Date: Wed, 13 Nov 2024 19:01:17 -0300 Subject: [PATCH 2/6] adding path to new file: slsa.yaml --- pkg/path/path.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/path/path.go b/pkg/path/path.go index 2ca2e6a..dc2fcac 100644 --- a/pkg/path/path.go +++ b/pkg/path/path.go @@ -71,4 +71,7 @@ const ( // VersionRulesFile is the file that contains the version rules for the current branch on charts-build-scripts VersionRulesFile = "version_rules.json" + + // SlsaYamlFile is the file that contains the list of images already synced and signed for SLSA. + SlsaYamlFile = "slsa.yaml" ) From ca33f3b62f0af4443d4e9dcbb2d66068773f6b40 Mon Sep 17 00:00:00 2001 From: nicholasSSUSE Date: Wed, 13 Nov 2024 21:09:02 -0300 Subject: [PATCH 3/6] small refactoring in GenerateConfigFile adding removeSlsaImages functions yet to be implemented --- pkg/regsync/generateconfig.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/regsync/generateconfig.go b/pkg/regsync/generateconfig.go index 4cd01c4..9d3dce9 100644 --- a/pkg/regsync/generateconfig.go +++ b/pkg/regsync/generateconfig.go @@ -26,18 +26,17 @@ var chartsToIgnoreTags = map[string]string{ func GenerateConfigFile() error { imageTagMap := make(map[string][]string) - err := walkAssetsFolder(imageTagMap) - if err != nil { + if err := walkAssetsFolder(imageTagMap); err != nil { return err } - // Create the regsync config file - err = createRegSyncConfigFile(imageTagMap) - if err != nil { + // Remove the images that are already signed from the imageTagMap + if err := removeSlsaImages(imageTagMap, readSlsaYaml); err != nil { return err } - return nil + // Create the regsync config file + return createRegSyncConfigFile(imageTagMap) } // walkAssetsFolder walks over the assets folder, untars files, stores the values.yaml content From 92075b84fa6023e15181ca5c7f555a7d9c0cfb33 Mon Sep 17 00:00:00 2001 From: nicholasSSUSE Date: Wed, 13 Nov 2024 21:09:37 -0300 Subject: [PATCH 4/6] implementing the removal of already signed slsa images from regsync job --- pkg/regsync/generateconfig.go | 45 +++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/pkg/regsync/generateconfig.go b/pkg/regsync/generateconfig.go index 9d3dce9..febdb78 100644 --- a/pkg/regsync/generateconfig.go +++ b/pkg/regsync/generateconfig.go @@ -11,10 +11,14 @@ import ( "sort" "strings" + "github.com/rancher/charts-build-scripts/pkg/path" "golang.org/x/exp/slices" "gopkg.in/yaml.v2" ) +// ReadSlsaYamlFunc is a function type that reads the slsa.yaml file and returns a list of images. +type ReadSlsaYamlFunc func() ([]string, error) + // chartsToIgnoreTags and systemChartsToIgnoreTags defines the charts and system charts in which a specified // image tag should be ignored. var chartsToIgnoreTags = map[string]string{ @@ -216,3 +220,44 @@ func walkMap(inputMap interface{}, callback func(map[interface{}]interface{})) { } } } + +// removeSlsaImages removes the images that are already signed from the imageTagMap. +func removeSlsaImages(imageTagMap map[string][]string, readSlsaYaml ReadSlsaYamlFunc) error { + // Get the list of images that should not be synced with the registry. + // These images are defined in the slsa.yaml file. + // We will remove these images from the imageTagMap. + slsaImgs, err := readSlsaYaml() + if err != nil { + return err + } + + // The images will not be synced because they are already: + // - Signed + // - Synced with the registry + if slsaImgs != nil { + for _, img := range slsaImgs { + delete(imageTagMap, img) + } + } + return nil +} + +func readSlsaYaml() ([]string, error) { + var slsaImgs []string + + file, err := os.Open(path.SlsaYamlFile) + if err != nil { + return nil, nil // backward version compatibility + } + defer file.Close() + + decoder := yaml.NewDecoder(file) + if err := decoder.Decode(&slsaImgs); err != nil { + if err == io.EOF { + return slsaImgs, nil // Handle EOF error gracefully + } + return nil, err + } + + return slsaImgs, nil +} From f3404f04d003ffba60d13fb18ab753c2c29b5188 Mon Sep 17 00:00:00 2001 From: nicholasSSUSE Date: Wed, 13 Nov 2024 21:09:48 -0300 Subject: [PATCH 5/6] unit-testing --- pkg/regsync/generateconfig_test.go | 118 +++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 pkg/regsync/generateconfig_test.go diff --git a/pkg/regsync/generateconfig_test.go b/pkg/regsync/generateconfig_test.go new file mode 100644 index 0000000..49bf5c2 --- /dev/null +++ b/pkg/regsync/generateconfig_test.go @@ -0,0 +1,118 @@ +package regsync + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_SlsaYaml(t *testing.T) { + + type input struct { + imageTagMap map[string][]string + readSlsaYamlMock func() ([]string, error) + } + + type expected struct { + err error + imageTagMap map[string][]string + } + + type test struct { + name string + input input + expected expected + } + + tests := []test{ + { + name: "#1", + input: input{ + imageTagMap: map[string][]string{ + "image1": {"tag1"}, + "image2": {"tag2"}, + "image3": {"tag3"}, + }, + readSlsaYamlMock: func() ([]string, error) { + return []string{"image1"}, nil + }, + }, + expected: expected{ + err: nil, + imageTagMap: map[string][]string{ + "image2": {"tag2"}, + "image3": {"tag3"}, + }, + }, + }, + { + name: "#2", + input: input{ + imageTagMap: map[string][]string{ + "image1": {"tag1", "tag2"}, + "image2": {"tag2"}, + "image3": {"tag3", "tag4"}, + }, + readSlsaYamlMock: func() ([]string, error) { + return []string{"image1", "image2"}, nil + }, + }, + expected: expected{ + err: nil, + imageTagMap: map[string][]string{ + "image3": {"tag3", "tag4"}, + }, + }, + }, + { + name: "#3", + input: input{ + imageTagMap: map[string][]string{ + "image1": {"tag1", "tag2"}, + }, + readSlsaYamlMock: func() ([]string, error) { + return []string{"image1", "image2"}, nil + }, + }, + expected: expected{ + err: nil, + imageTagMap: map[string][]string{}, + }, + }, + { + name: "#4", + input: input{ + imageTagMap: map[string][]string{}, + readSlsaYamlMock: func() ([]string, error) { + return []string{"image1", "image2"}, nil + }, + }, + expected: expected{ + err: nil, + imageTagMap: map[string][]string{}, + }, + }, + { + name: "#5", + input: input{ + imageTagMap: map[string][]string{}, + readSlsaYamlMock: func() ([]string, error) { + return []string{}, nil + }, + }, + expected: expected{ + err: nil, + imageTagMap: map[string][]string{}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := removeSlsaImages(tt.input.imageTagMap, tt.input.readSlsaYamlMock) + require.NoError(t, err) + require.Equal(t, tt.expected.imageTagMap, tt.input.imageTagMap) + + }) + } +} From c31aa1d0080662a6e5318454fe29793461058323 Mon Sep 17 00:00:00 2001 From: nicholasSSUSE Date: Tue, 19 Nov 2024 16:05:10 -0300 Subject: [PATCH 6/6] fixes --- pkg/path/path.go | 2 +- pkg/regsync/generateconfig.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/path/path.go b/pkg/path/path.go index dc2fcac..9d8dcd5 100644 --- a/pkg/path/path.go +++ b/pkg/path/path.go @@ -63,7 +63,7 @@ const ( // RepositoryLogosDir is a directory on your Staging/Live branch that contains the files with the logos of each chart RepositoryLogosDir = "assets/logos" - // RepositoryStateFile file is a file to hold the current status of the released and developed assets versions + // RepositoryStateFile is a file to hold the current status of the released and developed assets versions RepositoryStateFile = "state.json" // RepositoryReleaseYaml is the file on your Staging/Live branch that contains the release information diff --git a/pkg/regsync/generateconfig.go b/pkg/regsync/generateconfig.go index febdb78..0c3efd8 100644 --- a/pkg/regsync/generateconfig.go +++ b/pkg/regsync/generateconfig.go @@ -247,7 +247,7 @@ func readSlsaYaml() ([]string, error) { file, err := os.Open(path.SlsaYamlFile) if err != nil { - return nil, nil // backward version compatibility + return nil, err // backward version compatibility } defer file.Close()