From f920a042722d261e6197893ebbf9b732a0841a85 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Mon, 19 Feb 2024 08:04:18 -0700 Subject: [PATCH 01/10] fix: multi-part tarballs being mismatched sizes --- src/pkg/utils/io.go | 28 ++++++++++++++++------------ src/test/e2e/05_tarball_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 12 deletions(-) diff --git a/src/pkg/utils/io.go b/src/pkg/utils/io.go index 7bda78ff7f..bb6ae79db2 100755 --- a/src/pkg/utils/io.go +++ b/src/pkg/utils/io.go @@ -339,7 +339,7 @@ func ReadFileByChunks(path string, chunkSizeBytes int) (chunks [][]byte, sha256s // - fileNames: list of file paths srcFile was split across // - sha256sum: sha256sum of the srcFile before splitting // - err: any errors encountered -func SplitFile(srcFile string, chunkSizeBytes int) (err error) { +func SplitFile(srcPath string, chunkSizeBytes int) (err error) { var fileNames []string var sha256sum string hash := sha256.New() @@ -353,7 +353,7 @@ func SplitFile(srcFile string, chunkSizeBytes int) (err error) { buf := make([]byte, bufferSize) // get file size - fi, err := os.Stat(srcFile) + fi, err := os.Stat(srcPath) if err != nil { return err } @@ -364,15 +364,15 @@ func SplitFile(srcFile string, chunkSizeBytes int) (err error) { progressBar := message.NewProgressBar(fileSize, title) defer progressBar.Stop() - // open file - file, err := os.Open(srcFile) - defer file.Close() + // open srcFile + srcFile, err := os.Open(srcPath) if err != nil { return err } + defer srcFile.Close() // create file path starting from part 001 - path := fmt.Sprintf("%s.part001", srcFile) + path := fmt.Sprintf("%s.part001", srcPath) chunkFile, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0644) if err != nil { return err @@ -384,7 +384,7 @@ func SplitFile(srcFile string, chunkSizeBytes int) (err error) { chunkBytesRemaining := chunkSizeBytes // Loop over the tarball hashing as we go and breaking it into chunks based on the chunkSizeBytes for { - bytesRead, err := file.Read(buf) + bytesRead, err := srcFile.Read(buf) if err != nil { if err == io.EOF { @@ -404,10 +404,14 @@ func SplitFile(srcFile string, chunkSizeBytes int) (err error) { if err != nil { return err } + err = chunkFile.Close() + if err != nil { + return err + } // create new file - path = fmt.Sprintf("%s.part%03d", srcFile, len(fileNames)+1) - chunkFile, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0644) + path = fmt.Sprintf("%s.part%03d", srcPath, len(fileNames)+1) + chunkFile, err = os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0644) if err != nil { return err } @@ -435,8 +439,8 @@ func SplitFile(srcFile string, chunkSizeBytes int) (err error) { title := fmt.Sprintf("[%d/%d] MB bytes written", progressBar.GetCurrent()/1000/1000, fileSize/1000/1000) progressBar.UpdateTitle(title) } - file.Close() - _ = os.RemoveAll(srcFile) + srcFile.Close() + _ = os.RemoveAll(srcPath) // calculate sha256 sum sha256sum = fmt.Sprintf("%x", hash.Sum(nil)) @@ -452,7 +456,7 @@ func SplitFile(srcFile string, chunkSizeBytes int) (err error) { } // write header file - path = fmt.Sprintf("%s.part000", srcFile) + path = fmt.Sprintf("%s.part000", srcPath) if err := os.WriteFile(path, jsonData, 0644); err != nil { return fmt.Errorf("unable to write the file %s: %w", path, err) } diff --git a/src/test/e2e/05_tarball_test.go b/src/test/e2e/05_tarball_test.go index 1e0e3c8473..caf2cf90ba 100644 --- a/src/test/e2e/05_tarball_test.go +++ b/src/test/e2e/05_tarball_test.go @@ -5,6 +5,7 @@ package test import ( + "encoding/json" "fmt" "os" "path/filepath" @@ -36,6 +37,21 @@ func TestMultiPartPackage(t *testing.T) { require.NoError(t, err) // Length is 7 because there are 6 parts and 1 manifest require.Len(t, parts, 7) + // Check the file sizes are even + part1FileInfo, err := os.Stat(parts[1]) + require.NoError(t, err) + require.Equal(t, int64(1000000), part1FileInfo.Size()) + part2FileInfo, err := os.Stat(parts[2]) + require.NoError(t, err) + require.Equal(t, int64(1000000), part2FileInfo.Size()) + // Check the package data is correct + pkgData := types.ZarfSplitPackageData{} + part0File, err := os.ReadFile(parts[0]) + require.NoError(t, err) + err = json.Unmarshal(part0File, &pkgData) + require.NoError(t, err) + require.Equal(t, pkgData.Count, 6) + fmt.Printf("%#v", pkgData) stdOut, stdErr, err = e2e.Zarf("package", "deploy", deployPath, "--confirm") require.NoError(t, err, stdOut, stdErr) @@ -45,6 +61,17 @@ func TestMultiPartPackage(t *testing.T) { // deploying package combines parts back into single archive, check dir again to find all files parts, err = filepath.Glob("zarf-package-multi-part-*") + require.NoError(t, err) + // Length is 1 because `zarf package deploy` will recombine the file + require.Len(t, parts, 1) + // Ensure that the number of pkgData bytes was correct + fullFileInfo, err := os.Stat(parts[0]) + require.NoError(t, err) + require.Equal(t, pkgData.Bytes, fullFileInfo.Size()) + // Ensure that the pkgData shasum was correct (should be checked during deploy as well, but this is to double check) + err = utils.SHAsMatch(parts[0], pkgData.Sha256Sum) + require.NoError(t, err) + e2e.CleanFiles(parts...) e2e.CleanFiles(outputFile) } From 29cc6e2f349e10b30d9d960cd5e2761346b632e7 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Mon, 19 Feb 2024 14:07:36 -0700 Subject: [PATCH 02/10] update test to overcome 16MB buffer --- src/test/e2e/05_tarball_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/e2e/05_tarball_test.go b/src/test/e2e/05_tarball_test.go index caf2cf90ba..f5bb5f2020 100644 --- a/src/test/e2e/05_tarball_test.go +++ b/src/test/e2e/05_tarball_test.go @@ -30,27 +30,27 @@ func TestMultiPartPackage(t *testing.T) { e2e.CleanFiles(deployPath, outputFile) // Create the package with a max size of 1MB - stdOut, stdErr, err := e2e.Zarf("package", "create", createPath, "--max-package-size=1", "--confirm") + stdOut, stdErr, err := e2e.Zarf("package", "create", createPath, "--max-package-size=2", "--confirm") require.NoError(t, err, stdOut, stdErr) parts, err := filepath.Glob("zarf-package-multi-part-*") require.NoError(t, err) - // Length is 7 because there are 6 parts and 1 manifest - require.Len(t, parts, 7) + // Length is 4 because there are 3 parts and 1 manifest + require.Len(t, parts, 4) // Check the file sizes are even part1FileInfo, err := os.Stat(parts[1]) require.NoError(t, err) - require.Equal(t, int64(1000000), part1FileInfo.Size()) + require.Equal(t, int64(2000000), part1FileInfo.Size()) part2FileInfo, err := os.Stat(parts[2]) require.NoError(t, err) - require.Equal(t, int64(1000000), part2FileInfo.Size()) + require.Equal(t, int64(2000000), part2FileInfo.Size()) // Check the package data is correct pkgData := types.ZarfSplitPackageData{} part0File, err := os.ReadFile(parts[0]) require.NoError(t, err) err = json.Unmarshal(part0File, &pkgData) require.NoError(t, err) - require.Equal(t, pkgData.Count, 6) + require.Equal(t, pkgData.Count, 3) fmt.Printf("%#v", pkgData) stdOut, stdErr, err = e2e.Zarf("package", "deploy", deployPath, "--confirm") From 9e7e080335588ff662ffe65f5d9ff401898ce294 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Mon, 19 Feb 2024 15:28:28 -0700 Subject: [PATCH 03/10] dynamically make a 50MiB file (more performant than a download) --- .gitignore | 1 + src/test/e2e/05_tarball_test.go | 8 ++++---- src/test/packages/05-multi-part/zarf.yaml | 9 ++++++--- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 4270dbb1fb..3d96988028 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ *.bak *.key *.crt +*.dat *.run.zstd *.tar *.tar.gz diff --git a/src/test/e2e/05_tarball_test.go b/src/test/e2e/05_tarball_test.go index f5bb5f2020..02715455cd 100644 --- a/src/test/e2e/05_tarball_test.go +++ b/src/test/e2e/05_tarball_test.go @@ -24,13 +24,13 @@ func TestMultiPartPackage(t *testing.T) { var ( createPath = "src/test/packages/05-multi-part" deployPath = fmt.Sprintf("zarf-package-multi-part-%s.tar.zst.part000", e2e.Arch) - outputFile = "multi-part-demo.dat" + outputFile = "devops.stackexchange.com_en_all_2023-05.zim" ) e2e.CleanFiles(deployPath, outputFile) // Create the package with a max size of 1MB - stdOut, stdErr, err := e2e.Zarf("package", "create", createPath, "--max-package-size=2", "--confirm") + stdOut, stdErr, err := e2e.Zarf("package", "create", createPath, "--max-package-size=20", "--confirm") require.NoError(t, err, stdOut, stdErr) parts, err := filepath.Glob("zarf-package-multi-part-*") @@ -40,10 +40,10 @@ func TestMultiPartPackage(t *testing.T) { // Check the file sizes are even part1FileInfo, err := os.Stat(parts[1]) require.NoError(t, err) - require.Equal(t, int64(2000000), part1FileInfo.Size()) + require.Equal(t, int64(20000000), part1FileInfo.Size()) part2FileInfo, err := os.Stat(parts[2]) require.NoError(t, err) - require.Equal(t, int64(2000000), part2FileInfo.Size()) + require.Equal(t, int64(20000000), part2FileInfo.Size()) // Check the package data is correct pkgData := types.ZarfSplitPackageData{} part0File, err := os.ReadFile(parts[0]) diff --git a/src/test/packages/05-multi-part/zarf.yaml b/src/test/packages/05-multi-part/zarf.yaml index 78b47ba606..0fb4a75652 100644 --- a/src/test/packages/05-multi-part/zarf.yaml +++ b/src/test/packages/05-multi-part/zarf.yaml @@ -6,8 +6,11 @@ metadata: components: - name: big-ol-file required: true - description: Single 5 MB file needed to demonstrate a multi-part package + description: Include a 50 MB file needed to demonstrate a multi-part package + actions: + onCreate: + before: + - cmd: dd if=/dev/urandom of=multi-part-demo.dat bs=1048576 count=50 files: - - source: https://zarf-public.s3-us-gov-west-1.amazonaws.com/examples/multi-part-demo.dat - shasum: 22ebd38c2f5e04821c87c924c910be57d2169c292f85b2936d53cae24ebf8055 + - source: multi-part-demo.dat target: multi-part-demo.dat From 3157403d32e658616a36cc9abb792bdd03c3aba1 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Mon, 19 Feb 2024 15:29:10 -0700 Subject: [PATCH 04/10] dynamically make a 50MiB file (more performant than a download) --- src/test/e2e/05_tarball_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/e2e/05_tarball_test.go b/src/test/e2e/05_tarball_test.go index 02715455cd..22d52d848a 100644 --- a/src/test/e2e/05_tarball_test.go +++ b/src/test/e2e/05_tarball_test.go @@ -24,7 +24,7 @@ func TestMultiPartPackage(t *testing.T) { var ( createPath = "src/test/packages/05-multi-part" deployPath = fmt.Sprintf("zarf-package-multi-part-%s.tar.zst.part000", e2e.Arch) - outputFile = "devops.stackexchange.com_en_all_2023-05.zim" + outputFile = "multi-part-demo.dat" ) e2e.CleanFiles(deployPath, outputFile) From 87794df9d101bde92b01e2d4fa5e3d1aca5ddff8 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Mon, 19 Feb 2024 15:52:08 -0700 Subject: [PATCH 05/10] close the file for windows file handlers --- src/pkg/packager/sources/split.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/pkg/packager/sources/split.go b/src/pkg/packager/sources/split.go index 1aade2eee5..152e28a17b 100644 --- a/src/pkg/packager/sources/split.go +++ b/src/pkg/packager/sources/split.go @@ -85,6 +85,12 @@ func (s *SplitTarballSource) Collect(dir string) (string, error) { if _, err = io.Copy(pkgFile, f); err != nil { return "", fmt.Errorf("unable to copy file %s: %w", file, err) } + + // Close the file when done copying + err = f.Close() + if err != nil { + return "", fmt.Errorf("unable to close file %s: %w", file, err) + } } if err := utils.SHAsMatch(reassembled, pkgData.Sha256Sum); err != nil { From 742378a2cd2e5e8d6ef5516f1086b49fae88b1c3 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Tue, 20 Feb 2024 17:10:07 -0700 Subject: [PATCH 06/10] fix: test fixes to the image push timeout --- src/internal/packager/images/push.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/internal/packager/images/push.go b/src/internal/packager/images/push.go index bc0e3aa4a8..235a85d7b9 100644 --- a/src/internal/packager/images/push.go +++ b/src/internal/packager/images/push.go @@ -7,6 +7,7 @@ package images import ( "fmt" "net/http" + "time" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/cluster" @@ -50,6 +51,7 @@ func (i *ImageConfig) PushToZarfRegistry() error { httpTransport := http.DefaultTransport.(*http.Transport).Clone() httpTransport.TLSClientConfig.InsecureSkipVerify = i.Insecure + httpTransport.ResponseHeaderTimeout = 10 * time.Second progressBar := message.NewProgressBar(totalSize, fmt.Sprintf("Pushing %d images to the zarf registry", len(i.ImageList))) defer progressBar.Stop() craneTransport := utils.NewTransport(httpTransport, progressBar) From 9fef304eedc353a3c5dadfb9c75d3b3bf16eda5c Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Tue, 20 Feb 2024 17:12:29 -0700 Subject: [PATCH 07/10] fix test --- src/test/e2e/05_tarball_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/e2e/05_tarball_test.go b/src/test/e2e/05_tarball_test.go index e0a04e5a5d..f6df5f7e1b 100644 --- a/src/test/e2e/05_tarball_test.go +++ b/src/test/e2e/05_tarball_test.go @@ -29,7 +29,6 @@ func TestMultiPartPackage(t *testing.T) { e2e.CleanFiles(deployPath, outputFile) - // Create the package with a max size of 20MB stdOut, stdErr, err := e2e.Zarf("package", "create", createPath, "--max-package-size=20", "--confirm") require.NoError(t, err, stdOut, stdErr) From 1f07fef7d0e152933c00d8d4b74e1a93d10f2703 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Tue, 20 Feb 2024 18:23:57 -0700 Subject: [PATCH 08/10] add spinner to registry prune --- src/cmd/tools/crane.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/cmd/tools/crane.go b/src/cmd/tools/crane.go index 80030fd361..1661f40f6b 100644 --- a/src/cmd/tools/crane.go +++ b/src/cmd/tools/crane.go @@ -246,6 +246,9 @@ func pruneImages(_ *cobra.Command, _ []string) error { func doPruneImagesForPackages(zarfState *types.ZarfState, zarfPackages []types.DeployedPackage, registryEndpoint string) error { authOption := config.GetCraneAuthOption(zarfState.RegistryInfo.PushUsername, zarfState.RegistryInfo.PushPassword) + spinner := message.NewProgressSpinner("Looking up images within package definitions") + defer spinner.Stop() + // Determine which image digests are currently used by Zarf packages pkgImages := map[string]bool{} for _, pkg := range zarfPackages { @@ -273,6 +276,8 @@ func doPruneImagesForPackages(zarfState *types.ZarfState, zarfPackages []types.D } } + spinner.Updatef("Cataloging images in the registry") + // Find which images and tags are in the registry currently imageCatalog, err := crane.Catalog(registryEndpoint, authOption) if err != nil { @@ -295,6 +300,8 @@ func doPruneImagesForPackages(zarfState *types.ZarfState, zarfPackages []types.D } } + spinner.Updatef("Calculating images to prune") + // Figure out which images are in the registry but not needed by packages imageDigestsToPrune := map[string]bool{} for digestRef, digest := range referenceToDigest { @@ -308,6 +315,8 @@ func doPruneImagesForPackages(zarfState *types.ZarfState, zarfPackages []types.D } } + spinner.Success() + if len(imageDigestsToPrune) > 0 { message.Note(lang.CmdToolsRegistryPruneImageList) @@ -328,6 +337,9 @@ func doPruneImagesForPackages(zarfState *types.ZarfState, zarfPackages []types.D } } if confirm { + spinner := message.NewProgressSpinner("Deleting unused images") + defer spinner.Stop() + // Delete the digest references that are to be pruned for digestRef := range imageDigestsToPrune { err = crane.Delete(digestRef, authOption) @@ -335,6 +347,8 @@ func doPruneImagesForPackages(zarfState *types.ZarfState, zarfPackages []types.D return err } } + + spinner.Success() } } else { message.Note(lang.CmdToolsRegistryPruneNoImages) From 7518206e8efec669a9f41f60096e39c94b46ba01 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Tue, 20 Feb 2024 18:55:38 -0700 Subject: [PATCH 09/10] update messages to lang --- src/cmd/tools/crane.go | 8 ++++---- src/config/lang/english.go | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/cmd/tools/crane.go b/src/cmd/tools/crane.go index 1661f40f6b..f1f1df6787 100644 --- a/src/cmd/tools/crane.go +++ b/src/cmd/tools/crane.go @@ -246,7 +246,7 @@ func pruneImages(_ *cobra.Command, _ []string) error { func doPruneImagesForPackages(zarfState *types.ZarfState, zarfPackages []types.DeployedPackage, registryEndpoint string) error { authOption := config.GetCraneAuthOption(zarfState.RegistryInfo.PushUsername, zarfState.RegistryInfo.PushPassword) - spinner := message.NewProgressSpinner("Looking up images within package definitions") + spinner := message.NewProgressSpinner(lang.CmdToolsRegistryPruneLookup) defer spinner.Stop() // Determine which image digests are currently used by Zarf packages @@ -276,7 +276,7 @@ func doPruneImagesForPackages(zarfState *types.ZarfState, zarfPackages []types.D } } - spinner.Updatef("Cataloging images in the registry") + spinner.Updatef(lang.CmdToolsRegistryPruneCatalog) // Find which images and tags are in the registry currently imageCatalog, err := crane.Catalog(registryEndpoint, authOption) @@ -300,7 +300,7 @@ func doPruneImagesForPackages(zarfState *types.ZarfState, zarfPackages []types.D } } - spinner.Updatef("Calculating images to prune") + spinner.Updatef(lang.CmdToolsRegistryPruneCalculate) // Figure out which images are in the registry but not needed by packages imageDigestsToPrune := map[string]bool{} @@ -337,7 +337,7 @@ func doPruneImagesForPackages(zarfState *types.ZarfState, zarfPackages []types.D } } if confirm { - spinner := message.NewProgressSpinner("Deleting unused images") + spinner := message.NewProgressSpinner(lang.CmdToolsRegistryPruneDelete) defer spinner.Stop() // Delete the digest references that are to be pruned diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 6a666474d7..14e3b19887 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -455,6 +455,10 @@ $ zarf tools registry digest reg.example.com/stefanprodan/podinfo:6.4.0 CmdToolsRegistryPruneFlagConfirm = "Confirm the image prune action to prevent accidental deletions" CmdToolsRegistryPruneImageList = "The following image digests will be pruned from the registry:" CmdToolsRegistryPruneNoImages = "There are no images to prune" + CmdToolsRegistryPruneLookup = "Looking up images within package definitions" + CmdToolsRegistryPruneCatalog = "Cataloging images in the registry" + CmdToolsRegistryPruneCalculate = "Calculating images to prune" + CmdToolsRegistryPruneDelete = "Deleting unused images" CmdToolsRegistryInvalidPlatformErr = "Invalid platform '%s': %s" CmdToolsRegistryFlagVerbose = "Enable debug logs" From b9a9a7f723a803fc7f696e1bf627e940434f5493 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Wed, 21 Feb 2024 12:59:29 -0700 Subject: [PATCH 10/10] clean PR --- src/internal/packager/images/push.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/internal/packager/images/push.go b/src/internal/packager/images/push.go index 235a85d7b9..bc0e3aa4a8 100644 --- a/src/internal/packager/images/push.go +++ b/src/internal/packager/images/push.go @@ -7,7 +7,6 @@ package images import ( "fmt" "net/http" - "time" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/cluster" @@ -51,7 +50,6 @@ func (i *ImageConfig) PushToZarfRegistry() error { httpTransport := http.DefaultTransport.(*http.Transport).Clone() httpTransport.TLSClientConfig.InsecureSkipVerify = i.Insecure - httpTransport.ResponseHeaderTimeout = 10 * time.Second progressBar := message.NewProgressBar(totalSize, fmt.Sprintf("Pushing %d images to the zarf registry", len(i.ImageList))) defer progressBar.Stop() craneTransport := utils.NewTransport(httpTransport, progressBar)