add generate images-sync-config command (#472)

* add generate images-sync-config command

* use image len

* add generate config path

* add source registry to config

* use backticks instead of double quotes

* add spaces

cmd/release/cmd/generate.go

@@ -35,6 +35,10 @@ var (
  rancherImagesDigestsOutputFile string
  rancherImagesDigestsRegistry string
  rancherImagesDigestsImagesURL string
+ rancherSyncImages []string
+ rancherSourceRegistry string
+ rancherTargetRegistry string
+ rancherSyncConfigOutputPath string
 )
 
 // generateCmd represents the generate command
@@ -160,6 +164,14 @@ var rancherGenerateDockerImagesDigestsSubCmd = &cobra.Command{
  },
 }
 
+var rancherGenerateImagesSyncConfigSubCmd = &cobra.Command{
+ Use: "images-sync-config",
+ Short: "Generate a regsync config file for images sync",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ return rancher.GenerateImagesSyncConfig(rancherSyncImages, rancherSourceRegistry, rancherTargetRegistry, rancherSyncConfigOutputPath)
+ },
+}
+
 func init() {
  rootCmd.AddCommand(generateCmd)
 
@@ -169,6 +181,7 @@ func init() {
  rancherGenerateSubCmd.AddCommand(rancherGenerateArtifactsIndexSubCmd)
  rancherGenerateSubCmd.AddCommand(rancherGenerateMissingImagesListSubCmd)
  rancherGenerateSubCmd.AddCommand(rancherGenerateDockerImagesDigestsSubCmd)
+ rancherGenerateSubCmd.AddCommand(rancherGenerateImagesSyncConfigSubCmd)
 
  generateCmd.AddCommand(k3sGenerateSubCmd)
  generateCmd.AddCommand(rke2GenerateSubCmd)
@@ -230,4 +243,21 @@ func init() {
  fmt.Println(err.Error())
  os.Exit(1)
  }
+ // rancher generate images-sync-config
+ rancherGenerateImagesSyncConfigSubCmd.Flags().StringSliceVarP(&rancherSyncImages, "images", "k", make([]string, 0), "List of images to sync to a registry")
+ rancherGenerateImagesSyncConfigSubCmd.Flags().StringVarP(&rancherSourceRegistry, "source-registry", "s", "", "Source registry, where the images are located")
+ rancherGenerateImagesSyncConfigSubCmd.Flags().StringVarP(&rancherTargetRegistry, "target-registry", "t", "", "Target registry, where the images should be synced to")
+ rancherGenerateImagesSyncConfigSubCmd.Flags().StringVarP(&rancherSyncConfigOutputPath, "output", "o", "./config.yaml", "Output path of the generated config file")
+ if err := rancherGenerateImagesSyncConfigSubCmd.MarkFlagRequired("images"); err != nil {
+ fmt.Println(err.Error())
+ os.Exit(1)
+ }
+ if err := rancherGenerateImagesSyncConfigSubCmd.MarkFlagRequired("source-registry"); err != nil {
+ fmt.Println(err.Error())
+ os.Exit(1)
+ }
+ if err := rancherGenerateImagesSyncConfigSubCmd.MarkFlagRequired("target-registry"); err != nil {
+ fmt.Println(err.Error())
+ os.Exit(1)
+ }
 }

release/rancher/rancher.go

@@ -32,6 +32,7 @@ import (
  "github.com/rancher/ecm-distro-tools/repository"
  "golang.org/x/mod/semver"
  "golang.org/x/sync/errgroup"
+ "gopkg.in/yaml.v3"
 )
 
 const (
@@ -53,28 +54,43 @@ const (
  dockerService = "registry.docker.io"
 )
 
+var regsyncDefaultMediaTypes = []string{
+ "application/vnd.docker.distribution.manifest.v2+json",
+ "application/vnd.docker.distribution.manifest.list.v2+json",
+ "application/vnd.oci.image.manifest.v1+json",
+ "application/vnd.oci.image.index.v1+json",
+}
+
 var registriesInfo = map[string]registryInfo{
  "registry.rancher.com": {
- BaseURL: rancherRegistryBaseURL,
- AuthURL: sccSUSEURL,
- Service: sccSUSEService,
+ BaseURL: rancherRegistryBaseURL,
+ AuthURL: sccSUSEURL,
+ Service: sccSUSEService,
+ UserEnv: `{{env "PRIME_REGISTRY_USERNAME"}}`,
+ PasswordEnv: `{{env "PRIME_REGISTRY_PASSWORD"}}`,
  },
  "stgregistry.suse.com": {
- BaseURL: stagingRancherRegistryBaseURL,
- AuthURL: stagingSccSUSEURL,
- Service: sccSUSEService,
+ BaseURL: stagingRancherRegistryBaseURL,
+ AuthURL: stagingSccSUSEURL,
+ Service: sccSUSEService,
+ UserEnv: `{{env "STAGING_REGISTRY_USERNAME"}}`,
+ PasswordEnv: `{{env "STAGING_REGISTRY_PASSWORD"}}`,
  },
  "docker.io": {
- BaseURL: dockerRegistryURL,
- AuthURL: dockerAuthURL,
- Service: dockerService,
+ BaseURL: dockerRegistryURL,
+ AuthURL: dockerAuthURL,
+ Service: dockerService,
+ UserEnv: `{{env "DOCKERIO_REGISTRY_USERNAME"}}`,
+ PasswordEnv: `{{env "DOCKERIO_REGISTRY_PASSWORD"}}`,
  },
 }
 
 type registryInfo struct {
- BaseURL string
- AuthURL string
- Service string
+ BaseURL string
+ AuthURL string
+ Service string
+ UserEnv string
+ PasswordEnv string
 }
 
 type imageDigest map[string]string
@@ -108,6 +124,35 @@ type registryAuthToken struct {
  Token string `json:"token"`
 }
 
+type regsyncConfig struct {
+ Version int `yaml:"version"`
+ Creds []regsyncCreds `yaml:"creds"`
+ Defaults regsyncDefaults `yaml:"defaults"`
+ Sync []regsyncSync `yaml:"sync"`
+}
+
+type regsyncCreds struct {
+ Registry string `yaml:"registry"`
+ User string `yaml:"user"`
+ Pass string `yaml:"pass"`
+}
+
+type regsyncDefaults struct {
+ Parallel int `yaml:"parallel"`
+ MediaTypes []string `yaml:"mediaTypes"`
+}
+
+type regsyncTags struct {
+ Allow []string `yaml:"allow"`
+}
+
+type regsyncSync struct {
+ Source string `yaml:"source"`
+ Target string `yaml:"target"`
+ Type string `yaml:"type"`
+ Tags regsyncTags `yaml:"tags"`
+}
+
 func listS3Objects(ctx context.Context, s3Client *s3.Client, bucketName string, prefix string) ([]string, error) {
  var keys []string
  var continuationToken *string
@@ -497,6 +542,67 @@ func GenerateMissingImagesList(imagesListURL, registry string, concurrencyLimit
  return missingImages, nil
 }
 
+func GenerateImagesSyncConfig(images []string, sourceRegistry, targetRegistry, outputPath string) error {
+ config, err := generateRegsyncConfig(images, sourceRegistry, targetRegistry)
+ if err != nil {
+ return err
+ }
+
+ f, err := os.Create(outputPath)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ return yaml.NewEncoder(f).Encode(config)
+}
+
+func generateRegsyncConfig(images []string, sourceRegistry, targetRegistry string) (*regsyncConfig, error) {
+ sourceRegistryInfo, ok := registriesInfo[sourceRegistry]
+ if !ok {
+ return nil, errors.New("invalid source registry")
+ }
+ targetRegistryInfo, ok := registriesInfo[targetRegistry]
+ if !ok {
+ return nil, errors.New("invalid target registry")
+ }
+
+ config := regsyncConfig{
+ Version: 1,
+ Creds: []regsyncCreds{
+ {
+ Registry: sourceRegistry,
+ User: sourceRegistryInfo.UserEnv,
+ Pass: sourceRegistryInfo.PasswordEnv,
+ },
+ {
+ Registry: targetRegistry,
+ User: targetRegistryInfo.UserEnv,
+ Pass: targetRegistryInfo.PasswordEnv,
+ },
+ },
+ Defaults: regsyncDefaults{
+ Parallel: 1,
+ MediaTypes: regsyncDefaultMediaTypes,
+ },
+ Sync: make([]regsyncSync, len(images)),
+ }
+
+ for i, imageAndVersion := range images {
+ image, imageVersion, err := splitImageAndVersion(imageAndVersion)
+ if err != nil {
+ return nil, err
+ }
+ config.Sync[i] = regsyncSync{
+ Source: sourceRegistry + "/" + image,
+ Target: targetRegistry + "/" + image,
+ Type: "repository",
+ Tags: regsyncTags{Allow: []string{imageVersion}},
+ }
+ }
+ return &config, nil
+}
+
 func imageSliceToMap(images []string) (map[string]bool, error) {
  imagesMap := make(map[string]bool, len(images))
  for _, image := range images {

release/rancher/rancher_test.go

@@ -58,3 +58,31 @@ func TestSplitImageAndVersion(t *testing.T) {
  t.Error("expected to flag image without version as malformed " + imagesWithoutVersion[0])
  }
 }
+
+func TestGenerateRegsyncConfig(t *testing.T) {
+ rancherImage := "rancher/rancher"
+ rancherAgentImage := "rancher/rancher-agent"
+ rancherVersion := "v2.9.0"
+ images := []string{rancherImage + ":" + rancherVersion, rancherAgentImage + ":" + rancherVersion}
+ sourceRegistry := "docker.io"
+ targetRegistry := "registry.rancher.com"
+ sourceRancherImage := sourceRegistry + "/" + rancherImage
+ sourceRancherAgentImage := sourceRegistry + "/" + rancherAgentImage
+ targetRancherImage := targetRegistry + "/" + rancherImage
+ config, err := generateRegsyncConfig(images, sourceRegistry, targetRegistry)
+ if err != nil {
+ t.Error(err)
+ }
+ if config.Sync[0].Source != sourceRancherImage {
+ t.Error("rancher image should be: '" + sourceRancherImage + "' instead, got: '" + config.Sync[0].Source + "'")
+ }
+ if config.Sync[0].Target != targetRancherImage {
+ t.Error("target rancher image should be: '" + targetRancherImage + "' instead, got: '" + config.Sync[0].Target + "'")
+ }
+ if config.Sync[0].Tags.Allow[0] != rancherVersion {
+ t.Error("rancher version should be: '" + rancherVersion + "' instead, got: '" + config.Sync[0].Tags.Allow[0] + "'")
+ }
+ if config.Sync[1].Source != sourceRancherAgentImage {
+ t.Error("rancher agent image should be: '" + sourceRancherAgentImage + "' instead, got: '" + config.Sync[1].Source + "'")
+ }
+}