RFD 58: Package Distribution #10746
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,215 @@ | ||
| /* | ||
| Copyright 2022 Gravitational, Inc. | ||
|
|
||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. | ||
| You may obtain a copy of the License at | ||
|
|
||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
|
|
||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| package main | ||
|
|
||
| import ( | ||
| "io/fs" | ||
| "path/filepath" | ||
| "strings" | ||
| "time" | ||
|
|
||
| "github.com/gravitational/trace" | ||
| "github.com/sirupsen/logrus" | ||
| "golang.org/x/mod/semver" | ||
| ) | ||
|
|
||
| type AptRepoTool struct { | ||
| config *Config | ||
| aptly *Aptly | ||
| s3Manager *S3manager | ||
| supportedOSs map[string][]string | ||
| } | ||
|
|
||
| // Instantiates a new apt repo tool instance and performs any required setup/config. | ||
| func NewAptRepoTool(config *Config, supportedOSs map[string][]string) (*AptRepoTool, error) { | ||
|
There was a problem hiding this comment. Why not make There was a problem hiding this comment. When writing this I thought that would be a great idea, but ultimately decided against it as it has several downsides:
Not pulling in the |
||
| art := &AptRepoTool{ | ||
| config: config, | ||
| s3Manager: NewS3Manager(config.bucketName), | ||
| supportedOSs: supportedOSs, | ||
| } | ||
|
|
||
| aptly, err := NewAptly(config.aptlyPath) | ||
| if err != nil { | ||
| return nil, trace.Wrap(err, "failed to create a new aptly instance") | ||
| } | ||
|
|
||
| art.aptly = aptly | ||
|
|
||
| return art, nil | ||
| } | ||
|
|
||
| // Runs the tool, creating and updating APT repos based upon the current configuration. | ||
| func (art *AptRepoTool) Run() error { | ||
| start := time.Now() | ||
| logrus.Infoln("Starting APT repo build process...") | ||
|
|
||
| isFirstRun, err := art.aptly.IsFirstRun() | ||
| if err != nil { | ||
| return trace.Wrap(err, "failed to check if Aptly needs (re)built") | ||
| } | ||
|
|
||
| if isFirstRun { | ||
| logrus.Warningln("First run or disaster recovery detected, attempting to rebuild existing repos from APT repository...") | ||
|
|
||
| err = art.s3Manager.DownloadExistingRepo(art.config.localBucketPath) | ||
| if err != nil { | ||
| return trace.Wrap(err, "failed to sync existing repo from S3 bucket") | ||
| } | ||
|
|
||
| _, err = art.recreateExistingRepos(art.config.localBucketPath) | ||
| if err != nil { | ||
| return trace.Wrap(err, "failed to recreate existing repos") | ||
| } | ||
| } | ||
|
|
||
| // Note: this logic will only push the artifact into the `art.supportedOSs` repos. | ||
| // This behavior is intended to allow deprecating old OS versions in the future | ||
| // without removing the associated repos entirely. | ||
| artifactRepos, err := art.getArtifactRepos() | ||
| if err != nil { | ||
| return trace.Wrap(err, "failed to create repos") | ||
| } | ||
|
|
||
| err = art.importNewDebs(artifactRepos) | ||
| if err != nil { | ||
| return trace.Wrap(err, "failed to import new debs") | ||
| } | ||
|
|
||
| err = art.publishRepos() | ||
| if err != nil { | ||
| return trace.Wrap(err, "failed to publish repos") | ||
| } | ||
|
|
||
| err = art.s3Manager.UploadBuiltRepo(filepath.Join(art.aptly.rootDir, "public")) | ||
| if err != nil { | ||
| return trace.Wrap(err, "failed to sync changes to S3 bucket") | ||
| } | ||
|
|
||
| logrus.Infof("APT repo build process completed in %s", time.Since(start).Round(time.Millisecond)) | ||
| return nil | ||
| } | ||
|
|
||
| func (art *AptRepoTool) publishRepos() error { | ||
| // Pull in all Aptly repos, not just the latest ones to ensure they all get built into APT repos correctly | ||
| repos, err := art.aptly.GetAllRepos() | ||
| if err != nil { | ||
| return trace.Wrap(err, "failed to get all Aptly repos") | ||
| } | ||
|
|
||
| // Build a map keyed by os info with value of all repos that support the os in the key | ||
| // This will be used to structure the publish command | ||
| logrus.Debugf("Categorizing repos according to OS info: %v", RepoNames(repos)) | ||
| categorizedRepos := make(map[string][]*Repo) | ||
| for _, r := range repos { | ||
| if osRepos, ok := categorizedRepos[r.OSInfo()]; ok { | ||
| categorizedRepos[r.OSInfo()] = append(osRepos, r) | ||
| } else { | ||
| categorizedRepos[r.OSInfo()] = []*Repo{r} | ||
| } | ||
| } | ||
| logrus.Debugf("Categorized repos: %v", categorizedRepos) | ||
|
|
||
| for osInfo, osRepoList := range categorizedRepos { | ||
| if len(osRepoList) < 1 { | ||
| continue | ||
| } | ||
|
|
||
| err := art.aptly.PublishRepos(osRepoList, osRepoList[0].os, osRepoList[0].osVersion) | ||
| if err != nil { | ||
| return trace.Wrap(err, "failed to publish for os %q", osInfo) | ||
| } | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func (art *AptRepoTool) recreateExistingRepos(localPublishedPath string) ([]*Repo, error) { | ||
| logrus.Infoln("Recreating previously published repos...") | ||
| createdRepos, err := art.aptly.CreateReposFromPublishedPath(localPublishedPath) | ||
| if err != nil { | ||
| return nil, trace.Wrap(err, "failed to recreate existing repos") | ||
| } | ||
|
|
||
| for _, repo := range createdRepos { | ||
| err := art.aptly.ImportDebsFromExistingRepo(repo) | ||
| if err != nil { | ||
| return nil, trace.Wrap(err, "failed to import debs from existing repo %q", repo.Name()) | ||
| } | ||
| } | ||
|
|
||
| logrus.Infof("Recreated and imported pre-existing artifacts for %d repos", len(createdRepos)) | ||
| return createdRepos, nil | ||
| } | ||
|
|
||
| func (art *AptRepoTool) getArtifactRepos() ([]*Repo, error) { | ||
| logrus.Infoln("Creating or getting Aptly repos for artifact requirements...") | ||
|
|
||
| artifactRepos, err := art.aptly.CreateReposFromArtifactRequirements(art.supportedOSs, | ||
| art.config.releaseChannel, semver.Major(art.config.artifactVersion)) | ||
| if err != nil { | ||
| return nil, trace.Wrap(err, "failed to create or get repos from artifact requirements") | ||
| } | ||
|
|
||
| logrus.Infof("Created or got %d artifact Aptly repos", len(artifactRepos)) | ||
| return artifactRepos, nil | ||
| } | ||
|
|
||
| func (art *AptRepoTool) importNewDebs(repos []*Repo) error { | ||
| logrus.Debugf("Importing new debs into %d repos: %q", len(repos), strings.Join(RepoNames(repos), "\", \"")) | ||
| err := filepath.WalkDir(art.config.artifactPath, | ||
| func(debPath string, d fs.DirEntry, err error) error { | ||
fheinecke marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| return art.importNewDebsWalker(debPath, d, err, repos) | ||
| }, | ||
| ) | ||
| if err != nil { | ||
| return trace.Wrap(err, "failed to find and import debs") | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| // This should not be used outside of importNewDebs | ||
| func (art *AptRepoTool) importNewDebsWalker(debPath string, d fs.DirEntry, err error, repos []*Repo) error { | ||
| if err != nil { | ||
| return trace.Wrap(err, "failure while searching %s for debs", debPath) | ||
| } | ||
|
|
||
| if d.IsDir() { | ||
| return nil | ||
| } | ||
|
|
||
| fileName := d.Name() | ||
| if filepath.Ext(fileName) != ".deb" { | ||
| return nil | ||
| } | ||
|
|
||
| // Import new artifacts into all repos that match the artifact's requirements | ||
| for _, repo := range repos { | ||
| // Other checks could be added here to ensure that a given deb gets added to the correct repo | ||
| // such as name or parent directory, facilitating os-specific artifacts | ||
| if repo.majorVersion != semver.Major(art.config.artifactVersion) || repo.releaseChannel != art.config.releaseChannel { | ||
| continue | ||
| } | ||
|
|
||
| err = art.aptly.ImportDeb(repo.Name(), debPath) | ||
| if err != nil { | ||
| return trace.Wrap(err, "failed to import deb from %s", debPath) | ||
| } | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Does this need to be a part of the drone pipeline?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Thought about this a lot while building this part out. Putting it in the pipeline is by far the easiest way to implement migrations for older versions. In 99.9% of cases this pipeline will never be ran, but when we need to add previous versions this will save significant time. This section along with this new dronegen function takes another migration process like I did last week and turns it into a 5m change.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I don't understand why this particular pipeline is here, though. I get
publish-apt-new-repos, but not this one. Is it just marking that you intend to implement the migration of legacy here?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This pipeline is the implementation of old artifact migration. It's just not enabled/ran unless dronegen is configured to migrate specific versions. When specific versions are added to this function then running
make dronegenwill add migration steps to this pipeline for Drone to run. When there are not any functions listed there, runningmake dronegenwill replace the pipeline with a "NOP" pipeline to prevent repeated migrations.