Skip to content

Commit

Permalink
chore: support gcp in cloud-image-uploader
Browse files Browse the repository at this point in the history
Add support for uploading images to GCP in cloud image uploader.

GCP is not enabled by default since it's going to be used for e2e-tests
for now.

Signed-off-by: Noel Georgi <git@frezbo.dev>
  • Loading branch information
frezbo committed Sep 4, 2024
1 parent 0a87020 commit 7edcbbb
Show file tree
Hide file tree
Showing 7 changed files with 495 additions and 3 deletions.
8 changes: 5 additions & 3 deletions .secrets.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ secrets:
AZURE_TENANT_ID: ENC[AES256_GCM,data:dZapmCqJeTx9C0us38mxDpPbdxBn39fJOmIc+5MgnAI6esT5,iv:s/GuStsQKgdc/6jpq2YMAE9GggLH/xGfrDzzgk/4kmQ=,tag:+dVM3/Joq3OA/opmSU6TSA==,type:str]
EM_PROJECT_ID: ENC[AES256_GCM,data:nPVZ+Uoul/W7UpxIoeMP1n3YhuEjq3fNKD+zoso4FBP2Obd0,iv:SSF8KZBczWvCJjZpvDo60mnoM21CrzdmmKs2reLi8w0=,tag:VKjsQSHqiQY+IzkIXO70MA==,type:str]
EM_API_TOKEN: ENC[AES256_GCM,data:PnNDZTRDTubebmtAuH1sAuEp5ZwzVie5WA0AhCUk26M=,iv:5MdcOwY+QrIdkFgCXcs2rBGCXQBnhi/EDxTPWr/vCMs=,tag:mcQ9qrWPYMaPalzr/GV7pQ==,type:str]
GOOGLE_PROJECT_ID: ENC[AES256_GCM,data:egcG5hIa5aq6tSRjhA==,iv:g/6pkcSJIQNNgoon6X+6DH2JaQgLKfTDpPUNFjlJ6Xg=,tag:ygF0I8bLRRbj2RdogQqxmw==,type:str]
GOOGLE_CREDENTIALS_JSON: ENC[AES256_GCM,data:o1ZMFswuXh1q4LalVO1etrrOGShA/Uv9uUPox9X6uCvBS8kmx+3ZHKKJET7nklzJ/vMse0KHYTvHblnv8IZ5KJ+7z/XhYaHMbk1WSfn5/QYmg607yvyHqtwkeas/dZGJqE8yDJMT5JQdwLD6xvIicny2dI+WWOnJnOxFwEtwlaz3FUBndzPFaFZu3guXU3dFehe1hwipRuxPyWWYPKnuWXmN+yIVDFeiQedGGGLfuIFaG35xC0s+1ixDsow52vWAu9uU6Y9C7GuWPlC5u/xKTXTF1NR7Ji33ULQTaAZPC9NKpQ+dKsasK2wHlQYQGDMGVd+aEJnZl/7lMgp0EzygFTBne1TDg3S12N75c7E7CC89viLzYDp4DYPJ9jZz4+VZbPTdKRaVh+RYTOefKLXAjvj1N+3klDV5u9rGf/hsWtG4PkfjOCeuIZMZ1lfb1OxS2AtbRB+JgIgGImv/CSq5FnRQii5KVs1/FLl/peg69chKRLNLjJekB1CUQoZhzH9/D/upF3MYAQbvuiA/YsAWCxv6nK8bYaZtHpmTX/EpCjqpZ0d3BwMiJ7jx1aiLpqqMQzm+42x3T9OhvrA8PyZKHy7BOyTgWBREpQqUzHNJ/8Y3UYcQsjUkKCUFmJsnjvs4sdSaepbTuQRqq6WdkXeDZiwiHA+sT+BqG+pUWHqtHWtt8mxGFMlJeWnjo0hKil2Lrv9sfjN+Pemo8+SJ11dqUkcweIophQIlPsWr1rfCnNOwiUdOXuUEtKTJddQWYY/t9wDjwwVKs5hHNTcSs4AfstsOfRxDNeCMKqYOzbcryjI1rhXOULSYLtP+jI0Kh0GarcudejpqlJBje566NrqDHi7F1ZfwM6jj8NuPTUt75JxgTO33sd9PtMv9u5HtS/JPyXistdLe6ul1zWfVgUYEB1G9/QexLI//PRh/6AdCwJXCCf59/TTtCq49B4PRtn8TFRnTJT8kMFCqQiPjJaqM8zBTzVR7Iq51+wKNGMsbAmK8uiQtP8EMso5K+mXQ1aHXouzM/ZXAiahI0ee1C92uGEDsJmkVHmefzuBtpKL1y7EFBfeO2jg/FyWlLOji/kjZTMlyz3pSYfZqjddhkW6MSEVeh0p1Y8v1FK5dBRS/IJo7SWg5NCkCKmAZjePJVbOfjuVyawXRyH/x5i1f7qgGkBZIRi6ddbIdZ+zivCEb7ehXkdPL/whFLtpQPSwHzpE8mxNNcs29niq3Dx3pSIz2ZsNWhr0J3bjP319o44fJaG2QgI2BJ2zeQV1LSiLL6peQ0e2Ay8bAotn/Xd+ncqKn6DbkUjiFTz2O/ZK71xHx9WNoVEisQWvYTNR4y5IL1Z8reYfAVzrxgacqhcgJJU9dTDMYVm/I3gcUbdTca3mkr0tDxtpy3ZKNZLoU/6Le/QoeJwqlbm0sz4ryobz5EetlQyQFZ0r78mYk7fZ9ROS3vI3HKC1c07bSP6vrtTuaxUymxv3zJcflztAN+32MGe+lylNMebSe8Q6sOeAQA+Zu1udoXkOA7kxP51ZpTmHyNxEKe+7NUQ7FF5Uv4s7PkivONxZdbFuh3EM5luukxF+oV4OJX51qwQvFzQzEQnawO8KD0zrOLHRRC/dwgQYCiZhdLfwBxkvpGPjZ+MF3DF3ACRKgi7KZETACMD/hyaOeIkAe11MjBHAZ9C8RBMVhzOkHORrFz6tyfDuDK5ylXa/LktKSJHv8zsfVoK1Cz6hp+RZl7gmT7L7MWv1zr5EwI2CHb5lqk8eRTAe9kVS04TIHz0q8ziw7b9qiNvclPRsJtWtZt3H+B8Nu1sYI20lq0iBdOAtGMHlgz65EVMF3X0Cs4669+fddkNmdv/Ad2nkuK2H3LVKTGI0o0hR5t/TrPNmuQqBuwx7HiBgtavReg5iHlL00YPwE8jTM7az3s5uiSndtw3YDgMxhB2S65orUeIGuEcwOoXnPV3+Xexqo3zRnIt6ryIWXKY3muINmhFtvZkVTp7KzuWt2Anbi9F2xhVn2n0GN6ODPQXEI8nwLxMh3ngmRaRD9EcX5+E2qfCouzfm/4eWKCjw/GggTP8NUjbIYPzFnfy2IIn677/kQDHUYwokItPWjiz811FMm/B6UAaKBDRME2HiVPSYJH10LvduVspaLXBx94Yr8e9inTSHSDM/8GnkdYtyNOfwn/ArwvwyrOXA3JkORIq2aNskGVWYQ5yRvbuK8CRpfo/WAZ52IaB4EmBy0R6YJrPBqlNoW0zw1wSijQZm1ZwtbxAvqY7K6sK85B9mgIVZa9EixyB9V6eU4Voe06opdSL8RE12D1sIiO0+NnDJl3BgFj0mg0Wxs8eiBVa2GnSi/+SNqpo0U15ExteW6wXq2rS7Rl/1LFdbaCDUKWn7joTIErgszrbL6G1KE92Nb07bj83R3Q2c+gvBDxWE80yPaJTLD5Xa9PC19WrgqKM5FGF9kxJ8q//NJ4MVDgA+qbdh4zB12zNBTK03/Q31d2DsNNngQGv+Lyv7Jx2nSQZLRw1gmOUq9ZEICm3LovBsXS4NDUKz4as0b0Mpp0iK6rdISkAXM0c2OJTPqwYhbFqFOf+PwxUv+YWeidgozq1qB9Dfc82OOfdZIbwKPBVJcEy+UGjg38KPq/GXLFy5FyeMepmzl+Wf+UxnWzqbM03n2qG9pb6Ur8wOtEN0NCvZ5znk3nAI5UC56VKw6edQ/M3B0GIfztcJ/agZTjgBgbovqcnNl7iv9lFGn9YVRocm/MiWHe590JaJdSySOxVIojSRJ9fSQvCAU/uHGSR5OvtY3WwVw8qjh06lnhpBWcKjwFA34wh5VLqkH6b8qke6yE3YD4/JhZWMriYsjXRPm4WUj3MrruwbSsh3EH0af9WV345WI5f29CeYkMwKdYk2ErZdQd5McOx4E1fznh2INfyiMjaQ8o1dxlvSZdx6/KC4O5F8r+UrJV7Gbr0m7ryeJK3NfSMPjMLwVCsPdCdtDTDnVbr/yLhfYQ/idlrLSBrolcrt9imEy+1axQzlR0LKZ6+EyZhkKKH45l0yU7U0R++j9/XkZWcbf+zmgYLqOe+tF3FgVSWsAE27Kw0h4dFjVeeqpdcg=,iv:R8UR53WDK3kFRbLwOvWXt0UhaaPgxBLkiSJaPSRvTVA=,tag:RdUgGICSm8P2Dh0tzSbFvw==,type:str]
sops:
kms: []
gcp_kms: []
Expand All @@ -23,8 +25,8 @@ sops:
ZE0zRWwxVzBLL3Q1WW1FNmVvc0txZm8K+GkjAq/WSduuDrsbeyqVi29Pj2IL25mA
a11K/HVqTCU834uHQXjpN3keJS23v5BJGZCpOwVXyZX8f1yAm/ZQAA==
-----END AGE ENCRYPTED FILE-----
lastmodified: "2024-05-06T06:16:37Z"
mac: ENC[AES256_GCM,data:q3NlR1Yi/4J/aCZUbatqL50gW7FPCMrYhYXSZWaZz4f+MLqzV+ymk4dO2QvS7ssgIX7TUVXjR2ClXgl+U3p31rqeVm8o8+LQPimJJnaQ0JrbO8tRZP3sQrQ4tghOKM1hFO/sz/52NTvoxl9OS9qIsq38fM+LUor4gEFekBQEyow=,iv:tZu7y6uezwvUFeHq4DdgNI0izg7DWspDIbzUxKTIBDs=,tag:PvYXbzD4HcWOP1Jw+zmHmA==,type:str]
lastmodified: "2024-09-04T04:28:03Z"
mac: ENC[AES256_GCM,data:PHRJmUueHiv84Pt8fHAze4LWl0RiuNuKozWC/G1ixhqL055Zfe3X5Iv//0qFXDMWBy9b09IiXbd4WwsexaoLhgRE1ZBul1AObK8gsbBHf1DyjxgCek/AJNlIS3WY6NPZ9L5lwxuD0BhqbgzGw6It3rcIyx0q++Zo2UoDQyMciWA=,iv:0R2VsKSFfIzapJsqQELM7LMXvEhDmDWr1f9Amg5i4hs=,tag:ig8uLTnupsmPlgV8GEZTgA==,type:str]
pgp:
- created_at: "2024-04-29T17:03:17Z"
enc: |-
Expand Down Expand Up @@ -86,4 +88,4 @@ sops:
-----END PGP MESSAGE-----
fp: AA5213AF261C1977AF38B03A94B473337258BFD5
unencrypted_suffix: _unencrypted
version: 3.8.1
version: 3.9.0
5 changes: 5 additions & 0 deletions hack/cloud-image-uploader/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -89,3 +89,8 @@ The App registration only needs permissions to the Compute Gallery and the Stora
- Select Access control (IAM)
- Select Add role assignment
- Select the **Storage Blob Data Contributor** role

## Google Cloud Pre-requisites

- `GOOGLE_PROJECT_ID` - Google Cloud Project ID
- `GOOGLE_CREDENTIALS_JSON` - Google Cloud Service Account JSON
304 changes: 304 additions & 0 deletions hack/cloud-image-uploader/gcp.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,304 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package main

import (
"context"
"errors"
"fmt"
"io"
"log"
"net/http"
"os"
"path/filepath"
"strings"
"time"

"cloud.google.com/go/storage"
"github.com/google/uuid"
"github.com/siderolabs/go-retry/retry"
"golang.org/x/sync/errgroup"
"google.golang.org/api/compute/v1"
"google.golang.org/api/googleapi"
"google.golang.org/api/iterator"
"google.golang.org/api/option"
)

// GCPUploder registers the image in GCP.
type GCPUploder struct {
Options Options

storageClient *storage.Client
computeService *compute.Service
projectID string

imagePath string
}

// NewGCPUploder creates a new GCPUploder.
func NewGCPUploder(options Options) (*GCPUploder, error) {
projectID := os.Getenv("GOOGLE_PROJECT_ID")
credentials := os.Getenv("GOOGLE_CREDENTIALS_JSON")

if projectID == "" {
return nil, fmt.Errorf("gcp: GOOGLE_PROJECT_ID is not set")
}

if credentials == "" {
return nil, fmt.Errorf("gcp: GOOGLE_CREDENTIALS_JSON is not set")
}

gcpUploader := &GCPUploder{
Options: options,
}

gcpUploader.projectID = projectID

var err error

gcpUploader.storageClient, err = storage.NewClient(context.Background(), option.WithCredentialsJSON([]byte(credentials)))
if err != nil {
return nil, fmt.Errorf("gcp: failed to create google storage client: %w", err)
}

gcpUploader.computeService, err = compute.NewService(context.Background(), option.WithCredentialsJSON([]byte(credentials)))
if err != nil {
return nil, fmt.Errorf("gcp: failed to create google compute service: %w", err)
}

return gcpUploader, nil
}

// Upload uploads the image to GCP.
func (u *GCPUploder) Upload(ctx context.Context) error {
bucketName := fmt.Sprintf("talos-image-upload-%s", uuid.New())

bucketHandle := u.storageClient.Bucket(bucketName)

if err := bucketHandle.Create(ctx, u.projectID, &storage.BucketAttrs{
PublicAccessPrevention: storage.PublicAccessPreventionEnforced,
}); err != nil {
return fmt.Errorf("gcp: failed to create bucket %s: %w", bucketName, err)
}

log.Println("gcp: created bucket", bucketName)

defer func() {
objects := bucketHandle.Objects(ctx, nil)

for {
objAttr, err := objects.Next()
if errors.Is(err, iterator.Done) {
break
}

if err != nil {
log.Printf("gcp: failed to list objects: %v", err)
}

if err := bucketHandle.Object(objAttr.Name).Delete(ctx); err != nil {
log.Printf("gcp: failed to delete object %s: %v", objAttr.Name, err)
}
}

if err := bucketHandle.Delete(ctx); err != nil {
log.Printf("gcp: failed to delete bucket %s: %v", bucketName, err)
}
}()

var g errgroup.Group

for _, arch := range u.Options.Architectures {
g.Go(func() error {
return u.uploadImage(ctx, arch, bucketName)
})
}

if err := g.Wait(); err != nil {
return fmt.Errorf("gcp: failed to upload images: %w", err)
}

return nil
}

func (u *GCPUploder) uploadImage(ctx context.Context, arch, bucketName string) error {
objectPath := u.Options.GCPImage(arch)

objectName := filepath.Base(objectPath)

objectReader, err := os.Open(objectPath)
if err != nil {
return fmt.Errorf("gcp: failed to open object data file %s: %w", objectPath, err)
}

objectHandle := u.storageClient.Bucket(bucketName).Object(objectName)

objectWriter := objectHandle.NewWriter(ctx)

defer objectWriter.Close() //nolint:errcheck

if _, err := io.Copy(objectWriter, objectReader); err != nil {
return fmt.Errorf("gcp: failed to write object data: %w", err)
}

if err := objectWriter.Close(); err != nil {
return fmt.Errorf("gcp: failed to close object writer: %w", err)
}

u.imagePath = fmt.Sprintf("https://storage.googleapis.com/%s/%s", bucketName, objectName)

log.Println("gcp: uploaded image", u.imagePath)

return u.registerImage(arch)
}

//nolint:gocyclo
func (u *GCPUploder) registerImage(arch string) error {
imageName := fmt.Sprintf("talos-%s-%s", strings.ReplaceAll(u.Options.Tag, ".", "-"), arch)

if u.Options.NamePrefix != "" {
imageName = fmt.Sprintf("%s-talos-%s-%s", u.Options.NamePrefix, strings.ReplaceAll(u.Options.Tag, ".", "-"), arch)
}

exists, err := u.checkImageExists(imageName)
if err != nil {
return err
}

if exists {
log.Printf("gcp: image %s already exists, deleting", imageName)

if deleteErr := u.deleteImage(imageName); deleteErr != nil {
return deleteErr
}
}

operationID, link, err := u.insertImage(imageName, arch)
if err != nil {
return err
}

if err := retry.Constant(15*time.Minute, retry.WithUnits(30*time.Second)).Retry(func() error {
op, err := u.computeService.GlobalOperations.Get(u.projectID, operationID).Do()
if err != nil {
return fmt.Errorf("gcp: failed to get operation: %w", err)
}

if op.HTTPStatusCode != http.StatusOK {
return fmt.Errorf("gcp: operation failed with http error message: %s", op.HttpErrorMessage)
}

if op.Error != nil {
return fmt.Errorf("gcp: operation faild with error message: %s", op.Error.Errors[0].Message)
}

if op.Status == "DONE" {
return nil
}

log.Printf("gcp: image creation progress: %d", op.Progress)

return retry.ExpectedError(fmt.Errorf("gcp: image status is %s", op.Status))
}); err != nil {
return fmt.Errorf("gcp: image creation is taking longer than expected: %w", err)
}

pushResult(CloudImage{
Cloud: "gcp",
Tag: u.Options.Tag,
Region: "us",
Arch: arch,
Type: "compute#image",
ID: link,
})

return nil
}

func (u *GCPUploder) checkImageExists(imageName string) (bool, error) {
_, err := u.computeService.Images.Get(u.projectID, imageName).Do()
if err != nil {
var googleErr *googleapi.Error

if errors.As(err, &googleErr) {
if googleErr.Code == http.StatusNotFound {
return false, nil
}
}

return false, fmt.Errorf("gcp: failed to get image %s: %w", imageName, err)
}

return true, nil
}

func (u *GCPUploder) insertImage(imageName, arch string) (operationID, imageLink string, err error) {
var archImage string

switch arch {
case "amd64":
archImage = "x86_64"
case "arm64":
archImage = "ARM64"
default:
return "", "", fmt.Errorf("gcp: unknown architecture %s", arch)
}

op, err := u.computeService.Images.Insert(u.projectID, &compute.Image{
Architecture: archImage,
Description: fmt.Sprintf("Talos %s %s", u.Options.Tag, arch),
GuestOsFeatures: []*compute.GuestOsFeature{
{
Type: "VIRTIO_SCSI_MULTIQUEUE",
},
},
Name: imageName,
RawDisk: &compute.ImageRawDisk{
Source: u.imagePath,
},
}).Do()
if err != nil {
return "", "", fmt.Errorf("gcp: failed to insert image: %w", err)
}

if op.HTTPStatusCode != http.StatusOK {
return "", "", fmt.Errorf("gcp: insert image failed with http error message: %s", op.HttpErrorMessage)
}

if op.Error != nil {
return "", "", fmt.Errorf("gcp: insert image failed with error message: %s", op.Error.Errors[0].Message)
}

log.Printf("gcp: image %s is being created with operation %s", imageName, op.Name)

return op.Name, op.TargetLink, nil
}

func (u *GCPUploder) deleteImage(imageName string) error {
if _, err := u.computeService.Images.Delete(u.projectID, imageName).Do(); err != nil {
return fmt.Errorf("gcp: failed to delete image %s: %w", imageName, err)
}

if err := retry.Constant(5*time.Minute, retry.WithUnits(30*time.Second)).Retry(func() error {
_, err := u.computeService.Images.Get(u.projectID, imageName).Do()
if err != nil {
var googleErr *googleapi.Error

if errors.As(err, &googleErr) {
if googleErr.Code == http.StatusNotFound {
return nil
}
}

return err
}

return retry.ExpectedError(fmt.Errorf("gcp: image %s still exists", imageName))
}); err != nil {
return fmt.Errorf("gcp: failed to delete image %s: %w", imageName, err)
}

return nil
}
28 changes: 28 additions & 0 deletions hack/cloud-image-uploader/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module github.com/siderolabs/cloud-image-uploader
go 1.22.4

require (
cloud.google.com/go/storage v1.43.0
github.com/Azure/azure-sdk-for-go v68.0.0+incompatible
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
Expand All @@ -19,9 +20,15 @@ require (
github.com/siderolabs/go-retry v0.3.3
github.com/spf13/pflag v1.0.5
golang.org/x/sync v0.8.0
google.golang.org/api v0.187.0
)

require (
cloud.google.com/go v0.115.0 // indirect
cloud.google.com/go/auth v0.6.1 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect
cloud.google.com/go/compute/metadata v0.3.0 // indirect
cloud.google.com/go/iam v1.1.8 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
github.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect
Expand All @@ -33,14 +40,35 @@ require (
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
github.com/dimchansky/utfbom v1.1.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/googleapis/gax-go/v2 v2.12.5 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
go.opentelemetry.io/otel v1.24.0 // indirect
go.opentelemetry.io/otel/metric v1.24.0 // indirect
go.opentelemetry.io/otel/trace v1.24.0 // indirect
golang.org/x/crypto v0.26.0 // indirect
golang.org/x/net v0.28.0 // indirect
golang.org/x/oauth2 v0.21.0 // indirect
golang.org/x/sys v0.24.0 // indirect
golang.org/x/text v0.17.0 // indirect
golang.org/x/time v0.5.0 // indirect
google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d // indirect
google.golang.org/grpc v1.64.0 // indirect
google.golang.org/protobuf v1.34.2 // indirect
)
Loading

0 comments on commit 7edcbbb

Please sign in to comment.