From f36b04a0e6f5f13554881947af1923f578999e15 Mon Sep 17 00:00:00 2001 From: Stephen Augustus Date: Mon, 25 May 2020 03:27:24 -0400 Subject: [PATCH] gh2gcs: Enable uploads to GCS Signed-off-by: Stephen Augustus --- cmd/gh2gcs/cmd/BUILD.bazel | 2 ++ cmd/gh2gcs/cmd/root.go | 16 ++++++--- pkg/gcp/BUILD.bazel | 1 + pkg/gcp/gcs/BUILD.bazel | 27 ++++++++++++++ pkg/gcp/gcs/gcs.go | 73 ++++++++++++++++++++++++++++++++++++++ pkg/gh2gcs/BUILD.bazel | 2 +- pkg/gh2gcs/gh2gcs.go | 22 ++++++++---- 7 files changed, 132 insertions(+), 11 deletions(-) create mode 100644 pkg/gcp/gcs/BUILD.bazel create mode 100644 pkg/gcp/gcs/gcs.go diff --git a/cmd/gh2gcs/cmd/BUILD.bazel b/cmd/gh2gcs/cmd/BUILD.bazel index 845732bf84cb..1f7dc992580f 100644 --- a/cmd/gh2gcs/cmd/BUILD.bazel +++ b/cmd/gh2gcs/cmd/BUILD.bazel @@ -6,9 +6,11 @@ go_library( importpath = "k8s.io/release/cmd/gh2gcs/cmd", visibility = ["//visibility:public"], deps = [ + "//pkg/gcp:go_default_library", "//pkg/gh2gcs:go_default_library", "//pkg/github:go_default_library", "//pkg/log:go_default_library", + "@com_github_pkg_errors//:go_default_library", "@com_github_sirupsen_logrus//:go_default_library", "@com_github_spf13_cobra//:go_default_library", ], diff --git a/cmd/gh2gcs/cmd/root.go b/cmd/gh2gcs/cmd/root.go index 94b293ccdcc1..723c652834d0 100644 --- a/cmd/gh2gcs/cmd/root.go +++ b/cmd/gh2gcs/cmd/root.go @@ -19,9 +19,11 @@ package cmd import ( "io/ioutil" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "k8s.io/release/pkg/gcp" "k8s.io/release/pkg/gh2gcs" "k8s.io/release/pkg/github" "k8s.io/release/pkg/log" @@ -136,6 +138,10 @@ func initLogging(*cobra.Command, []string) error { } func run(opts *options) error { + if err := gcp.PreCheck(); err != nil { + return errors.Wrap(err, "pre-checking for GCP package usage") + } + if opts.outputDir == "" { tmpDir, err := ioutil.TempDir("", "gh2gcs") if err != nil { @@ -151,9 +157,11 @@ func run(opts *options) error { // TODO: Support downloading releases via yaml config uploadConfig := &gh2gcs.Config{} releaseConfig := &gh2gcs.ReleaseConfig{ - Org: opts.org, - Name: opts.repo, - Tags: []string{}, + Org: opts.org, + Repo: opts.repo, + Tags: []string{}, + GCSBucket: opts.bucket, + ReleaseDir: opts.releaseDir, } if opts.tag != "" { @@ -174,7 +182,7 @@ func run(opts *options) error { return err } - if err := gh2gcs.UploadToGCS(&rc, gh, opts.outputDir); err != nil { + if err := gh2gcs.Upload(&rc, gh, opts.outputDir); err != nil { return err } } diff --git a/pkg/gcp/BUILD.bazel b/pkg/gcp/BUILD.bazel index a5bfcf5692c4..154898198b5e 100644 --- a/pkg/gcp/BUILD.bazel +++ b/pkg/gcp/BUILD.bazel @@ -24,6 +24,7 @@ filegroup( ":package-srcs", "//pkg/gcp/auth:all-srcs", "//pkg/gcp/build:all-srcs", + "//pkg/gcp/gcs:all-srcs", ], tags = ["automanaged"], visibility = ["//visibility:public"], diff --git a/pkg/gcp/gcs/BUILD.bazel b/pkg/gcp/gcs/BUILD.bazel new file mode 100644 index 000000000000..c33c0383a8fc --- /dev/null +++ b/pkg/gcp/gcs/BUILD.bazel @@ -0,0 +1,27 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["gcs.go"], + importpath = "k8s.io/release/pkg/gcp/gcs", + visibility = ["//visibility:public"], + deps = [ + "//pkg/command:go_default_library", + "//pkg/gcp:go_default_library", + "@com_github_sirupsen_logrus//:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/gcp/gcs/gcs.go b/pkg/gcp/gcs/gcs.go new file mode 100644 index 000000000000..68a86b58bac6 --- /dev/null +++ b/pkg/gcp/gcs/gcs.go @@ -0,0 +1,73 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gcs + +import ( + "strings" + + "github.com/sirupsen/logrus" + "k8s.io/release/pkg/command" + "k8s.io/release/pkg/gcp" +) + +var ( + gcsPrefix = "gs://" + concurrentFlag = "-m" + recursiveFlag = "-r" +) + +func CopyToGCS(src, gcsPath string, recursive, concurrent bool) error { + gcsPath = normalizeGCSPath(gcsPath) + + logrus.Infof("Copying %s to GCS (%s)", src, gcsPath) + return bucketCopy(src, gcsPath, recursive, concurrent) +} + +func CopyToLocal(gcsPath, dst string, recursive, concurrent bool) error { + gcsPath = normalizeGCSPath(gcsPath) + + logrus.Infof("Copying GCS (%s) to %s", gcsPath, dst) + return bucketCopy(gcsPath, dst, recursive, concurrent) +} + +func bucketCopy(src, dst string, recursive, concurrent bool) error { + args := []string{} + + if concurrent { + args = append(args, concurrentFlag) + } + + args = append(args, "cp") + if recursive { + args = append(args, recursiveFlag) + } + args = append(args, src, dst) + + cpErr := command.Execute(gcp.GSUtilExecutable, args...) + if cpErr != nil { + return cpErr + } + + return nil +} + +func normalizeGCSPath(gcsPath string) string { + gcsPath = strings.TrimPrefix(gcsPath, gcsPrefix) + gcsPath = gcsPrefix + gcsPath + + return gcsPath +} diff --git a/pkg/gh2gcs/BUILD.bazel b/pkg/gh2gcs/BUILD.bazel index ae838cbb5418..06a5c382e89f 100644 --- a/pkg/gh2gcs/BUILD.bazel +++ b/pkg/gh2gcs/BUILD.bazel @@ -6,8 +6,8 @@ go_library( importpath = "k8s.io/release/pkg/gh2gcs", visibility = ["//visibility:public"], deps = [ + "//pkg/gcp/gcs:go_default_library", "//pkg/github:go_default_library", - "@com_github_sirupsen_logrus//:go_default_library", ], ) diff --git a/pkg/gh2gcs/gh2gcs.go b/pkg/gh2gcs/gh2gcs.go index bfa019336c50..4ae71f1550f0 100644 --- a/pkg/gh2gcs/gh2gcs.go +++ b/pkg/gh2gcs/gh2gcs.go @@ -17,8 +17,9 @@ limitations under the License. package gh2gcs import ( - "github.com/sirupsen/logrus" + "path/filepath" + "k8s.io/release/pkg/gcp/gcs" "k8s.io/release/pkg/github" ) @@ -28,7 +29,7 @@ type Config struct { type ReleaseConfig struct { Org string - Name string + Repo string Tags []string GCSBucket string ReleaseDir string @@ -36,15 +37,24 @@ type ReleaseConfig struct { func DownloadReleases(releaseCfg *ReleaseConfig, ghClient *github.GitHub, outputDir string) error { tags := releaseCfg.Tags - if err := ghClient.DownloadReleaseAssets(releaseCfg.Org, releaseCfg.Name, tags, outputDir); err != nil { + if err := ghClient.DownloadReleaseAssets(releaseCfg.Org, releaseCfg.Repo, tags, outputDir); err != nil { return err } return nil } -// TODO: Add GCS upload logic -func UploadToGCS(releaseCfg *ReleaseConfig, ghClient *github.GitHub, outputDir string) error { - logrus.Info("Uploading to GCS...") +func Upload(releaseCfg *ReleaseConfig, ghClient *github.GitHub, outputDir string) error { + uploadBase := filepath.Join(outputDir, releaseCfg.Org, releaseCfg.Repo) + gcsPath := filepath.Join(releaseCfg.GCSBucket, releaseCfg.ReleaseDir) + + tags := releaseCfg.Tags + for _, tag := range tags { + srcDir := filepath.Join(uploadBase, tag) + if err := gcs.CopyToGCS(srcDir, gcsPath, true, true); err != nil { + return err + } + } + return nil }