From 96ac580effad4d9c4be45d91363b67a94cfda9f6 Mon Sep 17 00:00:00 2001 From: prezha Date: Wed, 14 Oct 2020 01:44:08 +0100 Subject: [PATCH 1/8] update: consolidation & automation --- hack/update/filesystem.go | 53 ++++ hack/update/github.go | 267 ++++++++++++++++++ .../golang_version/update_golang_version.go | 132 +++++++++ .../kicbase_version/update_kicbase_version.go | 148 ++++++++++ .../update_kubernetes_version.go | 84 ++++++ hack/update/registry.go | 134 +++++++++ hack/update/update.go | 212 ++++++++++++++ 7 files changed, 1030 insertions(+) create mode 100644 hack/update/filesystem.go create mode 100644 hack/update/github.go create mode 100644 hack/update/golang_version/update_golang_version.go create mode 100644 hack/update/kicbase_version/update_kicbase_version.go create mode 100644 hack/update/kubernetes_version/update_kubernetes_version.go create mode 100644 hack/update/registry.go create mode 100644 hack/update/update.go diff --git a/hack/update/filesystem.go b/hack/update/filesystem.go new file mode 100644 index 000000000000..037c0ffcd602 --- /dev/null +++ b/hack/update/filesystem.go @@ -0,0 +1,53 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package update + +import ( + "io/ioutil" + "os" + "path/filepath" +) + +// fsUpdate updates local filesystem repo files according to the given schema and data, +// returns if the update actually changed anything, and any error occurred +func fsUpdate(fsRoot string, schema map[string]Item, data interface{}) (changed bool, err error) { + for path, item := range schema { + path = filepath.Join(fsRoot, path) + blob, err := ioutil.ReadFile(path) + if err != nil { + return false, err + } + info, err := os.Stat(path) + if err != nil { + return false, err + } + mode := info.Mode() + + item.Content = blob + chg, err := item.apply(data) + if err != nil { + return false, err + } + if chg { + changed = true + } + if err := ioutil.WriteFile(path, item.Content, mode); err != nil { + return false, err + } + } + return changed, nil +} diff --git a/hack/update/github.go b/hack/update/github.go new file mode 100644 index 000000000000..bc82e2ea1dfc --- /dev/null +++ b/hack/update/github.go @@ -0,0 +1,267 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package update + +import ( + "context" + "fmt" + "os" + "strings" + "time" + + "golang.org/x/oauth2" + + "github.com/google/go-github/v32/github" + "k8s.io/klog/v2" +) + +const ( + // ghListPerPage uses max value (100) for PerPage to avoid hitting the rate limits + // (ref: https://godoc.org/github.com/google/go-github/github#hdr-Rate_Limiting) + ghListPerPage = 100 + + // ghSearchLimit limits the number of searched items to be <= N * ListPerPage + ghSearchLimit = 100 +) + +var ( + // GitHub repo data + ghToken = os.Getenv("GITHUB_TOKEN") + ghOwner = "kubernetes" + ghRepo = "minikube" + ghBase = "master" // could be "main" in the future? +) + +// ghCreatePR returns PR created in the GitHub owner/repo, applying the changes to the base head +// commit fork, as defined by the schema and data, and also returns any error occurred +// PR branch will be named by the branch, sufixed by '_' and first 7 characters of fork commit SHA +// PR itself will be named by the title and will reference the issue +func ghCreatePR(ctx context.Context, owner, repo, base, branch, title string, issue int, token string, schema map[string]Item, data interface{}) (*github.PullRequest, error) { + ghc := ghClient(ctx, token) + + // get base branch + baseBranch, _, err := ghc.Repositories.GetBranch(ctx, owner, repo, base) + if err != nil { + return nil, fmt.Errorf("error getting base branch: %w", err) + } + + // get base commit + baseCommit, _, err := ghc.Repositories.GetCommit(ctx, owner, repo, *baseBranch.Commit.SHA) + if err != nil { + return nil, fmt.Errorf("error getting base commit: %w", err) + } + + // get base tree + baseTree, _, err := ghc.Git.GetTree(ctx, owner, repo, baseCommit.GetSHA(), true) + if err != nil { + return nil, fmt.Errorf("error getting base tree: %w", err) + } + + // update files + changes, err := ghUpdate(ctx, owner, repo, baseTree, token, schema, data) + if err != nil { + return nil, fmt.Errorf("error updating files: %w", err) + } + if changes == nil { + return nil, nil + } + + // create fork + fork, resp, err := ghc.Repositories.CreateFork(ctx, owner, repo, nil) + // https://pkg.go.dev/github.com/google/go-github/v32@v32.1.0/github#RepositoriesService.CreateFork + // This method might return an *AcceptedError and a status code of 202. This is because this is + // the status that GitHub returns to signify that it is now computing creating the fork in a + // background task. In this event, the Repository value will be returned, which includes the + // details about the pending fork. A follow up request, after a delay of a second or so, should + // result in a successful request. + if resp.StatusCode == 202 { // *AcceptedError + time.Sleep(time.Second * 5) + } else if err != nil { + return nil, fmt.Errorf("error creating fork: %w", err) + } + + // create fork tree from base and changed files + forkTree, _, err := ghc.Git.CreateTree(ctx, *fork.Owner.Login, *fork.Name, *baseTree.SHA, changes) + if err != nil { + return nil, fmt.Errorf("error creating fork tree: %w", err) + } + + // create fork commit + forkCommit, _, err := ghc.Git.CreateCommit(ctx, *fork.Owner.Login, *fork.Name, &github.Commit{ + Message: github.String(title), + Tree: &github.Tree{SHA: forkTree.SHA}, + Parents: []*github.Commit{{SHA: baseCommit.SHA}}, + }) + if err != nil { + return nil, fmt.Errorf("error creating fork commit: %w", err) + } + klog.Infof("PR commit '%s' created: %s", forkCommit.GetSHA(), forkCommit.GetHTMLURL()) + + // create PR branch + prBranch := branch + forkCommit.GetSHA()[:7] + prRef, _, err := ghc.Git.CreateRef(ctx, *fork.Owner.Login, *fork.Name, &github.Reference{ + Ref: github.String("refs/heads/" + prBranch), + Object: &github.GitObject{ + Type: github.String("commit"), + SHA: forkCommit.SHA, + }, + }) + if err != nil { + return nil, fmt.Errorf("error creating PR branch: %w", err) + } + klog.Infof("PR branch '%s' created: %s", prBranch, prRef.GetURL()) + + // create PR + plan, err := GetPlan(schema, data) + if err != nil { + klog.Fatalf("Error parsing schema: %v\n%s", err, plan) + } + modifiable := true + pr, _, err := ghc.PullRequests.Create(ctx, owner, repo, &github.NewPullRequest{ + Title: github.String(title), + Head: github.String(*fork.Owner.Login + ":" + prBranch), + Base: github.String(base), + Body: github.String(fmt.Sprintf("fixes #%d\n\nAutomatically created PR to update repo according to the Plan:\n\n```\n%s\n```", issue, plan)), + MaintainerCanModify: &modifiable, + }) + if err != nil { + return nil, fmt.Errorf("error creating pull request: %w", err) + } + return pr, nil +} + +// ghUpdate updates remote GitHub owner/repo tree according to the given token, schema and data, +// returns resulting changes, and any error occurred +func ghUpdate(ctx context.Context, owner, repo string, tree *github.Tree, token string, schema map[string]Item, data interface{}) (changes []*github.TreeEntry, err error) { + ghc := ghClient(ctx, token) + + // load each schema item content and update it creating new GitHub TreeEntries + cnt := len(schema) // expected number of files to change + for _, org := range tree.Entries { + if *org.Type == "blob" { + if item, match := schema[*org.Path]; match { + blob, _, err := ghc.Git.GetBlobRaw(ctx, owner, repo, *org.SHA) + if err != nil { + return nil, fmt.Errorf("error getting file: %w", err) + } + item.Content = blob + changed, err := item.apply(data) + if err != nil { + return nil, fmt.Errorf("error updating file: %w", err) + } + if changed { + // add github.TreeEntry that will replace original path content with updated one + changes = append(changes, &github.TreeEntry{ + Path: org.Path, + Mode: org.Mode, + Type: org.Type, + Content: github.String(string(item.Content)), + }) + } + if cnt--; cnt == 0 { + break + } + } + } + } + if cnt != 0 { + return nil, fmt.Errorf("error finding all the files (%d missing) - check the Plan: %w", cnt, err) + } + return changes, nil +} + +// ghFindPR returns URL of the PR if found in the given GitHub ower/repo base and any error occurred +func ghFindPR(ctx context.Context, title, owner, repo, base, token string) (url string, err error) { + ghc := ghClient(ctx, token) + + // walk through the paginated list of all pull requests, from latest to older releases + opts := &github.PullRequestListOptions{State: "all", Base: base, ListOptions: github.ListOptions{PerPage: ghListPerPage}} + for (opts.Page+1)*ghListPerPage <= ghSearchLimit { + prs, resp, err := ghc.PullRequests.List(ctx, owner, repo, opts) + if err != nil { + return "", err + } + for _, pr := range prs { + if pr.GetTitle() == title { + return pr.GetHTMLURL(), nil + } + } + if resp.NextPage == 0 { + break + } + opts.Page = resp.NextPage + } + return "", nil +} + +// ghClient returns GitHub Client with a given context and optional token for authenticated requests +func ghClient(ctx context.Context, token string) *github.Client { + if token == "" { + return github.NewClient(nil) + } + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: token}, + ) + tc := oauth2.NewClient(ctx, ts) + return github.NewClient(tc) +} + +// GHVersions returns current stable release and latest rc or beta pre-release +// from GitHub owner/repo repository, and any error; +// if latest pre-release version is lower than current stable release, then it +// will return current stable release for both +func GHVersions(ctx context.Context, owner, repo string) (stable, latest string, err error) { + ghc := ghClient(ctx, ghToken) + + // walk through the paginated list of all owner/repo releases, from newest to oldest + opts := &github.ListOptions{PerPage: ghListPerPage} + for { + rls, resp, err := ghc.Repositories.ListReleases(ctx, owner, repo, opts) + if err != nil { + return "", "", err + } + for _, rl := range rls { + ver := rl.GetName() + if ver == "" { + continue + } + // check if ver version is a release (ie, 'v1.19.2') or a + // pre-release (ie, 'v1.19.3-rc.0' or 'v1.19.0-beta.2') channel ch + // note: github.RepositoryRelease GetPrerelease() bool would be useful for all pre-rels + ch := strings.Split(ver, "-") + if len(ch) == 1 && stable == "" { + stable = ver + } else if len(ch) > 1 && latest == "" { + if strings.HasPrefix(ch[1], "rc") || strings.HasPrefix(ch[1], "beta") { + latest = ver + } + } + if stable != "" && latest != "" { + // make sure that v.Latest >= stable + if latest < stable { + latest = stable + } + return stable, latest, nil + } + } + if resp.NextPage == 0 { + break + } + opts.Page = resp.NextPage + } + return stable, latest, nil +} diff --git a/hack/update/golang_version/update_golang_version.go b/hack/update/golang_version/update_golang_version.go new file mode 100644 index 000000000000..0e88b0debc95 --- /dev/null +++ b/hack/update/golang_version/update_golang_version.go @@ -0,0 +1,132 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Script expects the following env variables: + - UPDATE_TARGET=: optional - if unset/absent, default option is "fs"; valid options are: + - "fs" - update only local filesystem repo files [default] + - "gh" - update only remote GitHub repo files and create PR (if one does not exist already) + - "all" - update local and remote repo files and create PR (if one does not exist already) + - GITHUB_TOKEN=: GitHub [personal] access token + - note: GITHUB_TOKEN is required if UPDATE_TARGET is "gh" or "all" +*/ + +package main + +import ( + "context" + "io/ioutil" + "net/http" + "strings" + "time" + + "k8s.io/klog/v2" + "k8s.io/minikube/hack/update" +) + +const ( + // default context timeout + cxTimeout = 300 * time.Second +) + +var ( + schema = map[string]update.Item{ + ".github/workflows/iso.yml": { + Replace: map[string]string{ + `go-version: '.*`: `go-version: '{{.StableVersion}}'`, + }, + }, + ".github/workflows/kic_image.yml": { + Replace: map[string]string{ + `go-version: '.*`: `go-version: '{{.StableVersion}}'`, + }, + }, + ".github/workflows/master.yml": { + Replace: map[string]string{ + `go-version: '.*`: `go-version: '{{.StableVersion}}'`, + }, + }, + ".github/workflows/pr.yml": { + Replace: map[string]string{ + `go-version: '.*`: `go-version: '{{.StableVersion}}'`, + }, + }, + ".travis.yml": { + Replace: map[string]string{ + `go:\n - .*`: `go:{{printf "\n - %s" .StableVersion}}`, + `go: .*`: `go: {{.StableVersion}}`, + }, + }, + "go.mod": { + Replace: map[string]string{ + `(?m)^go .*`: `go {{.StableVersionMM}}`, + }, + }, + "hack/jenkins/common.sh": { + Replace: map[string]string{ + `sudo \.\/installers\/check_install_golang\.sh \".*\" \"\/usr\/local\"`: `sudo ./installers/check_install_golang.sh "{{.StableVersion}}" "/usr/local"`, + }, + }, + "Makefile": { + Replace: map[string]string{ + `GO_VERSION \?= .*`: `GO_VERSION ?= {{.StableVersion}}`, + }, + }, + } + + // pull request data + prBranchPrefix = "update-golang-version_" // will be appended with first 7 characters of the PR commit SHA + prTitle = `update_golang_version: {stable:"{{.StableVersion}}"}` + prIssue = 9264 +) + +// Data holds stable Golang version +type Data struct { + StableVersion string `json:"stableVersion"` + StableVersionMM string `json:"stableVersionMM"` // go.mod wants go version in . format +} + +func main() { + // set a context with defined timeout + ctx, cancel := context.WithTimeout(context.Background(), cxTimeout) + defer cancel() + + // get Golang stable version + stable, stableMM, err := goVersions() + if err != nil || stable == "" || stableMM == "" { + klog.Fatalf("Error getting Golang stable version: %v", err) + } + data := Data{StableVersion: stable, StableVersionMM: stableMM} + klog.Infof("Golang stable version: %s", data.StableVersion) + + update.Apply(ctx, schema, data, prBranchPrefix, prTitle, prIssue) +} + +// goVersion returns Golang stable version +func goVersions() (stable, stableMM string, err error) { + resp, err := http.Get("https://golang.org/VERSION?m=text") + if err != nil { + return "", "", err + } + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", "", err + } + stable = strings.TrimPrefix(string(body), "go") + mmp := strings.SplitN(stable, ".", 3) + stableMM = strings.Join(mmp[0:2], ".") // . version + return stable, stableMM, nil +} diff --git a/hack/update/kicbase_version/update_kicbase_version.go b/hack/update/kicbase_version/update_kicbase_version.go new file mode 100644 index 000000000000..64d9432475ed --- /dev/null +++ b/hack/update/kicbase_version/update_kicbase_version.go @@ -0,0 +1,148 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Script promotes current KIC base image as stable, ie: + - strips current version suffix starting from '-' in pkg/drivers/kic/types.go => release version + (eg, 'v0.0.13-snapshot1' -> 'v0.0.13') + - makes sure current KIC base image exists locally, tries to pull one if not + - tags current KIC base image with the release version, and + - pushes it to all relevant container registries + +Script expects the following env variables: + - UPDATE_TARGET=: optional - if unset/absent, default option is "fs"; valid options are: + - "fs" - update only local filesystem repo files [default] + - "gh" - update only remote GitHub repo files and create PR (if one does not exist already) + - "all" - update local and remote repo files and create PR (if one does not exist already) + +Script also requires following credentials as env variables (injected by Jenkins credential provider): + @GCR (ref: https://cloud.google.com/container-registry/docs/advanced-authentication): + - GCR_USERNAME=: GCR username, eg: + = "oauth2accesstoken" if Access Token is used for GCR_TOKEN, or + = "_json_key" if JSON Key File is used for GCR_TOKEN + - GCR_TOKEN=: GCR JSON token + + @Docker (ref: https://docs.docker.com/docker-hub/access-tokens/) + - DOCKER_USERNAME=: Docker username + - DOCKER_TOKEN=: Docker personal access token or password + + @GitHub (ref: https://docs.github.com/en/free-pro-team@latest/packages/using-github-packages-with-your-projects-ecosystem/configuring-docker-for-use-with-github-packages) + - GITHUB_USERNAME=: GitHub username + - GITHUB_TOKEN=: GitHub [personal] access token +*/ + +package main + +import ( + "context" + "io/ioutil" + "path/filepath" + "regexp" + "strings" + "time" + + "k8s.io/klog/v2" + "k8s.io/minikube/hack/update" +) + +const ( + // default context timeout + cxTimeout = 600 * time.Second +) + +var ( + kicFile = "pkg/drivers/kic/types.go" + versionRE = `Version = "(.*)"` + + schema = map[string]update.Item{ + kicFile: { + Replace: map[string]string{ + `Version = ".*"`: `Version = "{{.StableVersion}}"`, + }, + }, + } + + // pull request data + prBranchPrefix = "update-kicbase-version_" // will be appended with first 7 characters of the PR commit SHA + prTitle = `update-kicbase-version: {"{{.StableVersion}}"}` + prIssue = 9420 +) + +// Data holds current and stable KIC Base image versions +type Data struct { + CurrentVersion string `json:"CurrentVersion"` + StableVersion string `json:"StableVersion"` +} + +func main() { + // set a context with defined timeout + ctx, cancel := context.WithTimeout(context.Background(), cxTimeout) + defer cancel() + + // determine current and stable kic base image versions + current, stable, err := KICVersions() + if err != nil { + klog.Fatalf("failed getting kic base image versions: %v", err) + } + if len(current) == 0 || len(stable) == 0 { + klog.Fatalf("cannot determine kic base image versions") + } + data := Data{CurrentVersion: current, StableVersion: stable} + klog.Infof("kic base image versions: 'current' is %s and 'stable' would be %s", data.CurrentVersion, data.StableVersion) + + // prepare local kic base image + image, err := prepareImage(ctx, data) + if err != nil { + klog.Fatalf("failed preparing local kic base reference image: %v", err) + } + klog.Infof("local kic base reference image: %s", image) + + // update registries + if updated := update.CRUpdateAll(ctx, image, data.StableVersion); !updated { + klog.Fatalf("failed updating all registries") + } + + update.Apply(ctx, schema, data, prBranchPrefix, prTitle, prIssue) +} + +// KICVersions returns current and stable kic base image versions and any error +func KICVersions() (current, stable string, err error) { + blob, err := ioutil.ReadFile(filepath.Join(update.FSRoot, kicFile)) + if err != nil { + return "", "", err + } + re := regexp.MustCompile(versionRE) + ver := re.FindSubmatch(blob) + if ver == nil { + return "", "", nil + } + current = string(ver[1]) + stable = strings.Split(current, "-")[0] + return current, stable, nil +} + +// prepareImage checks if current image exists locally, tries to pull it if not, +// tags it with release version, returns reference image url and any error +func prepareImage(ctx context.Context, data Data) (image string, err error) { + image, err = update.PullImage(ctx, data.CurrentVersion, data.StableVersion) + if err != nil { + return "", err + } + if err := update.TagImage(ctx, image, data.CurrentVersion, data.StableVersion); err != nil { + return "", err + } + return image, nil +} diff --git a/hack/update/kubernetes_version/update_kubernetes_version.go b/hack/update/kubernetes_version/update_kubernetes_version.go new file mode 100644 index 000000000000..b5bf60cef760 --- /dev/null +++ b/hack/update/kubernetes_version/update_kubernetes_version.go @@ -0,0 +1,84 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Script expects the following env variables: + - UPDATE_TARGET=: optional - if unset/absent, default option is "fs"; valid options are: + - "fs" - update only local filesystem repo files [default] + - "gh" - update only remote GitHub repo files and create PR (if one does not exist already) + - "all" - update local and remote repo files and create PR (if one does not exist already) + - GITHUB_TOKEN=: GitHub [personal] access token + - note: GITHUB_TOKEN is required if UPDATE_TARGET is "gh" or "all" +*/ + +package main + +import ( + "context" + "time" + + "k8s.io/klog/v2" + "k8s.io/minikube/hack/update" +) + +const ( + // default context timeout + cxTimeout = 300 * time.Second +) + +var ( + schema = map[string]update.Item{ + "pkg/minikube/constants/constants.go": { + Replace: map[string]string{ + `DefaultKubernetesVersion = ".*`: `DefaultKubernetesVersion = "{{.StableVersion}}"`, + `NewestKubernetesVersion = ".*`: `NewestKubernetesVersion = "{{.LatestVersion}}"`, + }, + }, + "site/content/en/docs/commands/start.md": { + Replace: map[string]string{ + `'stable' for .*,`: `'stable' for {{.StableVersion}},`, + `'latest' for .*\)`: `'latest' for {{.LatestVersion}})`, + }, + }, + } + + // pull request data + prBranchPrefix = "update-kubernetes-version_" // will be appended with first 7 characters of the PR commit SHA + prTitle = `update_kubernetes_version: {stable:"{{.StableVersion}}", latest:"{{.LatestVersion}}"}` + prIssue = 4392 +) + +// Data holds stable and latest Kubernetes versions +type Data struct { + StableVersion string `json:"StableVersion"` + LatestVersion string `json:"LatestVersion"` +} + +func main() { + // set a context with defined timeout + ctx, cancel := context.WithTimeout(context.Background(), cxTimeout) + defer cancel() + + // get Kubernetes versions from GitHub Releases + stable, latest, err := update.GHVersions(ctx, "kubernetes", "kubernetes") + if err != nil || stable == "" || latest == "" { + klog.Fatalf("Error getting Kubernetes versions: %v", err) + } + data := Data{StableVersion: stable, LatestVersion: latest} + klog.Infof("Kubernetes versions: 'stable' is %s and 'latest' is %s", data.StableVersion, data.LatestVersion) + + update.Apply(ctx, schema, data, prBranchPrefix, prTitle, prIssue) +} diff --git a/hack/update/registry.go b/hack/update/registry.go new file mode 100644 index 000000000000..5ce7bb1a93f8 --- /dev/null +++ b/hack/update/registry.go @@ -0,0 +1,134 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package update + +import ( + "context" + "fmt" + "os" + "os/exec" + "strings" + "time" + + "k8s.io/klog/v2" +) + +var ( + // keep list of registries in sync with those in "pkg/drivers/kic/types.go" + registries = []registry{ + { + name: "Google Cloud Container Registry", + image: "gcr.io/k8s-minikube/kicbase", + username: os.Getenv("GCR_USERNAME"), + password: os.Getenv("GCR_TOKEN"), + }, + { + name: "Docker Hub Container Registry", + image: "docker.io/kicbase/stable", + username: os.Getenv("DOCKER_USERNAME"), + password: os.Getenv("DOCKER_TOKEN"), + }, + { + name: "GitHub Packages Registry", + image: "docker.pkg.github.com/kubernetes/minikube/kicbase", + username: os.Getenv("GITHUB_USERNAME"), + password: os.Getenv("GITHUB_TOKEN"), + }, + } +) + +// container registry name, image path, credentials, and updated flag +type registry struct { + name string + image string + username string + password string +} + +// crUpdate tags image with version, pushes it to container registry, and returns any error +func crUpdate(ctx context.Context, reg registry, image, version string) error { + login := exec.CommandContext(ctx, "docker", "login", "--username", reg.username, "--password-stdin", reg.image) + if err := RunWithRetryNotify(ctx, login, strings.NewReader(reg.password), 1*time.Minute, 10); err != nil { + return fmt.Errorf("failed logging in to %s: %w", reg.name, err) + } + klog.Infof("successfully logged in to %s", reg.name) + + tag := exec.CommandContext(ctx, "docker", "tag", image+":"+version, reg.image+":"+version) + if err := RunWithRetryNotify(ctx, tag, nil, 1*time.Minute, 10); err != nil { + return fmt.Errorf("failed tagging %s for %s: %w", reg.image+":"+version, reg.name, err) + } + klog.Infof("successfully tagged %s for %s", reg.image+":"+version, reg.name) + + push := exec.CommandContext(ctx, "docker", "push", reg.image+":"+version) + if err := RunWithRetryNotify(ctx, push, nil, 2*time.Minute, 10); err != nil { + return fmt.Errorf("failed pushing %s to %s: %w", reg.image+":"+version, reg.name, err) + } + klog.Infof("successfully pushed %s to %s", reg.image+":"+version, reg.name) + + return nil +} + +// CRUpdateAll calls crUpdate for each available registry, and returns if at least one got updated +func CRUpdateAll(ctx context.Context, image, version string) (updated bool) { + for _, reg := range registries { + if err := crUpdate(ctx, reg, image, version); err != nil { + klog.Errorf("failed updating %s", reg.name) + continue + } + klog.Infof("successfully updated %s", reg.name) + updated = true + } + return updated +} + +// PullImage checks if current image exists locally, tries to pull it if not, and +// returns reference image url and any error +func PullImage(ctx context.Context, current, release string) (image string, err error) { + // check if image exists locally + for _, reg := range registries { + inspect := exec.CommandContext(ctx, "docker", "inspect", reg.image+":"+current, "--format", "{{.Id}}") + if err := RunWithRetryNotify(ctx, inspect, nil, 1*time.Second, 10); err != nil { + continue + } + image = reg.image + break + } + if image == "" { + // try to pull image locally + for _, reg := range registries { + pull := exec.CommandContext(ctx, "docker", "pull", reg.image+":"+current) + if err := RunWithRetryNotify(ctx, pull, nil, 2*time.Minute, 10); err != nil { + continue + } + image = reg.image + break + } + } + if image == "" { + return "", fmt.Errorf("cannot find current image version tag %s locally nor in any registry", current) + } + return image, nil +} + +// TagImage tags local image:current with stable version, and returns any error +func TagImage(ctx context.Context, image, current, stable string) error { + tag := exec.CommandContext(ctx, "docker", "tag", image+":"+current, image+":"+stable) + if err := RunWithRetryNotify(ctx, tag, nil, 1*time.Second, 10); err != nil { + return err + } + return nil +} diff --git a/hack/update/update.go b/hack/update/update.go new file mode 100644 index 000000000000..385964bfa341 --- /dev/null +++ b/hack/update/update.go @@ -0,0 +1,212 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Script expects the following env variables: + - UPDATE_TARGET=: optional - if unset/absent, default option is "fs"; valid options are: + - "fs" - update only local filesystem repo files [default] + - "gh" - update only remote GitHub repo files and create PR (if one does not exist already) + - "all" - update local and remote repo files and create PR (if one does not exist already) + - GITHUB_TOKEN=: GitHub [personal] access token + - note: GITHUB_TOKEN is required if UPDATE_TARGET is "gh" or "all" +*/ + +package update + +import ( + "bytes" + "context" + "encoding/json" + "flag" + "fmt" + "io" + "os" + "os/exec" + "regexp" + "text/template" + "time" + + "k8s.io/klog/v2" + + "github.com/cenkalti/backoff/v4" +) + +const ( + // FSRoot is relative (to scripts in subfolders) root folder of local filesystem repo to update + FSRoot = "../../../" +) + +var ( + target = os.Getenv("UPDATE_TARGET") +) + +// init klog and check general requirements +func init() { + // write log statements to stderr instead of to files + if err := flag.Set("logtostderr", "true"); err != nil { + fmt.Printf("Error setting 'logtostderr' klog flag: %v", err) + } + flag.Parse() + defer klog.Flush() + + if target == "" { + target = "fs" + } else if target != "fs" && target != "gh" && target != "all" { + klog.Fatalf("Invalid UPDATE_TARGET option: '%s'; Valid options are: unset/absent (defaults to 'fs'), 'fs', 'gh', or 'all'", target) + } else if (target == "gh" || target == "all") && ghToken == "" { + klog.Fatalf("GITHUB_TOKEN is required if UPDATE_TARGET is 'gh' or 'all'") + } +} + +// Item defines Content where all occurrences of each Replace map key, corresponding to +// GitHub TreeEntry.Path and/or local filesystem repo file path (prefixed with FSRoot), +// would be swapped with its respective actual map value (having placeholders replaced with data), +// creating a concrete update plan. +// Replace map keys can use RegExp and map values can use Golang Text Template +type Item struct { + Content []byte `json:"-"` + Replace map[string]string `json:"replace"` +} + +// apply updates Item Content by replacing all occurrences of Replace map's keys +// with their actual map values (with placeholders replaced with data)) +func (i *Item) apply(data interface{}) (changed bool, err error) { + if i.Content == nil || i.Replace == nil { + return false, fmt.Errorf("want something, got nothing to update") + } + org := string(i.Content) + str := org + for src, dst := range i.Replace { + tmpl := template.Must(template.New("").Parse(dst)) + buf := new(bytes.Buffer) + if err := tmpl.Execute(buf, data); err != nil { + return false, err + } + re := regexp.MustCompile(src) + str = re.ReplaceAllString(str, buf.String()) + } + i.Content = []byte(str) + + return str != org, nil +} + +// Apply applies concrete update plan (schema + data) to GitHub or local filesystem repo +func Apply(ctx context.Context, schema map[string]Item, data interface{}, prBranchPrefix, prTitle string, prIssue int) { + plan, err := GetPlan(schema, data) + if err != nil { + klog.Fatalf("Error parsing schema: %v\n%s", err, plan) + } + klog.Infof("The Plan:\n%s", plan) + + if target == "fs" || target == "all" { + changed, err := fsUpdate(FSRoot, schema, data) + if err != nil { + klog.Errorf("Error updating local repo: %v", err) + } else if !changed { + klog.Infof("Local repo update skipped: nothing changed") + } else { + klog.Infof("Local repo updated") + } + } + + if target == "gh" || target == "all" { + // update prTitle replacing template placeholders with actual data values + tmpl := template.Must(template.New("prTitle").Parse(prTitle)) + buf := new(bytes.Buffer) + if err := tmpl.Execute(buf, data); err != nil { + klog.Fatalf("Error parsing PR Title: %v", err) + } + prTitle = buf.String() + + // check if PR already exists + prURL, err := ghFindPR(ctx, prTitle, ghOwner, ghRepo, ghBase, ghToken) + if err != nil { + klog.Errorf("Error checking if PR already exists: %v", err) + } else if prURL != "" { + klog.Infof("PR create skipped: already exists (%s)", prURL) + } else { + // create PR + pr, err := ghCreatePR(ctx, ghOwner, ghRepo, ghBase, prBranchPrefix, prTitle, prIssue, ghToken, schema, data) + if err != nil { + klog.Fatalf("Error creating PR: %v", err) + } else if pr == nil { + klog.Infof("PR create skipped: nothing changed") + } else { + klog.Infof("PR created: %s", *pr.HTMLURL) + } + } + } +} + +// GetPlan returns concrete plan replacing placeholders in schema with actual data values, +// returns JSON-formatted representation of the plan and any error +func GetPlan(schema map[string]Item, data interface{}) (prettyprint string, err error) { + for _, item := range schema { + for src, dst := range item.Replace { + tmpl := template.Must(template.New("").Parse(dst)) + buf := new(bytes.Buffer) + if err := tmpl.Execute(buf, data); err != nil { + return fmt.Sprintf("%+v", schema), err + } + item.Replace[src] = buf.String() + } + } + str, err := json.MarshalIndent(schema, "", " ") + if err != nil { + return fmt.Sprintf("%+v", schema), err + } + return string(str), nil +} + +// RunWithRetryNotify runs command cmd with stdin using exponential backoff for maxTime duration +// up to maxRetries (negative values will make it ignored), +// notifies about any intermediary errors and return any final error. +// similar to pkg/util/retry/retry.go:Expo(), just for commands with params and also with context +func RunWithRetryNotify(ctx context.Context, cmd *exec.Cmd, stdin io.Reader, maxTime time.Duration, maxRetries uint64) error { + be := backoff.NewExponentialBackOff() + be.Multiplier = 2 + be.MaxElapsedTime = maxTime + bm := backoff.WithMaxRetries(be, maxRetries) + bc := backoff.WithContext(bm, ctx) + + notify := func(err error, wait time.Duration) { + klog.Errorf("Temporary error running '%s' (will retry in %s): %v", cmd.String(), wait, err) + } + if err := backoff.RetryNotify(func() error { + cmd.Stdin = stdin + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + time.Sleep(be.NextBackOff().Round(1 * time.Second)) + return fmt.Errorf("%w: %s", err, stderr.String()) + } + return nil + }, bc, notify); err != nil { + return err + } + return nil +} + +// Run runs command cmd with stdin +func Run(cmd *exec.Cmd, stdin io.Reader) error { + cmd.Stdin = stdin + var out bytes.Buffer + cmd.Stderr = &out + if err := cmd.Run(); err != nil { + return fmt.Errorf("%w: %s", err, out.String()) + } + return nil +} From f5901636d32f7759135125e3e405675f7bf0d9d1 Mon Sep 17 00:00:00 2001 From: prezha Date: Wed, 14 Oct 2020 01:44:57 +0100 Subject: [PATCH 2/8] update: consolidation & automation --- go.mod | 2 ++ go.sum | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/go.mod b/go.mod index 6170d4656e42..66cc1f1b32e1 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/blang/semver v3.5.0+incompatible github.com/c4milo/gotoolkit v0.0.0-20170318115440-bcc06269efa9 // indirect github.com/cenkalti/backoff v2.2.1+incompatible + github.com/cenkalti/backoff/v4 v4.1.0 github.com/cheggaaa/pb/v3 v3.0.1 github.com/cloudevents/sdk-go/v2 v2.1.0 github.com/cloudfoundry-attic/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21 @@ -85,6 +86,7 @@ require ( k8s.io/api v0.17.4 k8s.io/apimachinery v0.17.4 k8s.io/client-go v0.17.4 + k8s.io/klog/v2 v2.3.0 k8s.io/kubectl v0.0.0 k8s.io/kubernetes v1.18.5 sigs.k8s.io/sig-storage-lib-external-provisioner v4.0.0+incompatible // indirect diff --git a/go.sum b/go.sum index 854420fd06c2..31b39eb4de05 100644 --- a/go.sum +++ b/go.sum @@ -196,6 +196,8 @@ github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e/go.mod h1:oD github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/cenkalti/backoff/v4 v4.1.0 h1:c8LkOFQTzuO0WBM/ae5HdGQuZPfPxp7lqBRwQRm4fSc= +github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/prettybench v0.0.0-20150116022406-03b8cfe5406c/go.mod h1:Xe6ZsFhtM8HrDku0pxJ3/Lr51rwykrzgFwpmTzleatY= @@ -352,6 +354,8 @@ github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTD github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= @@ -1638,6 +1642,8 @@ k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUc k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +k8s.io/klog/v2 v2.3.0 h1:WmkrnW7fdrm0/DMClc+HIxtftvxVIPAhlVwMQo5yLco= +k8s.io/klog/v2 v2.3.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/kube-aggregator v0.17.3/go.mod h1:1dMwMFQbmH76RKF0614L7dNenMl3dwnUJuOOyZ3GMXA= k8s.io/kube-controller-manager v0.17.3/go.mod h1:22B/TsgVviuCVuNwUrqgyTi5D4AYjMFaK9c8h1oonkY= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU= From 885421ac37d8ef3d28479f1c90a3f6b001c5ac74 Mon Sep 17 00:00:00 2001 From: Predrag Rogic Date: Fri, 16 Oct 2020 22:43:10 +0100 Subject: [PATCH 3/8] get greatest instead of just latest version --- .../update_kubernetes_version.go | 28 ++++++++----------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/hack/kubernetes_version/update_kubernetes_version.go b/hack/kubernetes_version/update_kubernetes_version.go index e61dde88db06..2698c82f14fc 100644 --- a/hack/kubernetes_version/update_kubernetes_version.go +++ b/hack/kubernetes_version/update_kubernetes_version.go @@ -40,6 +40,7 @@ import ( "text/template" "time" + "golang.org/x/mod/semver" "golang.org/x/oauth2" "github.com/golang/glog" @@ -398,26 +399,19 @@ func ghReleases(ctx context.Context, owner, repo, token string) (stable, latest } for _, rl := range rls { ver := rl.GetName() - if ver == "" { + if !semver.IsValid(ver) { continue } - // check if ver version is a release (ie, 'v1.19.2') or a - // pre-release (ie, 'v1.19.3-rc.0' or 'v1.19.0-beta.2') channel ch - // note: github.RepositoryRelease GetPrerelease() bool would be useful for all pre-rels - ch := strings.Split(ver, "-") - if len(ch) == 1 && stable == "" { - stable = ver - } else if len(ch) > 1 && latest == "" { - if strings.HasPrefix(ch[1], "rc") || strings.HasPrefix(ch[1], "beta") { - latest = ver - } + // check if ver version is release (ie, 'v1.19.2') or pre-release (ie, 'v1.19.3-rc.0' or 'v1.19.0-beta.2') + prerls := semver.Prerelease(ver) + if prerls == "" { + stable = semver.Max(ver, stable) + } else if strings.HasPrefix(prerls, "-rc") || strings.HasPrefix(prerls, "-beta") { + latest = semver.Max(ver, latest) } - if stable != "" && latest != "" { - // make sure that v.Latest >= stable - if latest < stable { - latest = stable - } - return stable, latest, nil + // make sure that latest >= stable + if semver.Compare(latest, stable) == -1 { + latest = stable } } if resp.NextPage == 0 { From 5e602b866284a7373e6f2c32cfca5b92c596eaf1 Mon Sep 17 00:00:00 2001 From: Predrag Rogic Date: Fri, 16 Oct 2020 22:46:12 +0100 Subject: [PATCH 4/8] get greatest instead of just latest version --- go.mod | 1 + 1 file changed, 1 insertion(+) diff --git a/go.mod b/go.mod index 66cc1f1b32e1..8189d18da4d6 100644 --- a/go.mod +++ b/go.mod @@ -75,6 +75,7 @@ require ( golang.org/x/build v0.0.0-20190927031335-2835ba2e683f golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 + golang.org/x/mod v0.3.0 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a golang.org/x/sys v0.0.0-20200523222454-059865788121 From fa17b9641c55352ea3daad41df2045b59b355001 Mon Sep 17 00:00:00 2001 From: Predrag Rogic Date: Sat, 17 Oct 2020 00:07:25 +0100 Subject: [PATCH 5/8] get greatest instead of just latest version --- .../update_kubernetes_version.go | 43 ++++++++++--------- hack/update/github.go | 30 ++++++------- 2 files changed, 34 insertions(+), 39 deletions(-) diff --git a/hack/kubernetes_version/update_kubernetes_version.go b/hack/kubernetes_version/update_kubernetes_version.go index 2698c82f14fc..50af8e6c8186 100644 --- a/hack/kubernetes_version/update_kubernetes_version.go +++ b/hack/kubernetes_version/update_kubernetes_version.go @@ -43,8 +43,9 @@ import ( "golang.org/x/mod/semver" "golang.org/x/oauth2" - "github.com/golang/glog" "github.com/google/go-github/v32/github" + + "k8s.io/klog/v2" ) const ( @@ -128,17 +129,17 @@ func (p *Patch) apply(data interface{}) (changed bool, err error) { func main() { // write log statements to stderr instead of to files if err := flag.Set("logtostderr", "true"); err != nil { - fmt.Printf("Error setting 'logtostderr' glog flag: %v", err) + fmt.Printf("Error setting 'logtostderr' klog flag: %v", err) } flag.Parse() - defer glog.Flush() + defer klog.Flush() if target == "" { target = "fs" } else if target != "fs" && target != "gh" && target != "all" { - glog.Fatalf("Invalid UPDATE_TARGET option: '%s'; Valid options are: unset/absent (defaults to 'fs'), 'fs', 'gh', or 'all'", target) + klog.Fatalf("Invalid UPDATE_TARGET option: '%s'; Valid options are: unset/absent (defaults to 'fs'), 'fs', 'gh', or 'all'", target) } else if (target == "gh" || target == "all") && ghToken == "" { - glog.Fatalf("GITHUB_TOKEN is required if UPDATE_TARGET is 'gh' or 'all'") + klog.Fatalf("GITHUB_TOKEN is required if UPDATE_TARGET is 'gh' or 'all'") } // set a context with defined timeout @@ -148,21 +149,21 @@ func main() { // get Kubernetes versions from GitHub Releases stable, latest, err := ghReleases(ctx, "kubernetes", "kubernetes", ghToken) if err != nil || stable == "" || latest == "" { - glog.Fatalf("Error getting Kubernetes versions: %v", err) + klog.Fatalf("Error getting Kubernetes versions: %v", err) } data := Data{K8sStableVersion: stable, K8sLatestVersion: latest} - glog.Infof("Kubernetes versions: 'stable' is %s and 'latest' is %s", data.K8sStableVersion, data.K8sLatestVersion) + klog.Infof("Kubernetes versions: 'stable' is %s and 'latest' is %s", data.K8sStableVersion, data.K8sLatestVersion) - glog.Infof("The Plan:\n%s", thePlan(plan, data)) + klog.Infof("The Plan:\n%s", thePlan(plan, data)) if target == "fs" || target == "all" { changed, err := fsUpdate(fsRoot, plan, data) if err != nil { - glog.Errorf("Error updating local repo: %v", err) + klog.Errorf("Error updating local repo: %v", err) } else if !changed { - glog.Infof("Local repo update skipped: nothing changed") + klog.Infof("Local repo update skipped: nothing changed") } else { - glog.Infof("Local repo updated") + klog.Infof("Local repo updated") } } @@ -171,25 +172,25 @@ func main() { tmpl := template.Must(template.New("prTitle").Parse(prTitle)) buf := new(bytes.Buffer) if err := tmpl.Execute(buf, data); err != nil { - glog.Fatalf("Error parsing PR Title: %v", err) + klog.Fatalf("Error parsing PR Title: %v", err) } prTitle = buf.String() // check if PR already exists prURL, err := ghFindPR(ctx, prTitle, ghOwner, ghRepo, ghBase, ghToken) if err != nil { - glog.Errorf("Error checking if PR already exists: %v", err) + klog.Errorf("Error checking if PR already exists: %v", err) } else if prURL != "" { - glog.Infof("PR create skipped: already exists (%s)", prURL) + klog.Infof("PR create skipped: already exists (%s)", prURL) } else { // create PR pr, err := ghCreatePR(ctx, ghOwner, ghRepo, ghBase, prBranchPrefix, prTitle, prIssue, ghToken, plan, data) if err != nil { - glog.Fatalf("Error creating PR: %v", err) + klog.Fatalf("Error creating PR: %v", err) } else if pr == nil { - glog.Infof("PR create skipped: nothing changed") + klog.Infof("PR create skipped: nothing changed") } else { - glog.Infof("PR created: %s", *pr.HTMLURL) + klog.Infof("PR created: %s", *pr.HTMLURL) } } } @@ -288,7 +289,7 @@ func ghCreatePR(ctx context.Context, owner, repo, base, branch, title string, is if err != nil { return nil, fmt.Errorf("error creating fork commit: %w", err) } - glog.Infof("PR commit '%s' created: %s", forkCommit.GetSHA(), forkCommit.GetHTMLURL()) + klog.Infof("PR commit '%s' created: %s", forkCommit.GetSHA(), forkCommit.GetHTMLURL()) // create PR branch prBranch := branch + forkCommit.GetSHA()[:7] @@ -302,7 +303,7 @@ func ghCreatePR(ctx context.Context, owner, repo, base, branch, title string, is if err != nil { return nil, fmt.Errorf("error creating PR branch: %w", err) } - glog.Infof("PR branch '%s' created: %s", prBranch, prRef.GetURL()) + klog.Infof("PR branch '%s' created: %s", prBranch, prRef.GetURL()) // create PR modifiable := true @@ -441,7 +442,7 @@ func thePlan(plan map[string]Patch, data Data) (prettyprint string) { tmpl := template.Must(template.New("").Parse(dst)) buf := new(bytes.Buffer) if err := tmpl.Execute(buf, data); err != nil { - glog.Fatalf("Error parsing the Plan: %v", err) + klog.Fatalf("Error parsing the Plan: %v", err) return fmt.Sprintf("%+v", plan) } p.Replace[src] = buf.String() @@ -449,7 +450,7 @@ func thePlan(plan map[string]Patch, data Data) (prettyprint string) { } str, err := json.MarshalIndent(plan, "", " ") if err != nil { - glog.Fatalf("Error parsing the Plan: %v", err) + klog.Fatalf("Error parsing the Plan: %v", err) return fmt.Sprintf("%+v", plan) } return string(str) diff --git a/hack/update/github.go b/hack/update/github.go index bc82e2ea1dfc..661cbc0fa5ce 100644 --- a/hack/update/github.go +++ b/hack/update/github.go @@ -23,6 +23,7 @@ import ( "strings" "time" + "golang.org/x/mod/semver" "golang.org/x/oauth2" "github.com/google/go-github/v32/github" @@ -220,7 +221,7 @@ func ghClient(ctx context.Context, token string) *github.Client { return github.NewClient(tc) } -// GHVersions returns current stable release and latest rc or beta pre-release +// GHVersions returns greatest current stable release and greatest latest rc or beta pre-release // from GitHub owner/repo repository, and any error; // if latest pre-release version is lower than current stable release, then it // will return current stable release for both @@ -236,26 +237,19 @@ func GHVersions(ctx context.Context, owner, repo string) (stable, latest string, } for _, rl := range rls { ver := rl.GetName() - if ver == "" { + if !semver.IsValid(ver) { continue } - // check if ver version is a release (ie, 'v1.19.2') or a - // pre-release (ie, 'v1.19.3-rc.0' or 'v1.19.0-beta.2') channel ch - // note: github.RepositoryRelease GetPrerelease() bool would be useful for all pre-rels - ch := strings.Split(ver, "-") - if len(ch) == 1 && stable == "" { - stable = ver - } else if len(ch) > 1 && latest == "" { - if strings.HasPrefix(ch[1], "rc") || strings.HasPrefix(ch[1], "beta") { - latest = ver - } + // check if ver version is release (ie, 'v1.19.2') or pre-release (ie, 'v1.19.3-rc.0' or 'v1.19.0-beta.2') + prerls := semver.Prerelease(ver) + if prerls == "" { + stable = semver.Max(ver, stable) + } else if strings.HasPrefix(prerls, "-rc") || strings.HasPrefix(prerls, "-beta") { + latest = semver.Max(ver, latest) } - if stable != "" && latest != "" { - // make sure that v.Latest >= stable - if latest < stable { - latest = stable - } - return stable, latest, nil + // make sure that latest >= stable + if semver.Compare(latest, stable) == -1 { + latest = stable } } if resp.NextPage == 0 { From e42791ea0281f11f1132f7d422b55724a45d513c Mon Sep 17 00:00:00 2001 From: Predrag Rogic Date: Sat, 17 Oct 2020 00:43:19 +0100 Subject: [PATCH 6/8] fix klog 'no such flag -logtostderr' with InitFlags --- hack/kubernetes_version/update_kubernetes_version.go | 3 ++- hack/update/update.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/hack/kubernetes_version/update_kubernetes_version.go b/hack/kubernetes_version/update_kubernetes_version.go index 50af8e6c8186..8d68b61324f6 100644 --- a/hack/kubernetes_version/update_kubernetes_version.go +++ b/hack/kubernetes_version/update_kubernetes_version.go @@ -127,9 +127,10 @@ func (p *Patch) apply(data interface{}) (changed bool, err error) { } func main() { + klog.InitFlags(nil) // write log statements to stderr instead of to files if err := flag.Set("logtostderr", "true"); err != nil { - fmt.Printf("Error setting 'logtostderr' klog flag: %v", err) + fmt.Printf("Error setting 'logtostderr' klog flag: %v\n", err) } flag.Parse() defer klog.Flush() diff --git a/hack/update/update.go b/hack/update/update.go index 385964bfa341..fc478b156317 100644 --- a/hack/update/update.go +++ b/hack/update/update.go @@ -55,9 +55,10 @@ var ( // init klog and check general requirements func init() { + klog.InitFlags(nil) // write log statements to stderr instead of to files if err := flag.Set("logtostderr", "true"); err != nil { - fmt.Printf("Error setting 'logtostderr' klog flag: %v", err) + fmt.Printf("Error setting 'logtostderr' klog flag: %v\n", err) } flag.Parse() defer klog.Flush() From 1337811d49f8c11ea89b3acf507da1cf15950308 Mon Sep 17 00:00:00 2001 From: Predrag Rogic Date: Sat, 17 Oct 2020 00:52:53 +0100 Subject: [PATCH 7/8] change hack/kubernetes_version/update_kubernetes_version.go in a separate pr[9480] --- .../update_kubernetes_version.go | 75 ++++++++++--------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/hack/kubernetes_version/update_kubernetes_version.go b/hack/kubernetes_version/update_kubernetes_version.go index 8d68b61324f6..7a912c9a2303 100644 --- a/hack/kubernetes_version/update_kubernetes_version.go +++ b/hack/kubernetes_version/update_kubernetes_version.go @@ -1,12 +1,9 @@ /* Copyright 2020 The Kubernetes Authors All rights reserved. - Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -40,12 +37,10 @@ import ( "text/template" "time" - "golang.org/x/mod/semver" "golang.org/x/oauth2" + "github.com/golang/glog" "github.com/google/go-github/v32/github" - - "k8s.io/klog/v2" ) const ( @@ -127,20 +122,19 @@ func (p *Patch) apply(data interface{}) (changed bool, err error) { } func main() { - klog.InitFlags(nil) // write log statements to stderr instead of to files if err := flag.Set("logtostderr", "true"); err != nil { - fmt.Printf("Error setting 'logtostderr' klog flag: %v\n", err) + fmt.Printf("Error setting 'logtostderr' glog flag: %v", err) } flag.Parse() - defer klog.Flush() + defer glog.Flush() if target == "" { target = "fs" } else if target != "fs" && target != "gh" && target != "all" { - klog.Fatalf("Invalid UPDATE_TARGET option: '%s'; Valid options are: unset/absent (defaults to 'fs'), 'fs', 'gh', or 'all'", target) + glog.Fatalf("Invalid UPDATE_TARGET option: '%s'; Valid options are: unset/absent (defaults to 'fs'), 'fs', 'gh', or 'all'", target) } else if (target == "gh" || target == "all") && ghToken == "" { - klog.Fatalf("GITHUB_TOKEN is required if UPDATE_TARGET is 'gh' or 'all'") + glog.Fatalf("GITHUB_TOKEN is required if UPDATE_TARGET is 'gh' or 'all'") } // set a context with defined timeout @@ -150,21 +144,21 @@ func main() { // get Kubernetes versions from GitHub Releases stable, latest, err := ghReleases(ctx, "kubernetes", "kubernetes", ghToken) if err != nil || stable == "" || latest == "" { - klog.Fatalf("Error getting Kubernetes versions: %v", err) + glog.Fatalf("Error getting Kubernetes versions: %v", err) } data := Data{K8sStableVersion: stable, K8sLatestVersion: latest} - klog.Infof("Kubernetes versions: 'stable' is %s and 'latest' is %s", data.K8sStableVersion, data.K8sLatestVersion) + glog.Infof("Kubernetes versions: 'stable' is %s and 'latest' is %s", data.K8sStableVersion, data.K8sLatestVersion) - klog.Infof("The Plan:\n%s", thePlan(plan, data)) + glog.Infof("The Plan:\n%s", thePlan(plan, data)) if target == "fs" || target == "all" { changed, err := fsUpdate(fsRoot, plan, data) if err != nil { - klog.Errorf("Error updating local repo: %v", err) + glog.Errorf("Error updating local repo: %v", err) } else if !changed { - klog.Infof("Local repo update skipped: nothing changed") + glog.Infof("Local repo update skipped: nothing changed") } else { - klog.Infof("Local repo updated") + glog.Infof("Local repo updated") } } @@ -173,25 +167,25 @@ func main() { tmpl := template.Must(template.New("prTitle").Parse(prTitle)) buf := new(bytes.Buffer) if err := tmpl.Execute(buf, data); err != nil { - klog.Fatalf("Error parsing PR Title: %v", err) + glog.Fatalf("Error parsing PR Title: %v", err) } prTitle = buf.String() // check if PR already exists prURL, err := ghFindPR(ctx, prTitle, ghOwner, ghRepo, ghBase, ghToken) if err != nil { - klog.Errorf("Error checking if PR already exists: %v", err) + glog.Errorf("Error checking if PR already exists: %v", err) } else if prURL != "" { - klog.Infof("PR create skipped: already exists (%s)", prURL) + glog.Infof("PR create skipped: already exists (%s)", prURL) } else { // create PR pr, err := ghCreatePR(ctx, ghOwner, ghRepo, ghBase, prBranchPrefix, prTitle, prIssue, ghToken, plan, data) if err != nil { - klog.Fatalf("Error creating PR: %v", err) + glog.Fatalf("Error creating PR: %v", err) } else if pr == nil { - klog.Infof("PR create skipped: nothing changed") + glog.Infof("PR create skipped: nothing changed") } else { - klog.Infof("PR created: %s", *pr.HTMLURL) + glog.Infof("PR created: %s", *pr.HTMLURL) } } } @@ -290,7 +284,7 @@ func ghCreatePR(ctx context.Context, owner, repo, base, branch, title string, is if err != nil { return nil, fmt.Errorf("error creating fork commit: %w", err) } - klog.Infof("PR commit '%s' created: %s", forkCommit.GetSHA(), forkCommit.GetHTMLURL()) + glog.Infof("PR commit '%s' created: %s", forkCommit.GetSHA(), forkCommit.GetHTMLURL()) // create PR branch prBranch := branch + forkCommit.GetSHA()[:7] @@ -304,7 +298,7 @@ func ghCreatePR(ctx context.Context, owner, repo, base, branch, title string, is if err != nil { return nil, fmt.Errorf("error creating PR branch: %w", err) } - klog.Infof("PR branch '%s' created: %s", prBranch, prRef.GetURL()) + glog.Infof("PR branch '%s' created: %s", prBranch, prRef.GetURL()) // create PR modifiable := true @@ -401,19 +395,26 @@ func ghReleases(ctx context.Context, owner, repo, token string) (stable, latest } for _, rl := range rls { ver := rl.GetName() - if !semver.IsValid(ver) { + if ver == "" { continue } - // check if ver version is release (ie, 'v1.19.2') or pre-release (ie, 'v1.19.3-rc.0' or 'v1.19.0-beta.2') - prerls := semver.Prerelease(ver) - if prerls == "" { - stable = semver.Max(ver, stable) - } else if strings.HasPrefix(prerls, "-rc") || strings.HasPrefix(prerls, "-beta") { - latest = semver.Max(ver, latest) + // check if ver version is a release (ie, 'v1.19.2') or a + // pre-release (ie, 'v1.19.3-rc.0' or 'v1.19.0-beta.2') channel ch + // note: github.RepositoryRelease GetPrerelease() bool would be useful for all pre-rels + ch := strings.Split(ver, "-") + if len(ch) == 1 && stable == "" { + stable = ver + } else if len(ch) > 1 && latest == "" { + if strings.HasPrefix(ch[1], "rc") || strings.HasPrefix(ch[1], "beta") { + latest = ver + } } - // make sure that latest >= stable - if semver.Compare(latest, stable) == -1 { - latest = stable + if stable != "" && latest != "" { + // make sure that v.Latest >= stable + if latest < stable { + latest = stable + } + return stable, latest, nil } } if resp.NextPage == 0 { @@ -443,7 +444,7 @@ func thePlan(plan map[string]Patch, data Data) (prettyprint string) { tmpl := template.Must(template.New("").Parse(dst)) buf := new(bytes.Buffer) if err := tmpl.Execute(buf, data); err != nil { - klog.Fatalf("Error parsing the Plan: %v", err) + glog.Fatalf("Error parsing the Plan: %v", err) return fmt.Sprintf("%+v", plan) } p.Replace[src] = buf.String() @@ -451,7 +452,7 @@ func thePlan(plan map[string]Patch, data Data) (prettyprint string) { } str, err := json.MarshalIndent(plan, "", " ") if err != nil { - klog.Fatalf("Error parsing the Plan: %v", err) + glog.Fatalf("Error parsing the Plan: %v", err) return fmt.Sprintf("%+v", plan) } return string(str) From bcad74de5df289aae6bd1e7c1156ed9f73181632 Mon Sep 17 00:00:00 2001 From: Predrag Rogic Date: Sat, 17 Oct 2020 00:55:29 +0100 Subject: [PATCH 8/8] change hack/kubernetes_version/update_kubernetes_version.go in a separate pr[9480] --- hack/kubernetes_version/update_kubernetes_version.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hack/kubernetes_version/update_kubernetes_version.go b/hack/kubernetes_version/update_kubernetes_version.go index 7a912c9a2303..e61dde88db06 100644 --- a/hack/kubernetes_version/update_kubernetes_version.go +++ b/hack/kubernetes_version/update_kubernetes_version.go @@ -1,9 +1,12 @@ /* Copyright 2020 The Kubernetes Authors All rights reserved. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.