From 93ec4601718b217b031e84f726c49b40ff681672 Mon Sep 17 00:00:00 2001 From: tadelesh Date: Thu, 12 Aug 2021 13:59:02 +0800 Subject: [PATCH 01/21] [tools/generator] add automation and release command for track2 --- tools/generator/cmd/root.go | 4 + .../cmd/track2/automation/automationCmd.go | 168 +++++++++++++ .../automation/testdata/generate_input.json | 14 ++ .../automation/testdata/generate_output.json | 1 + .../cmd/track2/common/cmdProcessor.go | 16 ++ .../cmd/track2/common/fileProcessor.go | 222 ++++++++++++++++++ .../generator/cmd/track2/common/generation.go | 194 +++++++++++++++ .../cmd/track2/release/releaseCmd.go | 139 +++++++++++ tools/generator/cmd/track2/release/sdk.go | 54 +++++ tools/generator/cmd/track2/release/spec.go | 37 +++ .../generator/cmd/track2/release/workTree.go | 165 +++++++++++++ tools/generator/go.mod | 1 + tools/generator/go.sum | 86 ++++++- 13 files changed, 1099 insertions(+), 2 deletions(-) create mode 100644 tools/generator/cmd/track2/automation/automationCmd.go create mode 100644 tools/generator/cmd/track2/automation/testdata/generate_input.json create mode 100644 tools/generator/cmd/track2/automation/testdata/generate_output.json create mode 100644 tools/generator/cmd/track2/common/cmdProcessor.go create mode 100644 tools/generator/cmd/track2/common/fileProcessor.go create mode 100644 tools/generator/cmd/track2/common/generation.go create mode 100644 tools/generator/cmd/track2/release/releaseCmd.go create mode 100644 tools/generator/cmd/track2/release/sdk.go create mode 100644 tools/generator/cmd/track2/release/spec.go create mode 100644 tools/generator/cmd/track2/release/workTree.go diff --git a/tools/generator/cmd/root.go b/tools/generator/cmd/root.go index 498b3adfb6db..129ee772885a 100644 --- a/tools/generator/cmd/root.go +++ b/tools/generator/cmd/root.go @@ -10,6 +10,8 @@ import ( "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/automation" "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/issue" "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/template" + automation_track2 "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/track2/automation" + release_track2 "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/track2/release" "github.com/spf13/cobra" ) @@ -29,6 +31,8 @@ func Command() *cobra.Command { rootCmd.AddCommand( automation.Command(), + automation_track2.Command(), + release_track2.Command(), issue.Command(), template.Command(), ) diff --git a/tools/generator/cmd/track2/automation/automationCmd.go b/tools/generator/cmd/track2/automation/automationCmd.go new file mode 100644 index 000000000000..9eed3ebbfc36 --- /dev/null +++ b/tools/generator/cmd/track2/automation/automationCmd.go @@ -0,0 +1,168 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +package automation + +import ( + "fmt" + "log" + "os" + "strings" + + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/automation/pipeline" + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/track2/common" + "github.com/Azure/azure-sdk-for-go/tools/internal/utils" + "github.com/spf13/cobra" +) + +// Command returns the automation for track2 command. Note that this command is designed to run in the root directory of +// azure-sdk-for-go. It does not work if you are running this tool in somewhere else +func Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "automation-track2 ", + Args: cobra.ExactArgs(2), + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + log.SetFlags(0) // remove the time stamp prefix + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + if err := execute(args[0], args[1]); err != nil { + logError(err) + return err + } + return nil + }, + SilenceUsage: true, // this command is used for a pipeline, the usage should never show + } + + return cmd +} + +func execute(inputPath, outputPath string) error { + log.Printf("Reading generate input file from '%s'...", inputPath) + input, err := pipeline.ReadInput(inputPath) + if err != nil { + return fmt.Errorf("cannot read generate input: %+v", err) + } + log.Printf("Generating using the following GenerateInput...\n%s", input.String()) + cwd, err := os.Getwd() + if err != nil { + return err + } + log.Printf("Using current directory as SDK root: %s", cwd) + + ctx := automationContext{ + sdkRoot: utils.NormalizePath(cwd), + specRoot: input.SpecFolder, + commitHash: input.HeadSha, + } + output, err := ctx.generate(input) + if err != nil { + return err + } + log.Printf("Output generated: \n%s", output.String()) + log.Printf("Writing output to file '%s'...", outputPath) + if err := pipeline.WriteOutput(outputPath, output); err != nil { + return fmt.Errorf("cannot write generate output: %+v", err) + } + return nil +} + +type automationContext struct { + sdkRoot string + specRoot string + commitHash string +} + +// TODO -- support dry run +func (ctx *automationContext) generate(input *pipeline.GenerateInput) (*pipeline.GenerateOutput, error) { + if input.DryRun { + return nil, fmt.Errorf("dry run not supported yet") + } + + // iterate over all the readme + results := make([]pipeline.PackageResult, 0) + errorBuilder := generateErrorBuilder{} + for _, readme := range input.RelatedReadmeMdFiles { + log.Printf("Start to process readme file: %s", readme) + generateCtx := common.GenerateContext{ + SdkPath: ctx.sdkRoot, + SpecPath: ctx.specRoot, + CommitHash: ctx.commitHash, + } + + namespaceResults, errors := generateCtx.GenerateForAutomation(readme) + if len(errors) != 0 { + errorBuilder.add(errors...) + continue + } + + for _, namespaceResult := range namespaceResults { + content := namespaceResult.ChangelogMd + breaking := namespaceResult.Changelog.HasBreakingChanges() + breakingChangeItems := namespaceResult.Changelog.GetBreakingChangeItems() + + results = append(results, pipeline.PackageResult{ + Version: namespaceResult.Version, + PackageName: namespaceResult.PackageName, + Path: []string{fmt.Sprintf("/sdk/%s/%s", namespaceResult.RpName, namespaceResult.PackageName)}, + ReadmeMd: []string{readme}, + Changelog: &pipeline.Changelog{ + Content: &content, + HasBreakingChange: &breaking, + BreakingChangeItems: &breakingChangeItems, + }, + }) + } + log.Printf("Finish to process readme file: %s", readme) + } + + return &pipeline.GenerateOutput{ + Packages: squashResults(results), + }, errorBuilder.build() +} + +// squashResults squashes the package results by appending all of the `path`s in the following items to the first item +// By doing this, the SDK automation pipeline will only create one PR that contains all of the generation results +// instead of creating one PR for each generation result. +// This is to reduce the resource cost on GitHub +func squashResults(packages []pipeline.PackageResult) []pipeline.PackageResult { + if len(packages) == 0 { + return packages + } + for i := 1; i < len(packages); i++ { + // append the path of the i-th item to the first + packages[0].Path = append(packages[0].Path, packages[i].Path...) + // erase the path on the i-th item + packages[i].Path = make([]string, 0) + } + + return packages +} + +type generateErrorBuilder struct { + errors []error +} + +func (b *generateErrorBuilder) add(err ...error) { + b.errors = append(b.errors, err...) +} + +func (b *generateErrorBuilder) build() error { + if len(b.errors) == 0 { + return nil + } + var messages []string + for _, err := range b.errors { + messages = append(messages, err.Error()) + } + return fmt.Errorf("total %d error(s): \n%s", len(b.errors), strings.Join(messages, "\n")) +} + +func logError(err error) { + for _, line := range strings.Split(err.Error(), "\n") { + if l := strings.TrimSpace(line); l != "" { + log.Printf("[ERROR] %s", l) + } + } +} diff --git a/tools/generator/cmd/track2/automation/testdata/generate_input.json b/tools/generator/cmd/track2/automation/testdata/generate_input.json new file mode 100644 index 000000000000..2cb4c277ceee --- /dev/null +++ b/tools/generator/cmd/track2/automation/testdata/generate_input.json @@ -0,0 +1,14 @@ +{ + "dryRun": false, + "specFolder": "D:\\Workspace\\openapi-env-test\\azure-rest-api-specs", + "headSha": "7cc0a47e7808f3247c71cda25b7c1da8503272ba", + "headRef": "refs/heads/main", + "repoHttpsUrl": "https://github.com/Azure/azure-rest-api-specs.git", + "trigger": "pull_request", + "changedFiles": [ + "specification/agrifood/resource-manager/Microsoft.AgFoodPlatform/preview/2020-05-12-preview/agfood.json" + ], + "relatedReadmeMdFiles": [ + "specification/agrifood/resource-manager/readme.md" + ] + } \ No newline at end of file diff --git a/tools/generator/cmd/track2/automation/testdata/generate_output.json b/tools/generator/cmd/track2/automation/testdata/generate_output.json new file mode 100644 index 000000000000..e4bf7a949097 --- /dev/null +++ b/tools/generator/cmd/track2/automation/testdata/generate_output.json @@ -0,0 +1 @@ +{"packages":[{"version":"0.1.1","packageName":"armagrifood","path":["/sdk/agrifood/armagrifood"],"readmeMd":["specification/agrifood/resource-manager/readme.md"],"changelog":{"content":"No exported changes","hasBreakingChange":false,"breakingChangeItems":[]}}]} \ No newline at end of file diff --git a/tools/generator/cmd/track2/common/cmdProcessor.go b/tools/generator/cmd/track2/common/cmdProcessor.go new file mode 100644 index 000000000000..3332ce460b42 --- /dev/null +++ b/tools/generator/cmd/track2/common/cmdProcessor.go @@ -0,0 +1,16 @@ +package common + +import ( + "fmt" + "os/exec" +) + +func ExecuteGoGenerate(path string) error { + cmd := exec.Command("go", "generate") + cmd.Dir = path + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to execute go generate '%s': %+v", string(output), err) + } + return nil +} diff --git a/tools/generator/cmd/track2/common/fileProcessor.go b/tools/generator/cmd/track2/common/fileProcessor.go new file mode 100644 index 000000000000..f609dc0a2bd6 --- /dev/null +++ b/tools/generator/cmd/track2/common/fileProcessor.go @@ -0,0 +1,222 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +package common + +import ( + "fmt" + "io/ioutil" + "log" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + + "github.com/Azure/azure-sdk-for-go/tools/generator/autorest/model" + "github.com/Azure/azure-sdk-for-go/tools/generator/common" +) + +const ( + sdk_generated_file_prefix = "zz_generated_" + autorest_md_swagger_url_prefix = "- https://github.com/Azure/azure-rest-api-specs/blob/" + autorest_md_module_version_prefix = "module-version: " + swagger_md_module_name_prefix = "module-name: " +) + +var ( + track2BeginRegex = regexp.MustCompile("^```\\s*yaml\\s*\\$\\(go\\)\\s*&&\\s*\\$\\(track2\\)") + track2EndRegex = regexp.MustCompile("^\\s*```\\s*$") +) + +// reads from readme.go.md, parses the `track2` section to get module and package name +func ReadTrack2ModuleNameToGetNamespace(path string) (map[string][]string, error) { + result := make(map[string][]string) + log.Printf("Reading from readme.go.md '%s'...", path) + file, err := os.Open(path) + if err != nil { + return nil, err + } + + log.Printf("Parsing module and package name from readme.go.md ...") + b, err := ioutil.ReadAll(file) + if err != nil { + return nil, err + } + + lines := strings.Split(string(b), "\n") + + var start []int + var end []int + for i, line := range lines { + if track2BeginRegex.MatchString(line) { + start = append(start, i) + } + if len(start) != len(end) && track2EndRegex.MatchString(line) { + end = append(end, i) + } + } + + if len(start) == 0 { + return nil, fmt.Errorf("cannot find any track2 section") + } + if len(start) != len(end) { + return nil, fmt.Errorf("last track2 section does not properly end") + } + + for i := range start { + // get the content of the track2 section + track2Section := lines[start[i]+1 : end[i]] + // iterate over the rest lines, get module name + for _, line := range track2Section { + if strings.HasPrefix(line, swagger_md_module_name_prefix) { + modules := strings.Split(strings.TrimSpace(line[len(swagger_md_module_name_prefix):]), "/") + if len(modules) != 3 { + return nil, fmt.Errorf("cannot parse module name from track2 section") + } + namespaceName := strings.TrimSuffix(strings.TrimSuffix(modules[2], "\n"), "\r") + log.Printf("RP: %s Package: %s", modules[1], namespaceName) + result[modules[1]] = append(result[modules[1]], namespaceName) + } + } + } + + return result, nil +} + +// remove all sdk generated files in given path +func CleanSDKGeneratedFiles(path string) error { + log.Printf("Removing all sdk generated files in '%s'...", path) + files, err := ioutil.ReadDir(path) + if err != nil { + return err + } + + for _, file := range files { + if strings.HasPrefix(file.Name(), sdk_generated_file_prefix) { + err = os.Remove(filepath.Join(path, file.Name())) + if err != nil { + return err + } + } + } + return nil +} + +// replace all commitid in autorest.md files +func ReplaceCommitid(path string, commitid string) error { + log.Printf("Replacing commitid from autorest.md ...") + b, err := ioutil.ReadFile(path) + if err != nil { + return err + } + + lines := strings.Split(string(b), "\n") + for i, line := range lines { + if strings.HasPrefix(line, autorest_md_swagger_url_prefix) { + lines[i] = lines[i][:len(autorest_md_swagger_url_prefix)] + commitid + lines[i][len(autorest_md_swagger_url_prefix)+len(commitid):] + } + } + + err = ioutil.WriteFile(path, []byte(strings.Join(lines, "\n")), 0644) + return err +} + +func GetLatestVersion(packageRootPath string) (string, error) { + b, err := ioutil.ReadFile(filepath.Join(packageRootPath, "autorest.md")) + if err != nil { + return "", err + } + + lines := strings.Split(string(b), "\n") + for _, line := range lines { + if strings.HasPrefix(line, autorest_md_module_version_prefix) { + return strings.TrimSuffix(strings.TrimSuffix(line[len(autorest_md_module_version_prefix):], "\n"), "\r"), nil + } + } + + return "", fmt.Errorf("cannot parse version from autorest.md") +} + +func ReplaceVersion(packageRootPath string, newVersion string) error { + path := filepath.Join(packageRootPath, "autorest.md") + b, err := ioutil.ReadFile(path) + if err != nil { + return err + } + + lines := strings.Split(string(b), "\n") + for i, line := range lines { + if strings.HasPrefix(line, autorest_md_module_version_prefix) { + lines[i] = lines[i][:len(autorest_md_module_version_prefix)] + newVersion + "\n" + break + } + } + + err = ioutil.WriteFile(path, []byte(strings.Join(lines, "\n")), 0644) + return err +} + +func CalculateNewVersion(changelog *model.Changelog, packageRootPath string) (string, error) { + version, err := GetLatestVersion(packageRootPath) + if err != nil { + return "", err + } + + versions := strings.Split(version, ".") + major, err := strconv.Atoi(versions[0]) + if err != nil { + return "", fmt.Errorf("wrong latest version") + } + minor, err := strconv.Atoi(versions[1]) + if err != nil { + return "", fmt.Errorf("wrong latest version") + } + patch, err := strconv.Atoi(versions[2]) + if err != nil { + return "", fmt.Errorf("wrong latest version") + } + log.Printf("Lastest version is: %d.%d.%d", major, minor, patch) + + if major == 0 { + // preview version calculation + if changelog.HasBreakingChanges() { + minor += 1 + patch = 0 + } else { + patch += 1 + } + } else { + // release version calculation + if changelog.HasBreakingChanges() { + major += 1 + minor = 0 + patch = 0 + } else if changelog.Modified.HasAdditiveChanges() { + minor += 1 + patch = 0 + } else { + patch += 1 + } + } + + log.Printf("New version is: %d.%d.%d", major, minor, patch) + return fmt.Sprintf("%d.%d.%d", major, minor, patch), nil +} + +func AddChangelogToFile(changelog *model.Changelog, version string, packageRootPath string) (string, error) { + path := filepath.Join(packageRootPath, common.ChangelogFilename) + b, err := ioutil.ReadFile(path) + if err != nil { + return "", err + } + oldChangelog := string(b) + insertPos := strings.Index(oldChangelog, "##") + additionalChangelog := changelog.ToCompactMarkdown() + newChangelog := oldChangelog[:insertPos] + "## v" + version + " (released)\n" + additionalChangelog + "\n\n" + oldChangelog[insertPos:] + err = ioutil.WriteFile(path, []byte(newChangelog), 0644) + if err != nil { + return "", err + } + return additionalChangelog, nil +} diff --git a/tools/generator/cmd/track2/common/generation.go b/tools/generator/cmd/track2/common/generation.go new file mode 100644 index 000000000000..f00242818576 --- /dev/null +++ b/tools/generator/cmd/track2/common/generation.go @@ -0,0 +1,194 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +package common + +import ( + "fmt" + "log" + "os" + "path/filepath" + "strings" + + "github.com/Azure/azure-sdk-for-go/tools/generator/autorest" + "github.com/Azure/azure-sdk-for-go/tools/generator/autorest/model" + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/template" + "github.com/Azure/azure-sdk-for-go/tools/generator/common" + "github.com/Azure/azure-sdk-for-go/tools/internal/exports" +) + +type GenerateContext struct { + SdkPath string + SpecPath string + CommitHash string +} + +type GenerateResult struct { + Version string + RpName string + PackageName string + PackageAbsPath string + Changelog model.Changelog + ChangelogMd string +} + +func (ctx GenerateContext) SDKRoot() string { + return ctx.SdkPath +} + +func (ctx GenerateContext) SpecRoot() string { + return ctx.SpecPath +} + +func (ctx GenerateContext) GenerateForAutomation(readme string) ([]GenerateResult, []error) { + absReadme := filepath.Join(ctx.SpecPath, readme) + absReadmeGo := filepath.Join(filepath.Dir(absReadme), "readme.go.md") + + var result []GenerateResult + var errors []error + + log.Printf("Get all namespaces from readme file") + rpMap, err := ReadTrack2ModuleNameToGetNamespace(absReadmeGo) + if err != nil { + return nil, []error{ + fmt.Errorf("cannot get rp and namespaces from readme '%s': %+v", readme, err), + } + } + + for rpName, namespaceNames := range rpMap { + for _, namespaceName := range namespaceNames { + log.Printf("Process rp: %s, namespace: %s", rpName, namespaceName) + singleResult, err := ctx.generateForSingleRpNamespace(rpName, namespaceName) + if err != nil { + errors = append(errors, err) + continue + } + result = append(result, *singleResult) + } + } + return result, errors +} + +func (ctx GenerateContext) GenerateForRelease(rpName, namespaceName string) (*GenerateResult, error) { + result, err := ctx.generateForSingleRpNamespace(rpName, namespaceName) + if err != nil { + return nil, err + } + if result.Version != "0.1.0" { + // update process need to do go generate once more to update version in code + packagePath := filepath.Join(ctx.SdkPath, "sdk", rpName, namespaceName) + + log.Printf("Remove all the files that start with `zz_generated_`...") + if err = CleanSDKGeneratedFiles(packagePath); err != nil { + return nil, err + } + + log.Printf("Replace version in autorest.md...") + if err = ReplaceVersion(packagePath, result.Version); err != nil { + return nil, err + } + + log.Printf("Run `go generate` to regenerate the code...") + if err = ExecuteGoGenerate(packagePath); err != nil { + return nil, err + } + } + return result, nil +} + +func (ctx GenerateContext) generateForSingleRpNamespace(rpName, namespaceName string) (*GenerateResult, error) { + packagePath := filepath.Join(ctx.SdkPath, "sdk", rpName, namespaceName) + changelogPath := filepath.Join(packagePath, common.ChangelogFilename) + if _, err := os.Stat(changelogPath); os.IsNotExist(err) { + log.Printf("Package '%s' not onboard, do onboard process", packagePath) + + log.Printf("Use template to generate new rp folder and basic package files...") + if err = template.GeneratePackageByTemplate(rpName, namespaceName, template.Flags{ + SDKRoot: ctx.SdkPath, + TemplatePath: "tools/generator/template/rpName/packageName", + PackageTitle: strings.Title(rpName), + Commit: ctx.CommitHash, + }); err != nil { + return nil, err + } + + log.Printf("Run `go generate` to regenerate the code...") + if err = ExecuteGoGenerate(packagePath); err != nil { + return nil, err + } + + log.Printf("Generate changelog for package...") + newExports, err := exports.Get(packagePath) + if err != nil { + return nil, err + } + changelog, err := autorest.GetChangelogForPackage(nil, &newExports) + if err != nil { + return nil, err + } + + return &GenerateResult{ + Version: "0.1.0", + RpName: rpName, + PackageName: namespaceName, + PackageAbsPath: packagePath, + Changelog: *changelog, + ChangelogMd: changelog.ToCompactMarkdown(), + }, nil + } else { + log.Printf("Package '%s' existed, do update process", packagePath) + + log.Printf("Get ori exports for changelog generation...") + oriExports, err := exports.Get(packagePath) + if err != nil { + return nil, err + } + + log.Printf("Remove all the files that start with `zz_generated_`...") + if err = CleanSDKGeneratedFiles(packagePath); err != nil { + return nil, err + } + + log.Printf("Change the commit hash in `autorest.md` to a new commit that corresponds to the new release...") + autorestMdPath := filepath.Join(packagePath, "autorest.md") + if err = ReplaceCommitid(autorestMdPath, ctx.CommitHash); err != nil { + return nil, err + } + + log.Printf("Run `go generate` to regenerate the code...") + if err = ExecuteGoGenerate(packagePath); err != nil { + return nil, err + } + + log.Printf("Generate changelog for package...") + newExports, err := exports.Get(packagePath) + if err != nil { + return nil, err + } + changelog, err := autorest.GetChangelogForPackage(&oriExports, &newExports) + if err != nil { + return nil, err + } + + log.Printf("Calculate new version...") + version, err := CalculateNewVersion(changelog, packagePath) + if err != nil { + return nil, err + } + + log.Printf("Add changelog to file...") + changelogMd, err := AddChangelogToFile(changelog, version, packagePath) + if err != nil { + return nil, err + } + + return &GenerateResult{ + Version: version, + RpName: rpName, + PackageName: namespaceName, + PackageAbsPath: packagePath, + Changelog: *changelog, + ChangelogMd: changelogMd, + }, nil + } +} diff --git a/tools/generator/cmd/track2/release/releaseCmd.go b/tools/generator/cmd/track2/release/releaseCmd.go new file mode 100644 index 000000000000..b695a402ddb8 --- /dev/null +++ b/tools/generator/cmd/track2/release/releaseCmd.go @@ -0,0 +1,139 @@ +package release + +import ( + "fmt" + "log" + "path/filepath" + "time" + + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/track2/common" + "github.com/Azure/azure-sdk-for-go/tools/generator/flags" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +// Release command +func Command() *cobra.Command { + releaseCmd := &cobra.Command{ + Use: "release-track2 ", + Short: "Generate a track2 release of azure-sdk-for-go", + Long: `This command will generate a new track2 release for azure-sdk-for-go with given rp name and namespace name. + +azure-sdk-for-go directory: the directory path of the azure-sdk-for-go with git control +azure-rest-api-specs directory: the directory path of the azure-rest-api-specs with git control +rp-name: name of resource provider to be released, same for the swagger folder name +namespaceName: name of namespace to be released, default value is arm+rp-name + +`, + Args: cobra.RangeArgs(3, 4), + RunE: func(cmd *cobra.Command, args []string) error { + sdkPath, err := filepath.Abs(args[0]) + if err != nil { + return fmt.Errorf("failed to get the directory of azure-sdk-for-go: %v", err) + } + specPath, err := filepath.Abs(args[1]) + if err != nil { + return fmt.Errorf("failed to get the directory of azure-rest-api-specs: %v", err) + } + rpName := args[2] + namespaceName := "arm" + rpName + if len(args) == 4 { + namespaceName = args[3] + } + + ctx := commandContext{ + rpName: rpName, + namespaceName: namespaceName, + sdkPath: sdkPath, + specPath: specPath, + flags: ParseFlags(cmd.Flags()), + } + return ctx.execute() + }, + } + + BindFlags(releaseCmd.Flags()) + + return releaseCmd +} + +type Flags struct { + VersionNumber string +} + +func BindFlags(flagSet *pflag.FlagSet) { + flagSet.String("version-number", "", "Specify the version number of this release") +} + +func ParseFlags(flagSet *pflag.FlagSet) Flags { + return Flags{ + VersionNumber: flags.GetString(flagSet, "version-number"), + } +} + +type commandContext struct { + rpName string + namespaceName string + sdkPath string + sdkRepo SDKRepository + specPath string + specRepo SpecRepository + flags Flags +} + +func (c *commandContext) execute() error { + var err error + // create sdk and spec git repo ref + c.sdkRepo, err = OpenSDKRepository(c.sdkPath) + if err != nil { + return fmt.Errorf("failed to get sdk repo: %+v", err) + } + c.specRepo, err = OpenSpecRepository(c.specPath) + if err != nil { + return fmt.Errorf("failed to get spec repo: %+v", err) + } + + // get sdk and spec repo head + sdkRef, err := c.sdkRepo.Head() + if err != nil { + return fmt.Errorf("failed to get HEAD ref of azure-sdk-for-go: %+v", err) + } + log.Printf("The release branch is based on HEAD ref '%s' (commit %s) of azure-sdk-for-go", sdkRef.Name(), sdkRef.Hash()) + + specRef, err := c.specRepo.Head() + if err != nil { + return fmt.Errorf("failed to get HEAD ref of azure-rest-api-specs: %+v", err) + } + log.Printf("The new version is generated from HEAD ref '%s' (commit %s) of azure-rest-api-specs", specRef.Name(), specRef.Hash()) + + log.Printf("Release generation for rp: %s, namespace: %s", c.rpName, c.namespaceName) + generateCtx := common.GenerateContext{ + SdkPath: c.sdkPath, + SpecPath: c.specPath, + CommitHash: specRef.Hash().String(), + } + result, err := generateCtx.GenerateForRelease(c.rpName, c.namespaceName) + if err != nil { + return fmt.Errorf("failed to finish release generation process: %+v", err) + } + // print generation result + log.Printf("Generation result: %s", result) + + log.Printf("Create new branch for release") + releaseBranchName := fmt.Sprintf(releaseBranchNamePattern, c.rpName, c.namespaceName, result.Version, time.Now().Unix()) + if err := c.sdkRepo.CreateReleaseBranch(releaseBranchName); err != nil { + return fmt.Errorf("failed to create release branch: %+v", err) + } + + log.Printf("Include the packages that is about to release in this release and do release commit...") + // append a time in long to avoid collision of branch names + if err := c.sdkRepo.AddReleaseCommit(c.rpName, c.namespaceName, generateCtx.CommitHash, result.Version); err != nil { + return fmt.Errorf("failed to add release package or do release commit: %+v", err) + } + + return nil +} + +const ( + releaseBranchNamePattern = "release-%s-%s-%s-%v" +) diff --git a/tools/generator/cmd/track2/release/sdk.go b/tools/generator/cmd/track2/release/sdk.go new file mode 100644 index 000000000000..c8aac406e0c3 --- /dev/null +++ b/tools/generator/cmd/track2/release/sdk.go @@ -0,0 +1,54 @@ +package release + +import ( + "fmt" + "log" + + "github.com/go-git/go-git/v5/plumbing" +) + +type SDKRepository interface { + WorkTree + CreateReleaseBranch(releaseBranchName string) error + AddReleaseCommit(rpName, namespaceName, specHash, version string) error +} + +func OpenSDKRepository(path string) (SDKRepository, error) { + wt, err := NewWorkTree(path) + if err != nil { + return nil, err + } + + return &sdkRepository{ + WorkTree: wt, + }, nil +} + +type sdkRepository struct { + WorkTree +} + +func (s *sdkRepository) AddReleaseCommit(rpName, namespaceName, specHash, version string) error { + log.Printf("Add release package and commit") + if err := s.Add(fmt.Sprintf("sdk\\%s\\%s", rpName, namespaceName)); err != nil { + return fmt.Errorf("failed to add 'profiles': %+v", err) + } + + message := fmt.Sprintf("Release: %s:%s from spec commit hash: %s", rpName, version, specHash) + if err := s.Commit(message); err != nil { + if IsNothingToCommit(err) { + log.Printf("There is nothing to commit. Message: %s", message) + return nil + } + return fmt.Errorf("failed to commit changes: %+v", err) + } + + return nil +} +func (s *sdkRepository) CreateReleaseBranch(releaseBranchName string) error { + log.Printf("Checking out to %s", plumbing.NewBranchReferenceName(releaseBranchName)) + return s.Checkout(&CheckoutOptions{ + Branch: plumbing.NewBranchReferenceName(releaseBranchName), + Create: true, + }) +} diff --git a/tools/generator/cmd/track2/release/spec.go b/tools/generator/cmd/track2/release/spec.go new file mode 100644 index 000000000000..6e017f148788 --- /dev/null +++ b/tools/generator/cmd/track2/release/spec.go @@ -0,0 +1,37 @@ +package release + +import ( + "github.com/go-git/go-git/v5/plumbing" +) + +type SpecRepository interface { + WorkTree + LastHead() *plumbing.Reference +} + +func OpenSpecRepository(path string) (SpecRepository, error) { + spec, err := NewWorkTree(path) + if err != nil { + return nil, err + } + + lastRef, err := spec.Head() + if err != nil { + return nil, err + } + + return &specRepository{ + WorkTree: spec, + lastRef: lastRef, + }, nil +} + +type specRepository struct { + WorkTree + + lastRef *plumbing.Reference +} + +func (s *specRepository) LastHead() *plumbing.Reference { + return s.lastRef +} diff --git a/tools/generator/cmd/track2/release/workTree.go b/tools/generator/cmd/track2/release/workTree.go new file mode 100644 index 000000000000..2718b53da40b --- /dev/null +++ b/tools/generator/cmd/track2/release/workTree.go @@ -0,0 +1,165 @@ +package release + +import ( + "fmt" + "os/exec" + "strings" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/storer" +) + +type WorkTree interface { + Root() string + Add(path string) error + Commit(message string) error + Checkout(opt *CheckoutOptions) error + CreateBranch(branch *Branch) error + DeleteBranch(name string) error + CherryPick(commit string) error + Head() (*plumbing.Reference, error) + Tags() (storer.ReferenceIter, error) +} + +type CheckoutOptions git.CheckoutOptions +type Branch config.Branch + +type repository struct { + *git.Repository + + wt *git.Worktree + root string +} + +func NewWorkTree(path string) (WorkTree, error) { + r, err := git.PlainOpenWithOptions(path, &git.PlainOpenOptions{ + DetectDotGit: true, + }) + if err != nil { + return nil, fmt.Errorf("cannot open '%s': %+v", path, err) + } + wt, err := r.Worktree() + if err != nil { + return nil, fmt.Errorf("cannot get the work tree of '%s': %+v", path, err) + } + return &repository{ + Repository: r, + wt: wt, + root: wt.Filesystem.Root(), + }, nil +} + +func (r *repository) Root() string { + return r.root +} + +// TODO -- go-git has some performance issue during Add, therefore we use the git command as a workaround +func (r *repository) Add(path string) error { + cmd := exec.Command("git", "add", path) + cmd.Dir = r.root + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf(string(output)) + } + return nil +} + +// TODO -- go-git has some performance and permission issue during Commit, therefore we use the git command as a workaround +func (r *repository) Commit(message string) error { + cmd := exec.Command("git", "commit", "-m", message) + cmd.Dir = r.root + output, err := cmd.CombinedOutput() + if err != nil { + m := string(output) + if strings.Contains(m, "nothing added to commit") { + return &NothingToCommit{ + message: m, + } + } + return fmt.Errorf(m) + } + return nil +} + +// TODO -- go-git has some issue on the Checkout command, it will keep the CRLF changes after switching branches in stage +func (r *repository) Checkout(opt *CheckoutOptions) error { + if len(opt.Branch) > 0 { + return r.checkoutBranch(opt.Branch.Short(), opt.Create) + } + if !opt.Hash.IsZero() { + return r.checkoutHash(opt.Hash.String()) + } + return fmt.Errorf("must set one of hash or branch") +} + +func (r *repository) checkoutBranch(branch string, create bool) error { + var cmd *exec.Cmd + if create { + cmd = exec.Command("git", "checkout", "-b", branch) + } else { + cmd = exec.Command("git", "checkout", branch) + } + cmd.Dir = r.root + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf(string(output)) + } + return nil +} + +func (r *repository) checkoutHash(hash string) error { + cmd := exec.Command("git", "checkout", hash) + cmd.Dir = r.root + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf(string(output)) + } + return nil +} + +func (r *repository) CreateBranch(branch *Branch) error { + cmd := exec.Command("git", "branch", branch.Name) + cmd.Dir = r.root + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf(string(output)) + } + return nil +} + +// TODO -- we cannot delete the branch that is not created using go-git, therefore we use the git command as a workaround +func (r *repository) DeleteBranch(name string) error { + cmd := exec.Command("git", "branch", "-D", name) + cmd.Dir = r.root + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf(string(output)) + } + return nil +} + +// TODO -- go-git now does not support cherry-pick (or I did not find this?), therefore we use the git command as a workaround +func (r *repository) CherryPick(commit string) error { + cmd := exec.Command("git", "cherry-pick", commit) + cmd.Dir = r.root + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf(string(output)) + } + return nil +} + +type NothingToCommit struct { + message string +} + +func (n *NothingToCommit) Error() string { + return n.message +} + +func IsNothingToCommit(err error) bool { + _, ok := err.(*NothingToCommit) + return ok +} diff --git a/tools/generator/go.mod b/tools/generator/go.mod index a054e686a516..0f5964741cab 100644 --- a/tools/generator/go.mod +++ b/tools/generator/go.mod @@ -6,6 +6,7 @@ require ( github.com/Azure/azure-sdk-for-go v54.2.1+incompatible github.com/Azure/azure-sdk-for-go/tools/internal v0.1.0 github.com/ahmetb/go-linq/v3 v3.2.0 + github.com/go-git/go-git/v5 v5.4.2 github.com/google/go-github/v32 v32.1.0 github.com/hashicorp/go-multierror v1.0.0 github.com/spf13/cobra v1.1.3 diff --git a/tools/generator/go.sum b/tools/generator/go.sum index 68d3611ac154..3a171ec3dcc4 100644 --- a/tools/generator/go.sum +++ b/tools/generator/go.sum @@ -25,14 +25,25 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= +github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk= +github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 h1:YoJbenK9C67SkzkDfmQuVln04ygHj3vjZfd9FL+GmQQ= +github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= +github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= +github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/ahmetb/go-linq/v3 v3.2.0 h1:BEuMfp+b59io8g5wYzNoFe9pWPalRklhlhbiU3hYZDE= github.com/ahmetb/go-linq/v3 v3.2.0/go.mod h1:haQ3JfOeWK8HpVxMtHHEMPVgBKiYyQ+f1/kLZh/cj9U= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= @@ -45,14 +56,31 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= +github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= +github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= +github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= +github.com/go-git/go-billy/v5 v5.2.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-billy/v5 v5.3.1 h1:CPiOUAzKtMRvolEKw+bG1PLRpT7D3LIs3/3ey4Aiu34= +github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-git-fixtures/v4 v4.2.1 h1:n9gGL1Ct/yIw+nfsfr8s4+sbhT+Ncu2SubfXjIWgci8= +github.com/go-git/go-git-fixtures/v4 v4.2.1/go.mod h1:K8zd3kDUAykwTdDCr+I0per6Y6vMiRR/nnVTBtavnB0= +github.com/go-git/go-git/v5 v5.4.2 h1:BXyZu9t0VkbiHtqrsvdq39UDhGJTl1h55VW6CSC4aY4= +github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti4ihgckDc= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -72,6 +100,7 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II= github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= @@ -110,27 +139,41 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck= +github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= +github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= @@ -140,11 +183,15 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -161,8 +208,11 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= @@ -180,8 +230,13 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI= +github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -191,11 +246,14 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b h1:7mWr3k41Qtv8XlltBkDkl8LoP3mpSgBW8BUoxtEdbXg= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -227,8 +285,10 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210326060303-6b1517762897 h1:KrsHThm5nFk34YtATK1LsThyGhGbGe1olrte/HInHvs= +golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= @@ -251,9 +311,20 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79 h1:RX8C8PRZc2hTIod4ds8ij+/4RQX3AqhYj3uOHmyaz4E= +golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -299,13 +370,24 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From db0bd9c6b21d27cda9a495c2218d7f65f30ea9c2 Mon Sep 17 00:00:00 2001 From: tadelesh Date: Thu, 12 Aug 2021 14:53:43 +0800 Subject: [PATCH 02/21] add copyright and do gofmt --- tools/generator/cmd/track2/common/cmdProcessor.go | 3 +++ tools/generator/cmd/track2/release/releaseCmd.go | 5 ++++- tools/generator/cmd/track2/release/sdk.go | 3 +++ tools/generator/cmd/track2/release/spec.go | 3 +++ tools/generator/cmd/track2/release/workTree.go | 3 +++ 5 files changed, 16 insertions(+), 1 deletion(-) diff --git a/tools/generator/cmd/track2/common/cmdProcessor.go b/tools/generator/cmd/track2/common/cmdProcessor.go index 3332ce460b42..a6b329c87310 100644 --- a/tools/generator/cmd/track2/common/cmdProcessor.go +++ b/tools/generator/cmd/track2/common/cmdProcessor.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package common import ( diff --git a/tools/generator/cmd/track2/release/releaseCmd.go b/tools/generator/cmd/track2/release/releaseCmd.go index b695a402ddb8..a484fca7cca3 100644 --- a/tools/generator/cmd/track2/release/releaseCmd.go +++ b/tools/generator/cmd/track2/release/releaseCmd.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package release import ( @@ -58,7 +61,7 @@ namespaceName: name of namespace to be released, default value is arm+rp-name } type Flags struct { - VersionNumber string + VersionNumber string } func BindFlags(flagSet *pflag.FlagSet) { diff --git a/tools/generator/cmd/track2/release/sdk.go b/tools/generator/cmd/track2/release/sdk.go index c8aac406e0c3..038ff8af32b4 100644 --- a/tools/generator/cmd/track2/release/sdk.go +++ b/tools/generator/cmd/track2/release/sdk.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package release import ( diff --git a/tools/generator/cmd/track2/release/spec.go b/tools/generator/cmd/track2/release/spec.go index 6e017f148788..cfad29902eb9 100644 --- a/tools/generator/cmd/track2/release/spec.go +++ b/tools/generator/cmd/track2/release/spec.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package release import ( diff --git a/tools/generator/cmd/track2/release/workTree.go b/tools/generator/cmd/track2/release/workTree.go index 2718b53da40b..273db0cb6de7 100644 --- a/tools/generator/cmd/track2/release/workTree.go +++ b/tools/generator/cmd/track2/release/workTree.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package release import ( From 245a69ff02b31412bcc625dd243e8ef9d18d9fd6 Mon Sep 17 00:00:00 2001 From: tadelesh Date: Thu, 12 Aug 2021 23:04:58 +0800 Subject: [PATCH 03/21] Use semver to pump version, move git operation to repo package and fix some typo --- .../cmd/track2/common/fileProcessor.go | 66 +++++++------------ .../generator/cmd/track2/common/generation.go | 4 +- .../cmd/track2/release/releaseCmd.go | 9 +-- tools/generator/go.mod | 1 + tools/generator/go.sum | 1 + .../{cmd/track2/release => repo}/sdk.go | 2 +- .../{cmd/track2/release => repo}/spec.go | 2 +- .../{cmd/track2/release => repo}/workTree.go | 2 +- 8 files changed, 37 insertions(+), 50 deletions(-) rename tools/generator/{cmd/track2/release => repo}/sdk.go (98%) rename tools/generator/{cmd/track2/release => repo}/spec.go (97%) rename tools/generator/{cmd/track2/release => repo}/workTree.go (99%) diff --git a/tools/generator/cmd/track2/common/fileProcessor.go b/tools/generator/cmd/track2/common/fileProcessor.go index f609dc0a2bd6..6a791c8552f8 100644 --- a/tools/generator/cmd/track2/common/fileProcessor.go +++ b/tools/generator/cmd/track2/common/fileProcessor.go @@ -10,11 +10,11 @@ import ( "os" "path/filepath" "regexp" - "strconv" "strings" "github.com/Azure/azure-sdk-for-go/tools/generator/autorest/model" "github.com/Azure/azure-sdk-for-go/tools/generator/common" + "github.com/Masterminds/semver" ) const ( @@ -103,9 +103,9 @@ func CleanSDKGeneratedFiles(path string) error { return nil } -// replace all commitid in autorest.md files -func ReplaceCommitid(path string, commitid string) error { - log.Printf("Replacing commitid from autorest.md ...") +// replace all commit id in autorest.md files +func ReplaceCommitID(path string, commitID string) error { + log.Printf("Replacing commit id from autorest.md ...") b, err := ioutil.ReadFile(path) if err != nil { return err @@ -114,7 +114,7 @@ func ReplaceCommitid(path string, commitid string) error { lines := strings.Split(string(b), "\n") for i, line := range lines { if strings.HasPrefix(line, autorest_md_swagger_url_prefix) { - lines[i] = lines[i][:len(autorest_md_swagger_url_prefix)] + commitid + lines[i][len(autorest_md_swagger_url_prefix)+len(commitid):] + lines[i] = line[:len(autorest_md_swagger_url_prefix)] + commitID + line[len(autorest_md_swagger_url_prefix)+len(commitID):] } } @@ -122,20 +122,21 @@ func ReplaceCommitid(path string, commitid string) error { return err } -func GetLatestVersion(packageRootPath string) (string, error) { +func GetLatestVersion(packageRootPath string) (*semver.Version, error) { b, err := ioutil.ReadFile(filepath.Join(packageRootPath, "autorest.md")) if err != nil { - return "", err + return nil, err } lines := strings.Split(string(b), "\n") for _, line := range lines { if strings.HasPrefix(line, autorest_md_module_version_prefix) { - return strings.TrimSuffix(strings.TrimSuffix(line[len(autorest_md_module_version_prefix):], "\n"), "\r"), nil + versionString := strings.TrimSuffix(strings.TrimSuffix(line[len(autorest_md_module_version_prefix):], "\n"), "\r") + return semver.NewVersion(versionString) } } - return "", fmt.Errorf("cannot parse version from autorest.md") + return nil, fmt.Errorf("cannot parse version from autorest.md") } func ReplaceVersion(packageRootPath string, newVersion string) error { @@ -148,7 +149,7 @@ func ReplaceVersion(packageRootPath string, newVersion string) error { lines := strings.Split(string(b), "\n") for i, line := range lines { if strings.HasPrefix(line, autorest_md_module_version_prefix) { - lines[i] = lines[i][:len(autorest_md_module_version_prefix)] + newVersion + "\n" + lines[i] = line[:len(autorest_md_module_version_prefix)] + newVersion + "\n" break } } @@ -157,54 +158,37 @@ func ReplaceVersion(packageRootPath string, newVersion string) error { return err } -func CalculateNewVersion(changelog *model.Changelog, packageRootPath string) (string, error) { +func CalculateNewVersion(changelog *model.Changelog, packageRootPath string) (*semver.Version, error) { version, err := GetLatestVersion(packageRootPath) if err != nil { - return "", err - } - - versions := strings.Split(version, ".") - major, err := strconv.Atoi(versions[0]) - if err != nil { - return "", fmt.Errorf("wrong latest version") - } - minor, err := strconv.Atoi(versions[1]) - if err != nil { - return "", fmt.Errorf("wrong latest version") - } - patch, err := strconv.Atoi(versions[2]) - if err != nil { - return "", fmt.Errorf("wrong latest version") + return nil, err } - log.Printf("Lastest version is: %d.%d.%d", major, minor, patch) + log.Printf("Lastest version is: %s", version.String()) - if major == 0 { + var newVersion semver.Version + if version.Major() == 0 { // preview version calculation if changelog.HasBreakingChanges() { - minor += 1 - patch = 0 + newVersion = version.IncMinor() } else { - patch += 1 + newVersion = version.IncPatch() } } else { // release version calculation if changelog.HasBreakingChanges() { - major += 1 - minor = 0 - patch = 0 + newVersion = version.IncMajor() } else if changelog.Modified.HasAdditiveChanges() { - minor += 1 - patch = 0 + newVersion = version.IncMinor() } else { - patch += 1 + newVersion = version.IncPatch() } } - log.Printf("New version is: %d.%d.%d", major, minor, patch) - return fmt.Sprintf("%d.%d.%d", major, minor, patch), nil + log.Printf("New version is: %s", newVersion.String()) + return &newVersion, nil } -func AddChangelogToFile(changelog *model.Changelog, version string, packageRootPath string) (string, error) { +func AddChangelogToFile(changelog *model.Changelog, version *semver.Version, packageRootPath string) (string, error) { path := filepath.Join(packageRootPath, common.ChangelogFilename) b, err := ioutil.ReadFile(path) if err != nil { @@ -213,7 +197,7 @@ func AddChangelogToFile(changelog *model.Changelog, version string, packageRootP oldChangelog := string(b) insertPos := strings.Index(oldChangelog, "##") additionalChangelog := changelog.ToCompactMarkdown() - newChangelog := oldChangelog[:insertPos] + "## v" + version + " (released)\n" + additionalChangelog + "\n\n" + oldChangelog[insertPos:] + newChangelog := oldChangelog[:insertPos] + "## v" + version.String() + " (released)\n" + additionalChangelog + "\n\n" + oldChangelog[insertPos:] err = ioutil.WriteFile(path, []byte(newChangelog), 0644) if err != nil { return "", err diff --git a/tools/generator/cmd/track2/common/generation.go b/tools/generator/cmd/track2/common/generation.go index f00242818576..9c4c42277812 100644 --- a/tools/generator/cmd/track2/common/generation.go +++ b/tools/generator/cmd/track2/common/generation.go @@ -151,7 +151,7 @@ func (ctx GenerateContext) generateForSingleRpNamespace(rpName, namespaceName st log.Printf("Change the commit hash in `autorest.md` to a new commit that corresponds to the new release...") autorestMdPath := filepath.Join(packagePath, "autorest.md") - if err = ReplaceCommitid(autorestMdPath, ctx.CommitHash); err != nil { + if err = ReplaceCommitID(autorestMdPath, ctx.CommitHash); err != nil { return nil, err } @@ -183,7 +183,7 @@ func (ctx GenerateContext) generateForSingleRpNamespace(rpName, namespaceName st } return &GenerateResult{ - Version: version, + Version: version.String(), RpName: rpName, PackageName: namespaceName, PackageAbsPath: packagePath, diff --git a/tools/generator/cmd/track2/release/releaseCmd.go b/tools/generator/cmd/track2/release/releaseCmd.go index a484fca7cca3..4383fffb19d5 100644 --- a/tools/generator/cmd/track2/release/releaseCmd.go +++ b/tools/generator/cmd/track2/release/releaseCmd.go @@ -11,6 +11,7 @@ import ( "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/track2/common" "github.com/Azure/azure-sdk-for-go/tools/generator/flags" + "github.com/Azure/azure-sdk-for-go/tools/generator/repo" "github.com/spf13/cobra" "github.com/spf13/pflag" ) @@ -78,20 +79,20 @@ type commandContext struct { rpName string namespaceName string sdkPath string - sdkRepo SDKRepository + sdkRepo repo.SDKRepository specPath string - specRepo SpecRepository + specRepo repo.SpecRepository flags Flags } func (c *commandContext) execute() error { var err error // create sdk and spec git repo ref - c.sdkRepo, err = OpenSDKRepository(c.sdkPath) + c.sdkRepo, err = repo.OpenSDKRepository(c.sdkPath) if err != nil { return fmt.Errorf("failed to get sdk repo: %+v", err) } - c.specRepo, err = OpenSpecRepository(c.specPath) + c.specRepo, err = repo.OpenSpecRepository(c.specPath) if err != nil { return fmt.Errorf("failed to get spec repo: %+v", err) } diff --git a/tools/generator/go.mod b/tools/generator/go.mod index 0f5964741cab..c4cf212bae2c 100644 --- a/tools/generator/go.mod +++ b/tools/generator/go.mod @@ -5,6 +5,7 @@ go 1.13 require ( github.com/Azure/azure-sdk-for-go v54.2.1+incompatible github.com/Azure/azure-sdk-for-go/tools/internal v0.1.0 + github.com/Masterminds/semver v1.5.0 github.com/ahmetb/go-linq/v3 v3.2.0 github.com/go-git/go-git/v5 v5.4.2 github.com/google/go-github/v32 v32.1.0 diff --git a/tools/generator/go.sum b/tools/generator/go.sum index 3a171ec3dcc4..37eda0584d94 100644 --- a/tools/generator/go.sum +++ b/tools/generator/go.sum @@ -24,6 +24,7 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk= diff --git a/tools/generator/cmd/track2/release/sdk.go b/tools/generator/repo/sdk.go similarity index 98% rename from tools/generator/cmd/track2/release/sdk.go rename to tools/generator/repo/sdk.go index 038ff8af32b4..0ad44e1c533b 100644 --- a/tools/generator/cmd/track2/release/sdk.go +++ b/tools/generator/repo/sdk.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -package release +package repo import ( "fmt" diff --git a/tools/generator/cmd/track2/release/spec.go b/tools/generator/repo/spec.go similarity index 97% rename from tools/generator/cmd/track2/release/spec.go rename to tools/generator/repo/spec.go index cfad29902eb9..e821aecd46f6 100644 --- a/tools/generator/cmd/track2/release/spec.go +++ b/tools/generator/repo/spec.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -package release +package repo import ( "github.com/go-git/go-git/v5/plumbing" diff --git a/tools/generator/cmd/track2/release/workTree.go b/tools/generator/repo/workTree.go similarity index 99% rename from tools/generator/cmd/track2/release/workTree.go rename to tools/generator/repo/workTree.go index 273db0cb6de7..e8d264ddc77f 100644 --- a/tools/generator/cmd/track2/release/workTree.go +++ b/tools/generator/repo/workTree.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -package release +package repo import ( "fmt" From c82b9eb0e2259bf937e899f3a2152cfd2200ca48 Mon Sep 17 00:00:00 2001 From: tadelesh Date: Fri, 13 Aug 2021 11:32:27 +0800 Subject: [PATCH 04/21] Add specfic version to release cmd, seperate package for automation and remove test config file --- .../cmd/track2/automation/automationCmd.go | 20 +----- .../automation/testdata/generate_input.json | 14 ----- .../automation/testdata/generate_output.json | 1 - .../generator/cmd/track2/common/generation.go | 62 +++++++++---------- .../cmd/track2/release/releaseCmd.go | 3 +- 5 files changed, 33 insertions(+), 67 deletions(-) delete mode 100644 tools/generator/cmd/track2/automation/testdata/generate_input.json delete mode 100644 tools/generator/cmd/track2/automation/testdata/generate_output.json diff --git a/tools/generator/cmd/track2/automation/automationCmd.go b/tools/generator/cmd/track2/automation/automationCmd.go index 9eed3ebbfc36..68976e505c60 100644 --- a/tools/generator/cmd/track2/automation/automationCmd.go +++ b/tools/generator/cmd/track2/automation/automationCmd.go @@ -118,28 +118,10 @@ func (ctx *automationContext) generate(input *pipeline.GenerateInput) (*pipeline } return &pipeline.GenerateOutput{ - Packages: squashResults(results), + Packages: results, }, errorBuilder.build() } -// squashResults squashes the package results by appending all of the `path`s in the following items to the first item -// By doing this, the SDK automation pipeline will only create one PR that contains all of the generation results -// instead of creating one PR for each generation result. -// This is to reduce the resource cost on GitHub -func squashResults(packages []pipeline.PackageResult) []pipeline.PackageResult { - if len(packages) == 0 { - return packages - } - for i := 1; i < len(packages); i++ { - // append the path of the i-th item to the first - packages[0].Path = append(packages[0].Path, packages[i].Path...) - // erase the path on the i-th item - packages[i].Path = make([]string, 0) - } - - return packages -} - type generateErrorBuilder struct { errors []error } diff --git a/tools/generator/cmd/track2/automation/testdata/generate_input.json b/tools/generator/cmd/track2/automation/testdata/generate_input.json deleted file mode 100644 index 2cb4c277ceee..000000000000 --- a/tools/generator/cmd/track2/automation/testdata/generate_input.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "dryRun": false, - "specFolder": "D:\\Workspace\\openapi-env-test\\azure-rest-api-specs", - "headSha": "7cc0a47e7808f3247c71cda25b7c1da8503272ba", - "headRef": "refs/heads/main", - "repoHttpsUrl": "https://github.com/Azure/azure-rest-api-specs.git", - "trigger": "pull_request", - "changedFiles": [ - "specification/agrifood/resource-manager/Microsoft.AgFoodPlatform/preview/2020-05-12-preview/agfood.json" - ], - "relatedReadmeMdFiles": [ - "specification/agrifood/resource-manager/readme.md" - ] - } \ No newline at end of file diff --git a/tools/generator/cmd/track2/automation/testdata/generate_output.json b/tools/generator/cmd/track2/automation/testdata/generate_output.json deleted file mode 100644 index e4bf7a949097..000000000000 --- a/tools/generator/cmd/track2/automation/testdata/generate_output.json +++ /dev/null @@ -1 +0,0 @@ -{"packages":[{"version":"0.1.1","packageName":"armagrifood","path":["/sdk/agrifood/armagrifood"],"readmeMd":["specification/agrifood/resource-manager/readme.md"],"changelog":{"content":"No exported changes","hasBreakingChange":false,"breakingChangeItems":[]}}]} \ No newline at end of file diff --git a/tools/generator/cmd/track2/common/generation.go b/tools/generator/cmd/track2/common/generation.go index 9c4c42277812..ef1fa967287d 100644 --- a/tools/generator/cmd/track2/common/generation.go +++ b/tools/generator/cmd/track2/common/generation.go @@ -15,6 +15,7 @@ import ( "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/template" "github.com/Azure/azure-sdk-for-go/tools/generator/common" "github.com/Azure/azure-sdk-for-go/tools/internal/exports" + "github.com/Masterminds/semver" ) type GenerateContext struct { @@ -58,7 +59,7 @@ func (ctx GenerateContext) GenerateForAutomation(readme string) ([]GenerateResul for rpName, namespaceNames := range rpMap { for _, namespaceName := range namespaceNames { log.Printf("Process rp: %s, namespace: %s", rpName, namespaceName) - singleResult, err := ctx.generateForSingleRpNamespace(rpName, namespaceName) + singleResult, err := ctx.GenerateForSingleRpNamespace(rpName, namespaceName, "") if err != nil { errors = append(errors, err) continue @@ -69,34 +70,7 @@ func (ctx GenerateContext) GenerateForAutomation(readme string) ([]GenerateResul return result, errors } -func (ctx GenerateContext) GenerateForRelease(rpName, namespaceName string) (*GenerateResult, error) { - result, err := ctx.generateForSingleRpNamespace(rpName, namespaceName) - if err != nil { - return nil, err - } - if result.Version != "0.1.0" { - // update process need to do go generate once more to update version in code - packagePath := filepath.Join(ctx.SdkPath, "sdk", rpName, namespaceName) - - log.Printf("Remove all the files that start with `zz_generated_`...") - if err = CleanSDKGeneratedFiles(packagePath); err != nil { - return nil, err - } - - log.Printf("Replace version in autorest.md...") - if err = ReplaceVersion(packagePath, result.Version); err != nil { - return nil, err - } - - log.Printf("Run `go generate` to regenerate the code...") - if err = ExecuteGoGenerate(packagePath); err != nil { - return nil, err - } - } - return result, nil -} - -func (ctx GenerateContext) generateForSingleRpNamespace(rpName, namespaceName string) (*GenerateResult, error) { +func (ctx GenerateContext) GenerateForSingleRpNamespace(rpName, namespaceName string, specficVersion string) (*GenerateResult, error) { packagePath := filepath.Join(ctx.SdkPath, "sdk", rpName, namespaceName) changelogPath := filepath.Join(packagePath, common.ChangelogFilename) if _, err := os.Stat(changelogPath); os.IsNotExist(err) { @@ -171,9 +145,18 @@ func (ctx GenerateContext) generateForSingleRpNamespace(rpName, namespaceName st } log.Printf("Calculate new version...") - version, err := CalculateNewVersion(changelog, packagePath) - if err != nil { - return nil, err + var version *semver.Version + if len(specficVersion) == 0 { + version, err = CalculateNewVersion(changelog, packagePath) + if err != nil { + return nil, err + } + } else { + log.Printf("Use specfic version: %s", specficVersion) + version, err = semver.NewVersion(specficVersion) + if err != nil { + return nil, err + } } log.Printf("Add changelog to file...") @@ -182,6 +165,21 @@ func (ctx GenerateContext) generateForSingleRpNamespace(rpName, namespaceName st return nil, err } + log.Printf("Remove all the files that start with `zz_generated_`...") + if err = CleanSDKGeneratedFiles(packagePath); err != nil { + return nil, err + } + + log.Printf("Replace version in autorest.md...") + if err = ReplaceVersion(packagePath, version.String()); err != nil { + return nil, err + } + + log.Printf("Run `go generate` to regenerate the code for new version...") + if err = ExecuteGoGenerate(packagePath); err != nil { + return nil, err + } + return &GenerateResult{ Version: version.String(), RpName: rpName, diff --git a/tools/generator/cmd/track2/release/releaseCmd.go b/tools/generator/cmd/track2/release/releaseCmd.go index 4383fffb19d5..57f8f9d843f8 100644 --- a/tools/generator/cmd/track2/release/releaseCmd.go +++ b/tools/generator/cmd/track2/release/releaseCmd.go @@ -116,7 +116,8 @@ func (c *commandContext) execute() error { SpecPath: c.specPath, CommitHash: specRef.Hash().String(), } - result, err := generateCtx.GenerateForRelease(c.rpName, c.namespaceName) + + result, err := generateCtx.GenerateForSingleRpNamespace(c.rpName, c.namespaceName, c.flags.VersionNumber) if err != nil { return fmt.Errorf("failed to finish release generation process: %+v", err) } From 5d4edd2e4caf3ae36bc57cde30504336ce77eb0f Mon Sep 17 00:00:00 2001 From: tadelesh Date: Sun, 15 Aug 2021 19:41:59 +0800 Subject: [PATCH 05/21] add automation script and config for track2 --- initScriptTrack2.sh | 18 ++++++++++++++++++ swagger_to_sdk_config_track2.json | 28 ++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 initScriptTrack2.sh create mode 100644 swagger_to_sdk_config_track2.json diff --git a/initScriptTrack2.sh b/initScriptTrack2.sh new file mode 100644 index 000000000000..77edfeb454e2 --- /dev/null +++ b/initScriptTrack2.sh @@ -0,0 +1,18 @@ +set -x +set -e +export GOPATH="$(realpath ../../../..)" +if [ ! -d "$GOPATH/bin" ] +then + mkdir $GOPATH/bin +fi +PATH=$PATH:$GOPATH/bin +export GO111MODULE=on +cd tools/generator && go build && cp generator $GOPATH/bin && cd ../.. +cat > $2 << EOF +{ + "envs": { + "PATH": "$PATH:$GOPATH", + "GOPATH": "$GOPATH" + } +} +EOF \ No newline at end of file diff --git a/swagger_to_sdk_config_track2.json b/swagger_to_sdk_config_track2.json new file mode 100644 index 000000000000..f629cdbae43d --- /dev/null +++ b/swagger_to_sdk_config_track2.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://raw.githubusercontent.com/Azure/azure-rest-api-specs/master/documentation/sdkautomation/SwaggerToSdkConfigSchema.json", + "generateOptions": { + "generateScript": { + "path": "generator automation-track2", + "stderr": { + "showInComment": "^\\[AUTOREST\\]", + "scriptError": "^\\[ERROR\\]", + "scriptWarning": "^\\[WARNING\\]" + } + } + }, + "advancedOptions": { + "cloneDir": "src/github.com/Azure/azure-sdk-for-go", + "breakingChangeTracking": true + }, + "initOptions": { + "initScript": { + "path": "sh ./initScriptTrack2.sh", + "stderr": { + "scriptWarning": false + } + } + }, + "packageOptions": { + "breakingChangeLabel": "CI-BreakingChange-Go" + } +} From 417cf293262fd9028f8b4d0269faffe0d3bfdf97 Mon Sep 17 00:00:00 2001 From: tadelesh Date: Sun, 15 Aug 2021 19:44:43 +0800 Subject: [PATCH 06/21] fix: three dots path not work in linux in build script --- eng/scripts/build.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/scripts/build.ps1 b/eng/scripts/build.ps1 index 7137bf38702f..adea3febc3db 100644 --- a/eng/scripts/build.ps1 +++ b/eng/scripts/build.ps1 @@ -63,7 +63,7 @@ $root = Resolve-Path ($PSScriptRoot + "/../..") Set-Location $root $sdks = @{}; -foreach ($sdk in (Get-ModuleDirs 'sdk/...')) { +foreach ($sdk in (Get-ModuleDirs 'sdk/')) { $name = $sdk | split-path -leaf $sdks[$name] = @{ 'path' = $sdk; From 5a531414693aefbcb1000e49b7041ddb08318b6c Mon Sep 17 00:00:00 2001 From: tadelesh Date: Sun, 15 Aug 2021 20:09:14 +0800 Subject: [PATCH 07/21] fix: ln pwsh.exe to pwsh to run in linux --- initScriptTrack2.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/initScriptTrack2.sh b/initScriptTrack2.sh index 77edfeb454e2..df9cc281eb25 100644 --- a/initScriptTrack2.sh +++ b/initScriptTrack2.sh @@ -8,6 +8,7 @@ fi PATH=$PATH:$GOPATH/bin export GO111MODULE=on cd tools/generator && go build && cp generator $GOPATH/bin && cd ../.. +ln -s /usr/bin/pwsh $GOPATH/bin/pwsh.exe cat > $2 << EOF { "envs": { From 0cd9e086f681ac6c5db1439725fa5677baf6ab14 Mon Sep 17 00:00:00 2001 From: tadelesh Date: Sun, 15 Aug 2021 22:30:22 +0800 Subject: [PATCH 08/21] fix: ignore dep check for go-git --- Gopkg.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gopkg.toml b/Gopkg.toml index bc5dcb61ece3..29dd735165a4 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -19,7 +19,7 @@ # name = "github.com/x/y" # version = "2.4.0" -ignored = ["github.com/ahmetb/go-linq/*", "github.com/google/go-github/*"] +ignored = ["github.com/ahmetb/go-linq/*", "github.com/google/go-github/*", "github.com/go-git/go-git/v5", "github.com/go-git/go-git/v5/*"] [prune] unused-packages = true From 92659f5c8b93223290b090340125e9135397fa83 Mon Sep 17 00:00:00 2001 From: tadelesh Date: Mon, 16 Aug 2021 11:38:11 +0800 Subject: [PATCH 09/21] chore: label name change for track2 --- swagger_to_sdk_config_track2.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swagger_to_sdk_config_track2.json b/swagger_to_sdk_config_track2.json index f629cdbae43d..a4141b6fbfbd 100644 --- a/swagger_to_sdk_config_track2.json +++ b/swagger_to_sdk_config_track2.json @@ -23,6 +23,6 @@ } }, "packageOptions": { - "breakingChangeLabel": "CI-BreakingChange-Go" + "breakingChangeLabel": "CI-BreakingChange-Go-Track2" } } From 3a27314f1b579dcf6d83d530c36d8c110be289d7 Mon Sep 17 00:00:00 2001 From: tadelesh Date: Mon, 16 Aug 2021 13:13:06 +0800 Subject: [PATCH 10/21] fix: wrong package path result in empty git commit --- tools/generator/cmd/track2/automation/automationCmd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/generator/cmd/track2/automation/automationCmd.go b/tools/generator/cmd/track2/automation/automationCmd.go index 68976e505c60..0f905291e7dd 100644 --- a/tools/generator/cmd/track2/automation/automationCmd.go +++ b/tools/generator/cmd/track2/automation/automationCmd.go @@ -105,7 +105,7 @@ func (ctx *automationContext) generate(input *pipeline.GenerateInput) (*pipeline results = append(results, pipeline.PackageResult{ Version: namespaceResult.Version, PackageName: namespaceResult.PackageName, - Path: []string{fmt.Sprintf("/sdk/%s/%s", namespaceResult.RpName, namespaceResult.PackageName)}, + Path: []string{fmt.Sprintf("sdk/%s/%s", namespaceResult.RpName, namespaceResult.PackageName)}, ReadmeMd: []string{readme}, Changelog: &pipeline.Changelog{ Content: &content, From 57a5c3917e683f604b878f02179fbf8c657e1c6f Mon Sep 17 00:00:00 2001 From: tadelesh Date: Mon, 16 Aug 2021 17:26:12 +0800 Subject: [PATCH 11/21] fix: repo addr replacement in autorest.md --- .../cmd/track2/automation/automationCmd.go | 2 +- .../cmd/track2/common/fileProcessor.go | 25 ++++++++++++++++--- .../generator/cmd/track2/common/generation.go | 23 ++++++++++++++--- .../cmd/track2/release/releaseCmd.go | 3 ++- 4 files changed, 44 insertions(+), 9 deletions(-) diff --git a/tools/generator/cmd/track2/automation/automationCmd.go b/tools/generator/cmd/track2/automation/automationCmd.go index 0f905291e7dd..8056dd8047d3 100644 --- a/tools/generator/cmd/track2/automation/automationCmd.go +++ b/tools/generator/cmd/track2/automation/automationCmd.go @@ -91,7 +91,7 @@ func (ctx *automationContext) generate(input *pipeline.GenerateInput) (*pipeline CommitHash: ctx.commitHash, } - namespaceResults, errors := generateCtx.GenerateForAutomation(readme) + namespaceResults, errors := generateCtx.GenerateForAutomation(readme, input.RepoHTTPSURL) if len(errors) != 0 { errorBuilder.add(errors...) continue diff --git a/tools/generator/cmd/track2/common/fileProcessor.go b/tools/generator/cmd/track2/common/fileProcessor.go index 6a791c8552f8..b8da520be88b 100644 --- a/tools/generator/cmd/track2/common/fileProcessor.go +++ b/tools/generator/cmd/track2/common/fileProcessor.go @@ -25,8 +25,9 @@ const ( ) var ( - track2BeginRegex = regexp.MustCompile("^```\\s*yaml\\s*\\$\\(go\\)\\s*&&\\s*\\$\\(track2\\)") - track2EndRegex = regexp.MustCompile("^\\s*```\\s*$") + track2BeginRegex = regexp.MustCompile("^```\\s*yaml\\s*\\$\\(go\\)\\s*&&\\s*\\$\\(track2\\)") + track2EndRegex = regexp.MustCompile("^\\s*```\\s*$") + autorestMdSwaggerURLBeginRegex = regexp.MustCompile(`https://github.com/.+/azure-rest-api-specs/`) ) // reads from readme.go.md, parses the `track2` section to get module and package name @@ -105,7 +106,7 @@ func CleanSDKGeneratedFiles(path string) error { // replace all commit id in autorest.md files func ReplaceCommitID(path string, commitID string) error { - log.Printf("Replacing commit id from autorest.md ...") + log.Printf("Replacing commit id in autorest.md ...") b, err := ioutil.ReadFile(path) if err != nil { return err @@ -122,6 +123,24 @@ func ReplaceCommitID(path string, commitID string) error { return err } +func ReplaceRepoURL(path string, repoUrl string) error { + log.Printf("Replacing repo url in autorest.md ...") + b, err := ioutil.ReadFile(path) + if err != nil { + return err + } + + lines := strings.Split(string(b), "\n") + for i, line := range lines { + if pos := autorestMdSwaggerURLBeginRegex.FindStringIndex(line); pos != nil { + lines[i] = line[:pos[0]] + repoUrl + "/" + line[pos[1]:] + } + } + + err = ioutil.WriteFile(path, []byte(strings.Join(lines, "\n")), 0644) + return err +} + func GetLatestVersion(packageRootPath string) (*semver.Version, error) { b, err := ioutil.ReadFile(filepath.Join(packageRootPath, "autorest.md")) if err != nil { diff --git a/tools/generator/cmd/track2/common/generation.go b/tools/generator/cmd/track2/common/generation.go index ef1fa967287d..5eea74d7f144 100644 --- a/tools/generator/cmd/track2/common/generation.go +++ b/tools/generator/cmd/track2/common/generation.go @@ -41,7 +41,7 @@ func (ctx GenerateContext) SpecRoot() string { return ctx.SpecPath } -func (ctx GenerateContext) GenerateForAutomation(readme string) ([]GenerateResult, []error) { +func (ctx GenerateContext) GenerateForAutomation(readme string, repo string) ([]GenerateResult, []error) { absReadme := filepath.Join(ctx.SpecPath, readme) absReadmeGo := filepath.Join(filepath.Dir(absReadme), "readme.go.md") @@ -59,7 +59,7 @@ func (ctx GenerateContext) GenerateForAutomation(readme string) ([]GenerateResul for rpName, namespaceNames := range rpMap { for _, namespaceName := range namespaceNames { log.Printf("Process rp: %s, namespace: %s", rpName, namespaceName) - singleResult, err := ctx.GenerateForSingleRpNamespace(rpName, namespaceName, "") + singleResult, err := ctx.GenerateForSingleRpNamespace(rpName, namespaceName, "", repo) if err != nil { errors = append(errors, err) continue @@ -70,7 +70,7 @@ func (ctx GenerateContext) GenerateForAutomation(readme string) ([]GenerateResul return result, errors } -func (ctx GenerateContext) GenerateForSingleRpNamespace(rpName, namespaceName string, specficVersion string) (*GenerateResult, error) { +func (ctx GenerateContext) GenerateForSingleRpNamespace(rpName, namespaceName string, specficVersion string, specficRepoURL string) (*GenerateResult, error) { packagePath := filepath.Join(ctx.SdkPath, "sdk", rpName, namespaceName) changelogPath := filepath.Join(packagePath, common.ChangelogFilename) if _, err := os.Stat(changelogPath); os.IsNotExist(err) { @@ -86,6 +86,14 @@ func (ctx GenerateContext) GenerateForSingleRpNamespace(rpName, namespaceName st return nil, err } + if specficRepoURL != "" { + log.Printf("Change the repo url in `autorest.md`...") + autorestMdPath := filepath.Join(packagePath, "autorest.md") + if err = ReplaceRepoURL(autorestMdPath, specficRepoURL); err != nil { + return nil, err + } + } + log.Printf("Run `go generate` to regenerate the code...") if err = ExecuteGoGenerate(packagePath); err != nil { return nil, err @@ -129,6 +137,13 @@ func (ctx GenerateContext) GenerateForSingleRpNamespace(rpName, namespaceName st return nil, err } + if specficRepoURL != "" { + log.Printf("Change the repo url in `autorest.md`...") + if err = ReplaceRepoURL(autorestMdPath, specficRepoURL); err != nil { + return nil, err + } + } + log.Printf("Run `go generate` to regenerate the code...") if err = ExecuteGoGenerate(packagePath); err != nil { return nil, err @@ -146,7 +161,7 @@ func (ctx GenerateContext) GenerateForSingleRpNamespace(rpName, namespaceName st log.Printf("Calculate new version...") var version *semver.Version - if len(specficVersion) == 0 { + if specficVersion == "" { version, err = CalculateNewVersion(changelog, packagePath) if err != nil { return nil, err diff --git a/tools/generator/cmd/track2/release/releaseCmd.go b/tools/generator/cmd/track2/release/releaseCmd.go index 57f8f9d843f8..1b15960f9fd9 100644 --- a/tools/generator/cmd/track2/release/releaseCmd.go +++ b/tools/generator/cmd/track2/release/releaseCmd.go @@ -63,6 +63,7 @@ namespaceName: name of namespace to be released, default value is arm+rp-name type Flags struct { VersionNumber string + RepoURL string } func BindFlags(flagSet *pflag.FlagSet) { @@ -117,7 +118,7 @@ func (c *commandContext) execute() error { CommitHash: specRef.Hash().String(), } - result, err := generateCtx.GenerateForSingleRpNamespace(c.rpName, c.namespaceName, c.flags.VersionNumber) + result, err := generateCtx.GenerateForSingleRpNamespace(c.rpName, c.namespaceName, c.flags.VersionNumber, c.flags.RepoURL) if err != nil { return fmt.Errorf("failed to finish release generation process: %+v", err) } From f77a02cf70ec3dea50a5a2810c972c399e2107f9 Mon Sep 17 00:00:00 2001 From: tadelesh Date: Tue, 17 Aug 2021 13:09:46 +0800 Subject: [PATCH 12/21] chore: move script to eng folder and change name --- initScriptTrack2.sh => eng/scripts/automation_init.sh | 0 .../swagger_to_sdk_config.json | 6 +++--- tools/generator/cmd/track2/automation/automationCmd.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename initScriptTrack2.sh => eng/scripts/automation_init.sh (100%) rename swagger_to_sdk_config_track2.json => eng/swagger_to_sdk_config.json (78%) diff --git a/initScriptTrack2.sh b/eng/scripts/automation_init.sh similarity index 100% rename from initScriptTrack2.sh rename to eng/scripts/automation_init.sh diff --git a/swagger_to_sdk_config_track2.json b/eng/swagger_to_sdk_config.json similarity index 78% rename from swagger_to_sdk_config_track2.json rename to eng/swagger_to_sdk_config.json index a4141b6fbfbd..84618c60b979 100644 --- a/swagger_to_sdk_config_track2.json +++ b/eng/swagger_to_sdk_config.json @@ -2,7 +2,7 @@ "$schema": "https://raw.githubusercontent.com/Azure/azure-rest-api-specs/master/documentation/sdkautomation/SwaggerToSdkConfigSchema.json", "generateOptions": { "generateScript": { - "path": "generator automation-track2", + "path": "generator automation-new-version", "stderr": { "showInComment": "^\\[AUTOREST\\]", "scriptError": "^\\[ERROR\\]", @@ -16,13 +16,13 @@ }, "initOptions": { "initScript": { - "path": "sh ./initScriptTrack2.sh", + "path": "sh ./eng/scripts/automation_init.sh", "stderr": { "scriptWarning": false } } }, "packageOptions": { - "breakingChangeLabel": "CI-BreakingChange-Go-Track2" + "breakingChangeLabel": "CI-BreakingChange-Go-NewVersion" } } diff --git a/tools/generator/cmd/track2/automation/automationCmd.go b/tools/generator/cmd/track2/automation/automationCmd.go index 8056dd8047d3..ed55bd14d410 100644 --- a/tools/generator/cmd/track2/automation/automationCmd.go +++ b/tools/generator/cmd/track2/automation/automationCmd.go @@ -19,7 +19,7 @@ import ( // azure-sdk-for-go. It does not work if you are running this tool in somewhere else func Command() *cobra.Command { cmd := &cobra.Command{ - Use: "automation-track2 ", + Use: "automation-new-version ", Args: cobra.ExactArgs(2), PersistentPreRunE: func(cmd *cobra.Command, args []string) error { log.SetFlags(0) // remove the time stamp prefix From db9c1f0f8e19a1c0ce4a5a2b840dac9908b90482 Mon Sep 17 00:00:00 2001 From: tadelesh Date: Tue, 17 Aug 2021 15:03:26 +0800 Subject: [PATCH 13/21] fix: onboard generation need to replace NewClientMethod --- .../cmd/track2/common/cmdProcessor.go | 3 ++ .../cmd/track2/common/fileProcessor.go | 39 ++++++++++++++++--- .../generator/cmd/track2/common/generation.go | 7 +++- 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/tools/generator/cmd/track2/common/cmdProcessor.go b/tools/generator/cmd/track2/common/cmdProcessor.go index a6b329c87310..1fdf20f84815 100644 --- a/tools/generator/cmd/track2/common/cmdProcessor.go +++ b/tools/generator/cmd/track2/common/cmdProcessor.go @@ -5,13 +5,16 @@ package common import ( "fmt" + "log" "os/exec" ) +// execute `go generate` command and fetch result func ExecuteGoGenerate(path string) error { cmd := exec.Command("go", "generate") cmd.Dir = path output, err := cmd.CombinedOutput() + log.Printf("Result of `go generate` execution: \n%s", string(output)) if err != nil { return fmt.Errorf("failed to execute go generate '%s': %+v", string(output), err) } diff --git a/tools/generator/cmd/track2/common/fileProcessor.go b/tools/generator/cmd/track2/common/fileProcessor.go index b8da520be88b..6e425b3eab15 100644 --- a/tools/generator/cmd/track2/common/fileProcessor.go +++ b/tools/generator/cmd/track2/common/fileProcessor.go @@ -14,6 +14,7 @@ import ( "github.com/Azure/azure-sdk-for-go/tools/generator/autorest/model" "github.com/Azure/azure-sdk-for-go/tools/generator/common" + "github.com/Azure/azure-sdk-for-go/tools/internal/exports" "github.com/Masterminds/semver" ) @@ -28,6 +29,7 @@ var ( track2BeginRegex = regexp.MustCompile("^```\\s*yaml\\s*\\$\\(go\\)\\s*&&\\s*\\$\\(track2\\)") track2EndRegex = regexp.MustCompile("^\\s*```\\s*$") autorestMdSwaggerURLBeginRegex = regexp.MustCompile(`https://github.com/.+/azure-rest-api-specs/`) + newClientMethodNameRegex = regexp.MustCompile("^New.+Client$") ) // reads from readme.go.md, parses the `track2` section to get module and package name @@ -119,10 +121,10 @@ func ReplaceCommitID(path string, commitID string) error { } } - err = ioutil.WriteFile(path, []byte(strings.Join(lines, "\n")), 0644) - return err + return ioutil.WriteFile(path, []byte(strings.Join(lines, "\n")), 0644) } +// replace repo url according to `https://github.com/.+/azure-rest-api-specs/` pattern in autorest.md files func ReplaceRepoURL(path string, repoUrl string) error { log.Printf("Replacing repo url in autorest.md ...") b, err := ioutil.ReadFile(path) @@ -137,10 +139,10 @@ func ReplaceRepoURL(path string, repoUrl string) error { } } - err = ioutil.WriteFile(path, []byte(strings.Join(lines, "\n")), 0644) - return err + return ioutil.WriteFile(path, []byte(strings.Join(lines, "\n")), 0644) } +// get latest version according to `module-version: ` prefix in autorest.md file func GetLatestVersion(packageRootPath string) (*semver.Version, error) { b, err := ioutil.ReadFile(filepath.Join(packageRootPath, "autorest.md")) if err != nil { @@ -158,6 +160,7 @@ func GetLatestVersion(packageRootPath string) (*semver.Version, error) { return nil, fmt.Errorf("cannot parse version from autorest.md") } +// replace version according to `module-version: ` prefix in autorest.md file func ReplaceVersion(packageRootPath string, newVersion string) error { path := filepath.Join(packageRootPath, "autorest.md") b, err := ioutil.ReadFile(path) @@ -173,10 +176,10 @@ func ReplaceVersion(packageRootPath string, newVersion string) error { } } - err = ioutil.WriteFile(path, []byte(strings.Join(lines, "\n")), 0644) - return err + return ioutil.WriteFile(path, []byte(strings.Join(lines, "\n")), 0644) } +// calculate new version by changelog using semver package func CalculateNewVersion(changelog *model.Changelog, packageRootPath string) (*semver.Version, error) { version, err := GetLatestVersion(packageRootPath) if err != nil { @@ -207,6 +210,7 @@ func CalculateNewVersion(changelog *model.Changelog, packageRootPath string) (*s return &newVersion, nil } +// add new changelog md to changelog file func AddChangelogToFile(changelog *model.Changelog, version *semver.Version, packageRootPath string) (string, error) { path := filepath.Join(packageRootPath, common.ChangelogFilename) b, err := ioutil.ReadFile(path) @@ -223,3 +227,26 @@ func AddChangelogToFile(changelog *model.Changelog, version *semver.Version, pac } return additionalChangelog, nil } + +// replace `{{NewClientMethod}}`` placeholder in README.md by first func name according to `^New.+Method$` pattern +func ReplaceNewClientMethodPlaceholder(packageRootPath string, exports exports.Content) error { + path := filepath.Join(packageRootPath, "README.md") + var clientName string + for k, v := range exports.Funcs { + if newClientMethodNameRegex.MatchString(k) && *v.Params == "*armcore.Connection, string" { + clientName = k + break + } + } + if clientName == "" { + return fmt.Errorf("cannot find any NewClientMethod in package") + } + + b, err := ioutil.ReadFile(path) + if err != nil { + return fmt.Errorf("cannot read from file '%s': %+v", path, err) + } + + content := strings.ReplaceAll(string(b), "{{NewClientMethod}}", clientName) + return ioutil.WriteFile(path, []byte(content), 0644) +} diff --git a/tools/generator/cmd/track2/common/generation.go b/tools/generator/cmd/track2/common/generation.go index 5eea74d7f144..aa62539bc9d9 100644 --- a/tools/generator/cmd/track2/common/generation.go +++ b/tools/generator/cmd/track2/common/generation.go @@ -74,7 +74,7 @@ func (ctx GenerateContext) GenerateForSingleRpNamespace(rpName, namespaceName st packagePath := filepath.Join(ctx.SdkPath, "sdk", rpName, namespaceName) changelogPath := filepath.Join(packagePath, common.ChangelogFilename) if _, err := os.Stat(changelogPath); os.IsNotExist(err) { - log.Printf("Package '%s' not onboard, do onboard process", packagePath) + log.Printf("Package '%s' changelog not exist, do onboard process", packagePath) log.Printf("Use template to generate new rp folder and basic package files...") if err = template.GeneratePackageByTemplate(rpName, namespaceName, template.Flags{ @@ -109,6 +109,11 @@ func (ctx GenerateContext) GenerateForSingleRpNamespace(rpName, namespaceName st return nil, err } + log.Printf("Replace {{NewClientMethod}} placeholder in the README.md ") + if err = ReplaceNewClientMethodPlaceholder(packagePath, newExports); err != nil { + return nil, err + } + return &GenerateResult{ Version: "0.1.0", RpName: rpName, From 657ec8dfe8245be7057b528773bbc01cfbb222b4 Mon Sep 17 00:00:00 2001 From: tadelesh Date: Wed, 18 Aug 2021 13:58:04 +0800 Subject: [PATCH 14/21] chore: change all naming from track2 to v2 --- eng/swagger_to_sdk_config.json | 4 ++-- tools/generator/cmd/root.go | 8 +++---- .../automation/automationCmd.go | 6 ++--- .../cmd/{track2 => v2}/common/cmdProcessor.go | 0 .../{track2 => v2}/common/fileProcessor.go | 22 +++++++++---------- .../cmd/{track2 => v2}/common/generation.go | 2 +- .../cmd/{track2 => v2}/release/releaseCmd.go | 8 +++---- 7 files changed, 25 insertions(+), 25 deletions(-) rename tools/generator/cmd/{track2 => v2}/automation/automationCmd.go (93%) rename tools/generator/cmd/{track2 => v2}/common/cmdProcessor.go (100%) rename tools/generator/cmd/{track2 => v2}/common/fileProcessor.go (91%) rename tools/generator/cmd/{track2 => v2}/common/generation.go (98%) rename tools/generator/cmd/{track2 => v2}/release/releaseCmd.go (92%) diff --git a/eng/swagger_to_sdk_config.json b/eng/swagger_to_sdk_config.json index 84618c60b979..2dbd8f199b43 100644 --- a/eng/swagger_to_sdk_config.json +++ b/eng/swagger_to_sdk_config.json @@ -2,7 +2,7 @@ "$schema": "https://raw.githubusercontent.com/Azure/azure-rest-api-specs/master/documentation/sdkautomation/SwaggerToSdkConfigSchema.json", "generateOptions": { "generateScript": { - "path": "generator automation-new-version", + "path": "generator automation-v2", "stderr": { "showInComment": "^\\[AUTOREST\\]", "scriptError": "^\\[ERROR\\]", @@ -23,6 +23,6 @@ } }, "packageOptions": { - "breakingChangeLabel": "CI-BreakingChange-Go-NewVersion" + "breakingChangeLabel": "CI-BreakingChange-Go-V2" } } diff --git a/tools/generator/cmd/root.go b/tools/generator/cmd/root.go index 129ee772885a..d2fb678b7619 100644 --- a/tools/generator/cmd/root.go +++ b/tools/generator/cmd/root.go @@ -10,8 +10,8 @@ import ( "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/automation" "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/issue" "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/template" - automation_track2 "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/track2/automation" - release_track2 "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/track2/release" + automation_v2 "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/v2/automation" + release_v2 "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/v2/release" "github.com/spf13/cobra" ) @@ -31,8 +31,8 @@ func Command() *cobra.Command { rootCmd.AddCommand( automation.Command(), - automation_track2.Command(), - release_track2.Command(), + automation_v2.Command(), + release_v2.Command(), issue.Command(), template.Command(), ) diff --git a/tools/generator/cmd/track2/automation/automationCmd.go b/tools/generator/cmd/v2/automation/automationCmd.go similarity index 93% rename from tools/generator/cmd/track2/automation/automationCmd.go rename to tools/generator/cmd/v2/automation/automationCmd.go index ed55bd14d410..30a2a11fdd4c 100644 --- a/tools/generator/cmd/track2/automation/automationCmd.go +++ b/tools/generator/cmd/v2/automation/automationCmd.go @@ -10,16 +10,16 @@ import ( "strings" "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/automation/pipeline" - "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/track2/common" + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/v2/common" "github.com/Azure/azure-sdk-for-go/tools/internal/utils" "github.com/spf13/cobra" ) -// Command returns the automation for track2 command. Note that this command is designed to run in the root directory of +// Command returns the automation v2 command. Note that this command is designed to run in the root directory of // azure-sdk-for-go. It does not work if you are running this tool in somewhere else func Command() *cobra.Command { cmd := &cobra.Command{ - Use: "automation-new-version ", + Use: "automation-v2 ", Args: cobra.ExactArgs(2), PersistentPreRunE: func(cmd *cobra.Command, args []string) error { log.SetFlags(0) // remove the time stamp prefix diff --git a/tools/generator/cmd/track2/common/cmdProcessor.go b/tools/generator/cmd/v2/common/cmdProcessor.go similarity index 100% rename from tools/generator/cmd/track2/common/cmdProcessor.go rename to tools/generator/cmd/v2/common/cmdProcessor.go diff --git a/tools/generator/cmd/track2/common/fileProcessor.go b/tools/generator/cmd/v2/common/fileProcessor.go similarity index 91% rename from tools/generator/cmd/track2/common/fileProcessor.go rename to tools/generator/cmd/v2/common/fileProcessor.go index 6e425b3eab15..60f73d9e49fe 100644 --- a/tools/generator/cmd/track2/common/fileProcessor.go +++ b/tools/generator/cmd/v2/common/fileProcessor.go @@ -26,14 +26,14 @@ const ( ) var ( - track2BeginRegex = regexp.MustCompile("^```\\s*yaml\\s*\\$\\(go\\)\\s*&&\\s*\\$\\(track2\\)") - track2EndRegex = regexp.MustCompile("^\\s*```\\s*$") + v2BeginRegex = regexp.MustCompile("^```\\s*yaml\\s*\\$\\(go\\)\\s*&&\\s*\\$\\(track2\\)") + v2EndRegex = regexp.MustCompile("^\\s*```\\s*$") autorestMdSwaggerURLBeginRegex = regexp.MustCompile(`https://github.com/.+/azure-rest-api-specs/`) newClientMethodNameRegex = regexp.MustCompile("^New.+Client$") ) // reads from readme.go.md, parses the `track2` section to get module and package name -func ReadTrack2ModuleNameToGetNamespace(path string) (map[string][]string, error) { +func ReadV2ModuleNameToGetNamespace(path string) (map[string][]string, error) { result := make(map[string][]string) log.Printf("Reading from readme.go.md '%s'...", path) file, err := os.Open(path) @@ -52,30 +52,30 @@ func ReadTrack2ModuleNameToGetNamespace(path string) (map[string][]string, error var start []int var end []int for i, line := range lines { - if track2BeginRegex.MatchString(line) { + if v2BeginRegex.MatchString(line) { start = append(start, i) } - if len(start) != len(end) && track2EndRegex.MatchString(line) { + if len(start) != len(end) && v2EndRegex.MatchString(line) { end = append(end, i) } } if len(start) == 0 { - return nil, fmt.Errorf("cannot find any track2 section") + return nil, fmt.Errorf("cannot find any `track2` section") } if len(start) != len(end) { - return nil, fmt.Errorf("last track2 section does not properly end") + return nil, fmt.Errorf("last `track2` section does not properly end") } for i := range start { - // get the content of the track2 section - track2Section := lines[start[i]+1 : end[i]] + // get the content of the `track2` section + section := lines[start[i]+1 : end[i]] // iterate over the rest lines, get module name - for _, line := range track2Section { + for _, line := range section { if strings.HasPrefix(line, swagger_md_module_name_prefix) { modules := strings.Split(strings.TrimSpace(line[len(swagger_md_module_name_prefix):]), "/") if len(modules) != 3 { - return nil, fmt.Errorf("cannot parse module name from track2 section") + return nil, fmt.Errorf("cannot parse module name from `track2` section") } namespaceName := strings.TrimSuffix(strings.TrimSuffix(modules[2], "\n"), "\r") log.Printf("RP: %s Package: %s", modules[1], namespaceName) diff --git a/tools/generator/cmd/track2/common/generation.go b/tools/generator/cmd/v2/common/generation.go similarity index 98% rename from tools/generator/cmd/track2/common/generation.go rename to tools/generator/cmd/v2/common/generation.go index aa62539bc9d9..215cc13a8691 100644 --- a/tools/generator/cmd/track2/common/generation.go +++ b/tools/generator/cmd/v2/common/generation.go @@ -49,7 +49,7 @@ func (ctx GenerateContext) GenerateForAutomation(readme string, repo string) ([] var errors []error log.Printf("Get all namespaces from readme file") - rpMap, err := ReadTrack2ModuleNameToGetNamespace(absReadmeGo) + rpMap, err := ReadV2ModuleNameToGetNamespace(absReadmeGo) if err != nil { return nil, []error{ fmt.Errorf("cannot get rp and namespaces from readme '%s': %+v", readme, err), diff --git a/tools/generator/cmd/track2/release/releaseCmd.go b/tools/generator/cmd/v2/release/releaseCmd.go similarity index 92% rename from tools/generator/cmd/track2/release/releaseCmd.go rename to tools/generator/cmd/v2/release/releaseCmd.go index 1b15960f9fd9..fe1d07b4ff6e 100644 --- a/tools/generator/cmd/track2/release/releaseCmd.go +++ b/tools/generator/cmd/v2/release/releaseCmd.go @@ -9,7 +9,7 @@ import ( "path/filepath" "time" - "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/track2/common" + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/v2/common" "github.com/Azure/azure-sdk-for-go/tools/generator/flags" "github.com/Azure/azure-sdk-for-go/tools/generator/repo" "github.com/spf13/cobra" @@ -19,9 +19,9 @@ import ( // Release command func Command() *cobra.Command { releaseCmd := &cobra.Command{ - Use: "release-track2 ", - Short: "Generate a track2 release of azure-sdk-for-go", - Long: `This command will generate a new track2 release for azure-sdk-for-go with given rp name and namespace name. + Use: "release-v2 ", + Short: "Generate a v2 release of azure-sdk-for-go", + Long: `This command will generate a new v2 release for azure-sdk-for-go with given rp name and namespace name. azure-sdk-for-go directory: the directory path of the azure-sdk-for-go with git control azure-rest-api-specs directory: the directory path of the azure-rest-api-specs with git control From 4eabb8fa2eb52f832a937e571cb2f7173150f1ad Mon Sep 17 00:00:00 2001 From: tadelesh Date: Wed, 18 Aug 2021 16:59:38 +0800 Subject: [PATCH 15/21] fix: add new build constraint --- tools/generator/go_mod_tidy_hack.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/generator/go_mod_tidy_hack.go b/tools/generator/go_mod_tidy_hack.go index b8f9221eac9a..6e23cdbaf18f 100644 --- a/tools/generator/go_mod_tidy_hack.go +++ b/tools/generator/go_mod_tidy_hack.go @@ -1,3 +1,4 @@ +//go:build modhack // +build modhack package main From ac3b18e7b98bd617678890479c38f2f47a577004 Mon Sep 17 00:00:00 2001 From: tadelesh Date: Wed, 18 Aug 2021 17:12:50 +0800 Subject: [PATCH 16/21] fix: template compat with go 1.17 and change commit msg for release --- tools/generator/repo/sdk.go | 2 +- .../template/rpName/packageName/go_mod_tidy_hack.go.tpl | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/generator/repo/sdk.go b/tools/generator/repo/sdk.go index 0ad44e1c533b..ff5c43b77c31 100644 --- a/tools/generator/repo/sdk.go +++ b/tools/generator/repo/sdk.go @@ -37,7 +37,7 @@ func (s *sdkRepository) AddReleaseCommit(rpName, namespaceName, specHash, versio return fmt.Errorf("failed to add 'profiles': %+v", err) } - message := fmt.Sprintf("Release: %s:%s from spec commit hash: %s", rpName, version, specHash) + message := fmt.Sprintf("[Track2 Release] sdk/%s/%s/%s generation from spec commit: %s", rpName, namespaceName, version, specHash) if err := s.Commit(message); err != nil { if IsNothingToCommit(err) { log.Printf("There is nothing to commit. Message: %s", message) diff --git a/tools/generator/template/rpName/packageName/go_mod_tidy_hack.go.tpl b/tools/generator/template/rpName/packageName/go_mod_tidy_hack.go.tpl index 3c627ffe904f..b5b9dcc73d26 100644 --- a/tools/generator/template/rpName/packageName/go_mod_tidy_hack.go.tpl +++ b/tools/generator/template/rpName/packageName/go_mod_tidy_hack.go.tpl @@ -1,3 +1,4 @@ +//go:build modhack // +build modhack // Copyright (c) Microsoft Corporation. All rights reserved. From c3bae2d876323fae411e3e3b656f09a2ee2b6d48 Mon Sep 17 00:00:00 2001 From: tadelesh Date: Thu, 19 Aug 2021 11:01:04 +0800 Subject: [PATCH 17/21] chore: add v2 to readme begin regex to support future change of `track2` naming --- tools/generator/cmd/v2/common/fileProcessor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/generator/cmd/v2/common/fileProcessor.go b/tools/generator/cmd/v2/common/fileProcessor.go index 60f73d9e49fe..fc00513cff06 100644 --- a/tools/generator/cmd/v2/common/fileProcessor.go +++ b/tools/generator/cmd/v2/common/fileProcessor.go @@ -26,7 +26,7 @@ const ( ) var ( - v2BeginRegex = regexp.MustCompile("^```\\s*yaml\\s*\\$\\(go\\)\\s*&&\\s*\\$\\(track2\\)") + v2BeginRegex = regexp.MustCompile("^```\\s*yaml\\s*\\$\\(go\\)\\s*&&\\s*\\$\\((track2|v2)\\)") v2EndRegex = regexp.MustCompile("^\\s*```\\s*$") autorestMdSwaggerURLBeginRegex = regexp.MustCompile(`https://github.com/.+/azure-rest-api-specs/`) newClientMethodNameRegex = regexp.MustCompile("^New.+Client$") From 4afbde623166511145b1a40d32e1ffcb3ff34839 Mon Sep 17 00:00:00 2001 From: tadelesh Date: Thu, 19 Aug 2021 11:05:56 +0800 Subject: [PATCH 18/21] chore: remove track2 naming --- tools/generator/repo/sdk.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/generator/repo/sdk.go b/tools/generator/repo/sdk.go index ff5c43b77c31..dab0d766ad3a 100644 --- a/tools/generator/repo/sdk.go +++ b/tools/generator/repo/sdk.go @@ -37,7 +37,7 @@ func (s *sdkRepository) AddReleaseCommit(rpName, namespaceName, specHash, versio return fmt.Errorf("failed to add 'profiles': %+v", err) } - message := fmt.Sprintf("[Track2 Release] sdk/%s/%s/%s generation from spec commit: %s", rpName, namespaceName, version, specHash) + message := fmt.Sprintf("[Release] sdk/%s/%s/%s generation from spec commit: %s", rpName, namespaceName, version, specHash) if err := s.Commit(message); err != nil { if IsNothingToCommit(err) { log.Printf("There is nothing to commit. Message: %s", message) From d6b470c4574ce32e529be38cc91c6036abbe1caf Mon Sep 17 00:00:00 2001 From: tadelesh Date: Thu, 19 Aug 2021 13:49:05 +0800 Subject: [PATCH 19/21] fix: duplicated and wrong expression --- Gopkg.toml | 2 +- tools/generator/cmd/v2/release/releaseCmd.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Gopkg.toml b/Gopkg.toml index 29dd735165a4..7e8ef58d5205 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -19,7 +19,7 @@ # name = "github.com/x/y" # version = "2.4.0" -ignored = ["github.com/ahmetb/go-linq/*", "github.com/google/go-github/*", "github.com/go-git/go-git/v5", "github.com/go-git/go-git/v5/*"] +ignored = ["github.com/ahmetb/go-linq/*", "github.com/google/go-github/*", "github.com/go-git/go-git/*"] [prune] unused-packages = true diff --git a/tools/generator/cmd/v2/release/releaseCmd.go b/tools/generator/cmd/v2/release/releaseCmd.go index fe1d07b4ff6e..c9631bcbb1d7 100644 --- a/tools/generator/cmd/v2/release/releaseCmd.go +++ b/tools/generator/cmd/v2/release/releaseCmd.go @@ -19,7 +19,7 @@ import ( // Release command func Command() *cobra.Command { releaseCmd := &cobra.Command{ - Use: "release-v2 ", + Use: "release-v2 [namespaceName]", Short: "Generate a v2 release of azure-sdk-for-go", Long: `This command will generate a new v2 release for azure-sdk-for-go with given rp name and namespace name. From 726242c2cb2672fbc834f0f8e0f8ba674b82c6b2 Mon Sep 17 00:00:00 2001 From: tadelesh Date: Fri, 20 Aug 2021 15:49:35 +0800 Subject: [PATCH 20/21] fix: add package title param to release cmd --- tools/generator/cmd/v2/common/generation.go | 10 +++++++--- tools/generator/cmd/v2/release/releaseCmd.go | 7 ++++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/tools/generator/cmd/v2/common/generation.go b/tools/generator/cmd/v2/common/generation.go index 215cc13a8691..e058b8141e64 100644 --- a/tools/generator/cmd/v2/common/generation.go +++ b/tools/generator/cmd/v2/common/generation.go @@ -59,7 +59,7 @@ func (ctx GenerateContext) GenerateForAutomation(readme string, repo string) ([] for rpName, namespaceNames := range rpMap { for _, namespaceName := range namespaceNames { log.Printf("Process rp: %s, namespace: %s", rpName, namespaceName) - singleResult, err := ctx.GenerateForSingleRpNamespace(rpName, namespaceName, "", repo) + singleResult, err := ctx.GenerateForSingleRpNamespace(rpName, namespaceName, "", "", repo) if err != nil { errors = append(errors, err) continue @@ -70,17 +70,21 @@ func (ctx GenerateContext) GenerateForAutomation(readme string, repo string) ([] return result, errors } -func (ctx GenerateContext) GenerateForSingleRpNamespace(rpName, namespaceName string, specficVersion string, specficRepoURL string) (*GenerateResult, error) { +func (ctx GenerateContext) GenerateForSingleRpNamespace(rpName, namespaceName, specficPackageTitle, specficVersion, specficRepoURL string) (*GenerateResult, error) { packagePath := filepath.Join(ctx.SdkPath, "sdk", rpName, namespaceName) changelogPath := filepath.Join(packagePath, common.ChangelogFilename) if _, err := os.Stat(changelogPath); os.IsNotExist(err) { log.Printf("Package '%s' changelog not exist, do onboard process", packagePath) + + if specficPackageTitle == "" { + specficPackageTitle = strings.Title(rpName) + } log.Printf("Use template to generate new rp folder and basic package files...") if err = template.GeneratePackageByTemplate(rpName, namespaceName, template.Flags{ SDKRoot: ctx.SdkPath, TemplatePath: "tools/generator/template/rpName/packageName", - PackageTitle: strings.Title(rpName), + PackageTitle: specficPackageTitle, Commit: ctx.CommitHash, }); err != nil { return nil, err diff --git a/tools/generator/cmd/v2/release/releaseCmd.go b/tools/generator/cmd/v2/release/releaseCmd.go index c9631bcbb1d7..15bcab17e874 100644 --- a/tools/generator/cmd/v2/release/releaseCmd.go +++ b/tools/generator/cmd/v2/release/releaseCmd.go @@ -64,15 +64,20 @@ namespaceName: name of namespace to be released, default value is arm+rp-name type Flags struct { VersionNumber string RepoURL string + PackageTitle string } func BindFlags(flagSet *pflag.FlagSet) { flagSet.String("version-number", "", "Specify the version number of this release") + flagSet.String("repo-url", "", "Specifies the swagger repo url for generation") + flagSet.String("package-title", "", "Specifies the title of this package") } func ParseFlags(flagSet *pflag.FlagSet) Flags { return Flags{ VersionNumber: flags.GetString(flagSet, "version-number"), + RepoURL: flags.GetString(flagSet, "repo-url"), + PackageTitle: flags.GetString(flagSet, "package-title"), } } @@ -118,7 +123,7 @@ func (c *commandContext) execute() error { CommitHash: specRef.Hash().String(), } - result, err := generateCtx.GenerateForSingleRpNamespace(c.rpName, c.namespaceName, c.flags.VersionNumber, c.flags.RepoURL) + result, err := generateCtx.GenerateForSingleRpNamespace(c.rpName, c.namespaceName, c.flags.PackageTitle, c.flags.VersionNumber, c.flags.RepoURL) if err != nil { return fmt.Errorf("failed to finish release generation process: %+v", err) } From 0fd2226d362c1306e6fe10f7903f8850382bc27f Mon Sep 17 00:00:00 2001 From: tadelesh Date: Fri, 20 Aug 2021 16:00:07 +0800 Subject: [PATCH 21/21] fix: ci --- tools/generator/cmd/v2/common/generation.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/generator/cmd/v2/common/generation.go b/tools/generator/cmd/v2/common/generation.go index e058b8141e64..8814665cd772 100644 --- a/tools/generator/cmd/v2/common/generation.go +++ b/tools/generator/cmd/v2/common/generation.go @@ -75,7 +75,7 @@ func (ctx GenerateContext) GenerateForSingleRpNamespace(rpName, namespaceName, s changelogPath := filepath.Join(packagePath, common.ChangelogFilename) if _, err := os.Stat(changelogPath); os.IsNotExist(err) { log.Printf("Package '%s' changelog not exist, do onboard process", packagePath) - + if specficPackageTitle == "" { specficPackageTitle = strings.Title(rpName) }