feat: mgmt plane release tool update (#16238)

* feat: change second generate cmd to version replacement, refine changelog file content set based on new changelog format

* feat: add refresh-v2 cmd

* fix: add package config support for muliti-package rp onboard

eng/tools/generator/autorest/model/changelog.go

@@ -81,7 +81,7 @@ func writeChangelogForPackage(r *report.Package) string {
 	}
 
 	// write additional changes
-	md.WriteHeader("New Content")
+	md.WriteHeader("Features Added")
 	for _, item := range getNewContents(r.AdditiveChanges) {
 		md.WriteListItem(item)
 	}

eng/tools/generator/cmd/root.go

@@ -14,6 +14,7 @@ import (
 	"github.com/Azure/azure-sdk-for-go/eng/tools/generator/cmd/release"
 	"github.com/Azure/azure-sdk-for-go/eng/tools/generator/cmd/template"
 	automation_v2 "github.com/Azure/azure-sdk-for-go/eng/tools/generator/cmd/v2/automation"
+	refresh_v2 "github.com/Azure/azure-sdk-for-go/eng/tools/generator/cmd/v2/refresh"
 	release_v2 "github.com/Azure/azure-sdk-for-go/eng/tools/generator/cmd/v2/release"
 	"github.com/Azure/azure-sdk-for-go/eng/tools/generator/common"
 	"github.com/spf13/cobra"
@@ -41,12 +42,13 @@ func Command() *cobra.Command {
 
 	rootCmd.AddCommand(
 		automation.Command(),
-		automation_v2.Command(),
-		release_v2.Command(),
 		issue.Command(),
 		template.Command(),
 		refresh.Command(),
 		release.Command(),
+		automation_v2.Command(),
+		release_v2.Command(),
+		refresh_v2.Command(),
 	)
 
 	return rootCmd

eng/tools/generator/cmd/template/templateCmd.go

@@ -59,26 +59,29 @@ func BindFlags(flagSet *pflag.FlagSet) {
 	flagSet.String("package-title", "", "Specifies the title of this package")
 	flagSet.String("commit", "", "Specifies the commit hash of azure-rest-api-specs")
 	flagSet.String("release-date", "", "Specifies the release date in changelog")
+	flagSet.String("package-config", "", "Additional config for package")
 }
 
 // ParseFlags parses the flags to a Flags struct
 func ParseFlags(flagSet *pflag.FlagSet) Flags {
 	return Flags{
-		SDKRoot:      flags.GetString(flagSet, "go-sdk-folder"),
-		TemplatePath: flags.GetString(flagSet, "template-path"),
-		PackageTitle: flags.GetString(flagSet, "package-title"),
-		Commit:       flags.GetString(flagSet, "commit"),
-		ReleaseDate:  flags.GetString(flagSet, "release-date"),
+		SDKRoot:       flags.GetString(flagSet, "go-sdk-folder"),
+		TemplatePath:  flags.GetString(flagSet, "template-path"),
+		PackageTitle:  flags.GetString(flagSet, "package-title"),
+		Commit:        flags.GetString(flagSet, "commit"),
+		ReleaseDate:   flags.GetString(flagSet, "release-date"),
+		PackageConfig: flags.GetString(flagSet, "package-config"),
 	}
 }
 
 // Flags ...
 type Flags struct {
-	SDKRoot      string
-	TemplatePath string
-	PackageTitle string
-	Commit       string
-	ReleaseDate  string
+	SDKRoot       string
+	TemplatePath  string
+	PackageTitle  string
+	Commit        string
+	ReleaseDate   string
+	PackageConfig string
 }
 
 // GeneratePackageByTemplate creates a new set of files based on the things in template directory
@@ -99,7 +102,7 @@ func GeneratePackageByTemplate(rpName, packageName string, flags Flags) error {
 	}
 
 	// build the replaceMap
-	buildReplaceMap(rpName, packageName, flags.PackageTitle, flags.Commit, flags.ReleaseDate)
+	buildReplaceMap(rpName, packageName, flags.PackageConfig, flags.PackageTitle, flags.Commit, flags.ReleaseDate)
 
 	// copy everything to destination directory
 	for _, file := range fileList {
@@ -123,11 +126,12 @@ func GeneratePackageByTemplate(rpName, packageName string, flags Flags) error {
 	return nil
 }
 
-func buildReplaceMap(rpName, packageName, packageTitle, commitID, releaseDate string) {
+func buildReplaceMap(rpName, packageName, packageConfig, packageTitle, commitID, releaseDate string) {
 	replaceMap = make(map[string]string)
 
 	replaceMap[RPNameKey] = rpName
 	replaceMap[PackageNameKey] = packageName
+	replaceMap[PackageConfigKey] = packageConfig
 	replaceMap[PackageTitleKey] = packageTitle
 	replaceMap[CommitIDKey] = commitID
 	if releaseDate == "" {
@@ -170,10 +174,11 @@ var (
 )
 
 const (
-	RPNameKey       = "{{rpName}}"
-	PackageNameKey  = "{{packageName}}"
-	PackageTitleKey = "{{PackageTitle}}"
-	CommitIDKey     = "{{commitID}}"
-	FilenameSuffix  = ".tpl"
-	ReleaseDate     = "{{releaseDate}}"
+	RPNameKey        = "{{rpName}}"
+	PackageNameKey   = "{{packageName}}"
+	PackageTitleKey  = "{{PackageTitle}}"
+	CommitIDKey      = "{{commitID}}"
+	FilenameSuffix   = ".tpl"
+	ReleaseDate      = "{{releaseDate}}"
+	PackageConfigKey = "{{packageConfig}}"
 )

eng/tools/generator/cmd/v2/common/fileProcessor.go

@@ -8,6 +8,7 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
+	"path"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -30,11 +31,19 @@ var (
 	v2BeginRegex             = regexp.MustCompile("^```\\s*yaml\\s*\\$\\(go\\)\\s*&&\\s*\\$\\((track2|v2)\\)")
 	v2EndRegex               = regexp.MustCompile("^\\s*```\\s*$")
 	newClientMethodNameRegex = regexp.MustCompile("^New.+Client$")
+	versionLineRegex         = regexp.MustCompile(`version\s*=\s*\".*v\d+\.\d+\.\d+\"`)
+	changelogVersionRegex    = regexp.MustCompile(`##\s*(?P<version>\d+\.\d+\.\d+)\s*\((\d{4}-\d{2}-\d{2}|Unreleased)\)`)
+	packageConfigRegex       = regexp.MustCompile(`\$\((package-.+)\)`)
 )
 
+type PackageInfo struct {
+	Name   string
+	Config string
+}
+
 // reads from readme.go.md, parses the `track2` section to get module and package name
-func ReadV2ModuleNameToGetNamespace(path string) (map[string][]string, error) {
-	result := make(map[string][]string)
+func ReadV2ModuleNameToGetNamespace(path string) (map[string][]PackageInfo, error) {
+	result := make(map[string][]PackageInfo)
 	log.Printf("Reading from readme.go.md '%s'...", path)
 	file, err := os.Open(path)
 	if err != nil {
@@ -79,7 +88,12 @@ func ReadV2ModuleNameToGetNamespace(path string) (map[string][]string, error) {
 				}
 				namespaceName := strings.TrimSuffix(strings.TrimSuffix(modules[3], "\n"), "\r")
 				log.Printf("RP: %s Package: %s", modules[2], namespaceName)
-				result[modules[2]] = append(result[modules[2]], namespaceName)
+				packageConfig := ""
+				matchResults := packageConfigRegex.FindAllStringSubmatch(lines[start[i]], -1)
+				for _, matchResult := range matchResults {
+					packageConfig = matchResult[1] + ": true"
+				}
+				result[modules[2]] = append(result[modules[2]], PackageInfo{Name: namespaceName, Config: packageConfig})
 			}
 		}
 	}
@@ -106,7 +120,7 @@ func CleanSDKGeneratedFiles(path string) error {
 	return nil
 }
 
-// replace repo commit with local path in autorest.md files
+// replace repo commit with local path in autorest.md file
 func ChangeConfigWithLocalPath(path, specPath, specRPName string) error {
 	log.Printf("Replacing repo commit with local path in autorest.md ...")
 	b, err := ioutil.ReadFile(path)
@@ -131,7 +145,7 @@ func ChangeConfigWithLocalPath(path, specPath, specRPName string) error {
 	return ioutil.WriteFile(path, []byte(strings.Join(lines, "\n")), 0644)
 }
 
-// replace repo URL and commit id in autorest.md files
+// replace repo URL and commit id in autorest.md file
 func ChangeConfigWithCommitID(path, repoURL, commitID, specRPName string) error {
 	log.Printf("Replacing repo URL and commit id in autorest.md ...")
 	b, err := ioutil.ReadFile(path)
@@ -151,25 +165,49 @@ func ChangeConfigWithCommitID(path, repoURL, commitID, specRPName string) error
 	return ioutil.WriteFile(path, []byte(strings.Join(lines, "\n")), 0644)
 }
 
-// get latest version according to `module-version: ` prefix in autorest.md file
+// get swagger rp folder name from autorest.md file
+func GetSpecRpName(packageRootPath string) (string, error) {
+	b, err := ioutil.ReadFile(path.Join(packageRootPath, "autorest.md"))
+	if err != nil {
+		return "", err
+	}
+
+	lines := strings.Split(string(b), "\n")
+	for _, line := range lines {
+		if strings.Contains(line, autorest_md_file_suffix) {
+			allParts := strings.Split(line, "/")
+			for i, part := range allParts {
+				if part == "specification" {
+					return allParts[i+1], nil
+				}
+			}
+		}
+	}
+	return "", fmt.Errorf("cannot get sepc rp name from config")
+}
+
+// get latest version from changelog file according to first line with: `## 0.2.1 (2021-11-22)`
 func GetLatestVersion(packageRootPath string) (*semver.Version, error) {
-	b, err := ioutil.ReadFile(filepath.Join(packageRootPath, "autorest.md"))
+	path := filepath.Join(packageRootPath, common.ChangelogFilename)
+	b, err := ioutil.ReadFile(path)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("cannot parse version from changelog")
 	}
 
 	lines := strings.Split(string(b), "\n")
 	for _, line := range lines {
-		if strings.HasPrefix(line, autorest_md_module_version_prefix) {
-			versionString := strings.TrimSuffix(strings.TrimSuffix(line[len(autorest_md_module_version_prefix):], "\n"), "\r")
-			return semver.NewVersion(versionString)
+		matchResults := changelogVersionRegex.FindAllStringSubmatch(line, -1)
+		for _, matchResult := range matchResults {
+			if matchResult[2] != "Unreleased" {
+				return semver.NewVersion(matchResult[1])
+			}
 		}
 	}
 
-	return nil, fmt.Errorf("cannot parse version from autorest.md")
+	return nil, fmt.Errorf("cannot parse version from changelog")
 }
 
-// replace version according to `module-version: ` prefix in autorest.md file
+// replace version: use `module-version: ` prefix to locate version in autorest.md file, use version = "v*.*.*" regrex to locate version in constants.go file
 func ReplaceVersion(packageRootPath string, newVersion string) error {
 	path := filepath.Join(packageRootPath, "autorest.md")
 	b, err := ioutil.ReadFile(path)
@@ -185,7 +223,17 @@ func ReplaceVersion(packageRootPath string, newVersion string) error {
 		}
 	}
 
-	return ioutil.WriteFile(path, []byte(strings.Join(lines, "\n")), 0644)
+	if err = ioutil.WriteFile(path, []byte(strings.Join(lines, "\n")), 0644); err != nil {
+		return err
+	}
+
+	path = filepath.Join(packageRootPath, sdk_generated_file_prefix+"constants.go")
+	if b, err = ioutil.ReadFile(path); err != nil {
+		return err
+	}
+	contents := versionLineRegex.ReplaceAllString(string(b), "version = \"v"+newVersion+"\"")
+
+	return ioutil.WriteFile(path, []byte(contents), 0644)
 }
 
 // calculate new version by changelog using semver package
@@ -226,13 +274,27 @@ func AddChangelogToFile(changelog *model.Changelog, version *semver.Version, pac
 	if err != nil {
 		return "", err
 	}
+
 	oldChangelog := string(b)
-	insertPos := strings.Index(oldChangelog, "##")
+	newChangelog := ""
+	matchResults := changelogVersionRegex.FindAllStringSubmatchIndex(oldChangelog, -1)
 	additionalChangelog := changelog.ToCompactMarkdown()
 	if releaseDate == "" {
 		releaseDate = time.Now().Format("2006-01-02")
 	}
-	newChangelog := oldChangelog[:insertPos] + "## " + version.String() + " (" + releaseDate + ")\n" + additionalChangelog + "\n\n" + oldChangelog[insertPos:]
+
+	for _, matchResult := range matchResults {
+		if oldChangelog[matchResult[4]:matchResult[5]] == "Unreleased" {
+			newChangelog = newChangelog + oldChangelog[0:matchResult[0]]
+		} else {
+			if newChangelog == "" {
+				newChangelog = newChangelog + oldChangelog[0:matchResult[0]]
+			}
+			newChangelog = newChangelog + "## " + version.String() + " (" + releaseDate + ")\r\n" + additionalChangelog + "\r\n\r\n" + oldChangelog[matchResult[0]:]
+			break
+		}
+	}
+
 	err = ioutil.WriteFile(path, []byte(newChangelog), 0644)
 	if err != nil {
 		return "", err

eng/tools/generator/cmd/v2/common/generation.go

@@ -39,6 +39,7 @@ type GenerateResult struct {
 type GenerateParam struct {
 	RPName              string
 	NamespaceName       string
+	NamespaceConfig     string
 	SpecficVersion      string
 	SpecficPackageTitle string
 	SpecRPName          string
@@ -62,14 +63,15 @@ func (ctx GenerateContext) GenerateForAutomation(readme, repo string) ([]Generat
 		}
 	}
 
-	for rpName, namespaceNames := range rpMap {
-		for _, namespaceName := range namespaceNames {
-			log.Printf("Process rp: %s, namespace: %s", rpName, namespaceName)
+	for rpName, packageInfos := range rpMap {
+		for _, packageInfo := range packageInfos {
+			log.Printf("Process rp: %s, namespace: %s", rpName, packageInfo.Name)
 			singleResult, err := ctx.GenerateForSingleRPNamespace(&GenerateParam{
 				RPName:              rpName,
-				NamespaceName:       namespaceName,
+				NamespaceName:       packageInfo.Name,
 				SpecRPName:          specRPName,
 				SkipGenerateExample: true,
+				NamespaceConfig:     packageInfo.Config,
 			})
 			if err != nil {
 				errors = append(errors, err)
@@ -97,10 +99,11 @@ func (ctx GenerateContext) GenerateForSingleRPNamespace(generateParam *GenerateP
 
 		log.Printf("Use template to generate new rp folder and basic package files...")
 		if err = template.GeneratePackageByTemplate(generateParam.RPName, generateParam.NamespaceName, template.Flags{
-			SDKRoot:      ctx.SDKPath,
-			TemplatePath: "eng/tools/generator/template/rpName/packageName",
-			PackageTitle: generateParam.SpecficPackageTitle,
-			Commit:       ctx.SpecCommitHash,
+			SDKRoot:       ctx.SDKPath,
+			TemplatePath:  "eng/tools/generator/template/rpName/packageName",
+			PackageTitle:  generateParam.SpecficPackageTitle,
+			Commit:        ctx.SpecCommitHash,
+			PackageConfig: generateParam.NamespaceConfig,
 		}); err != nil {
 			return nil, err
 		}
@@ -192,21 +195,11 @@ func (ctx GenerateContext) GenerateForSingleRPNamespace(generateParam *GenerateP
 			return nil, err
 		}
 
-		log.Printf("Remove all the files that start with `zz_generated_`...")
-		if err = CleanSDKGeneratedFiles(packagePath); err != nil {
-			return nil, err
-		}
-
-		log.Printf("Replace version in autorest.md...")
+		log.Printf("Replace version in autorest.md and constants...")
 		if err = ReplaceVersion(packagePath, version.String()); err != nil {
 			return nil, err
 		}
 
-		log.Printf("Run `go generate` to regenerate the code for new version...")
-		if err = ExecuteGoGenerate(packagePath); err != nil {
-			return nil, err
-		}
-
 		return &GenerateResult{
 			Version:        version.String(),
 			RPName:         generateParam.RPName,