diff --git a/go/tools/release-notes/release_notes.go b/go/tools/release-notes/release_notes.go index b366c9d0a24..376e5eab2f3 100644 --- a/go/tools/release-notes/release_notes.go +++ b/go/tools/release-notes/release_notes.go @@ -35,10 +35,15 @@ type ( Name string `json:"name"` } + author struct { + Login string `json:"login"` + } + prInfo struct { Labels []label `json:"labels"` Number int `json:"number"` Title string `json:"title"` + Author author `json:"author"` } prsByComponent = map[string][]prInfo @@ -69,43 +74,82 @@ const ( {{- end }} ` - prefixType = "Type: " - prefixComponent = "Component: " + prefixType = "Type: " + prefixComponent = "Component: " + numberOfThreads = 10 + lengthOfSingleSHA = 40 ) -func loadMergedPRs(from, to string) ([]string, error) { - cmd := exec.Command("git", "log", "--oneline", fmt.Sprintf("%s..%s", from, to)) - out, err := cmd.Output() +func loadMergedPRs(from, to string) (prs []string, authors []string, commitCount int, err error) { + // load the git log with "author \t title \t parents" + out, err := execCmd("git", "log", `--pretty=format:%ae%x09%s%x09%P%x09%h`, fmt.Sprintf("%s..%s", from, to)) + if err != nil { - execErr := err.(*exec.ExitError) - return nil, fmt.Errorf("%s:\nstderr: %s\nstdout: %s", err.Error(), execErr.Stderr, out) + return } - var prs []string - rgx := regexp.MustCompile(`Merge pull request #(\d+)`) - lines := strings.Split(string(out), "\n") + return parseGitLog(string(out)) +} + +func parseGitLog(s string) (prs []string, authorCommits []string, commitCount int, err error) { + rx := regexp.MustCompile(`(.+)\t(.+)\t(.+)\t(.+)`) + mergePR := regexp.MustCompile(`Merge pull request #(\d+)`) + authMap := map[string]string{} // here we will store email <-> gh user mappings + lines := strings.Split(s, "\n") for _, line := range lines { - lineInfo := rgx.FindStringSubmatch(line) - if len(lineInfo) == 2 { - prs = append(prs, lineInfo[1]) + lineInfo := rx.FindStringSubmatch(line) + if len(lineInfo) != 5 { + log.Fatalf("failed to parse the output from git log: %s", line) + } + authorEmail := lineInfo[1] + title := lineInfo[2] + parents := lineInfo[3] + sha := lineInfo[4] + merged := mergePR.FindStringSubmatch(title) + if len(merged) == 2 { + // this is a merged PR. remember the PR # + prs = append(prs, merged[1]) + continue + } + + if len(parents) > lengthOfSingleSHA { + // if we have two parents, it means this is a merge commit. we only count non-merge commits + continue } + commitCount++ + if _, exists := authMap[authorEmail]; !exists { + authMap[authorEmail] = sha + } + } + + for _, author := range authMap { + authorCommits = append(authorCommits, author) } sort.Strings(prs) - return prs, nil + sort.Strings(authorCommits) // not really needed, but makes testing easier + + return } -func loadPRinfo(pr string) (prInfo, error) { - cmd := exec.Command("gh", "pr", "view", pr, "--json", "title,number,labels") - out, err := cmd.Output() +func execCmd(name string, arg ...string) ([]byte, error) { + out, err := exec.Command(name, arg...).Output() if err != nil { execErr, ok := err.(*exec.ExitError) if ok { - return prInfo{}, fmt.Errorf("%s:\nstderr: %s\nstdout: %s", err.Error(), execErr.Stderr, out) + return nil, fmt.Errorf("%s:\nstderr: %s\nstdout: %s", err.Error(), execErr.Stderr, out) } if strings.Contains(err.Error(), " executable file not found in") { - return prInfo{}, fmt.Errorf("the command `gh` seems to be missing. Please install it from https://github.com/cli/cli") + return nil, fmt.Errorf("the command `gh` seems to be missing. Please install it from https://github.com/cli/cli") } + return nil, err + } + return out, nil +} + +func loadPRInfo(pr string) (prInfo, error) { + out, err := execCmd("gh", "pr", "view", pr, "--json", "title,number,labels,author") + if err != nil { return prInfo{}, err } var prInfo prInfo @@ -113,35 +157,95 @@ func loadPRinfo(pr string) (prInfo, error) { return prInfo, err } -func loadAllPRs(prs []string) ([]prInfo, error) { +func loadAuthorInfo(sha string) (string, error) { + out, err := execCmd("gh", "api", "/repos/vitessio/vitess/commits/"+sha) + if err != nil { + return "", err + } + var prInfo prInfo + err = json.Unmarshal(out, &prInfo) + if err != nil { + return "", err + } + return prInfo.Author.Login, nil +} + +type req struct { + isPR bool + key string +} + +func loadAllPRs(prs, authorCommits []string) ([]prInfo, []string, error) { errChan := make(chan error) wgDone := make(chan bool) - prChan := make(chan string, len(prs)) + prChan := make(chan req, len(prs)+len(authorCommits)) // fill the work queue for _, s := range prs { - prChan <- s + prChan <- req{isPR: true, key: s} + } + for _, s := range authorCommits { + prChan <- req{isPR: false, key: s} } close(prChan) var prInfos []prInfo + var authors []string fmt.Printf("Found %d merged PRs. Loading PR info", len(prs)) wg := sync.WaitGroup{} mu := sync.Mutex{} - for i := 0; i < 10; i++ { + + shouldLoad := func(in string) bool { + if in == "" { + return false + } + mu.Lock() + defer mu.Unlock() + + for _, existing := range authors { + if existing == in { + return false + } + } + return true + } + addAuthor := func(in string) { + mu.Lock() + defer mu.Unlock() + authors = append(authors, in) + } + addPR := func(in prInfo) { + mu.Lock() + defer mu.Unlock() + prInfos = append(prInfos, in) + } + + for i := 0; i < numberOfThreads; i++ { wg.Add(1) go func() { // load meta data about PRs defer wg.Done() + for b := range prChan { fmt.Print(".") - prInfo, err := loadPRinfo(b) + + if b.isPR { + prInfo, err := loadPRInfo(b.key) + if err != nil { + errChan <- err + break + } + addPR(prInfo) + continue + } + author, err := loadAuthorInfo(b.key) if err != nil { errChan <- err break } - mu.Lock() - prInfos = append(prInfos, prInfo) - mu.Unlock() + if shouldLoad(author) { + addAuthor(author) + } + } }() } @@ -161,7 +265,10 @@ func loadAllPRs(prs []string) ([]prInfo, error) { } fmt.Println() - return prInfos, err + + sort.Strings(authors) + + return prInfos, authors, err } func groupPRs(prInfos []prInfo) prsByType { @@ -226,15 +333,15 @@ func createSortedPrTypeSlice(prPerType prsByType) []sortedPRType { return data } -func writePrInfos(fileout string, prPerType prsByType) (err error) { - writeTo := os.Stdout - if fileout != "" { - writeTo, err = os.OpenFile(fileout, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) - if err != nil { - return err - } +func getOutput(fileout string) (*os.File, error) { + if fileout == "" { + return os.Stdout, nil } + return os.OpenFile(fileout, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) +} + +func writePrInfos(writeTo *os.File, prPerType prsByType) (err error) { data := createSortedPrTypeSlice(prPerType) t := template.Must(template.New("markdownTemplate").Parse(markdownTemplate)) @@ -252,19 +359,36 @@ func main() { flag.Parse() - prs, err := loadMergedPRs(*from, *to) + prs, authorCommits, commits, err := loadMergedPRs(*from, *to) if err != nil { log.Fatal(err) } - prInfos, err := loadAllPRs(prs) + prInfos, authors, err := loadAllPRs(prs, authorCommits) if err != nil { log.Fatal(err) } prPerType := groupPRs(prInfos) + out, err := getOutput(*fileout) + if err != nil { + log.Fatal(err) + } + defer func() { + _ = out.Close() + }() + + err = writePrInfos(out, prPerType) + if err != nil { + log.Fatal(err) + } + + _, err = out.WriteString(fmt.Sprintf("\n\nThe release includes %d commits (excluding merges)\n", commits)) + if err != nil { + log.Fatal(err) + } - err = writePrInfos(*fileout, prPerType) + _, err = out.WriteString(fmt.Sprintf("Thanks to all our contributors: @%s\n", strings.Join(authors, ", @"))) if err != nil { log.Fatal(err) } diff --git a/go/tools/release-notes/release_notes_test.go b/go/tools/release-notes/release_notes_test.go index a98b0415025..8b4a2299336 100644 --- a/go/tools/release-notes/release_notes_test.go +++ b/go/tools/release-notes/release_notes_test.go @@ -19,6 +19,9 @@ package main import ( "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "vitess.io/vitess/go/test/utils" ) @@ -53,3 +56,51 @@ func Test_groupPRs(t *testing.T) { }) } } + +func TestParseGitLogOutput(t *testing.T) { + in := `harshTEST@planetscale.com Merge pull request #7968 from planetscale/bump_java_snapshot_v11 7e8ebbb5b79b65d2d45fd6c838efb51bdafc7c0b 195a09df191d3e86a32ebcc7a1f1dde168fe819e 168fe819e +deeptTEST@planetscale.com Merge pull request #7970 from planetscale/vttestserver-default-charset 887be6914690b6d106aba001c72deea80a4d8dab ff8c750eda4b30787e772547a451ed1f50931150 f50931150 +deeptTEST@planetscale.com Merge pull request #7943 from planetscale/fix-mysql80-container-image 01fb7e55ab92df7c3f300b85976fdf3fd5bd35b3 3cc94a10752014c9ce311d88af9e1aa18e7fa2d8 18e7fa2d8 +57520317+rohit-nayak-TEST@users.noreply.github.com Merge pull request #7831 from planetscale/rn-vr-log2 37c09d3be83922a8ef936fbc028a5031f96b7dbf f57350c3ea1720496e5f1cec35d58f069e4df515 69e4df515 +TEST@planetscale.com docker/vttestserver/run.sh: Add $CHARSET environment variable 482a7008117ee3215663aeb33cad981e5242a88a e5242a88a +rohTEST@planetscale.com Add ability to select from vreplication_log in VReplicationExec 427cac89cd6b143d3a1928ee682b3a9538709da5 538709da5 +rohTEST@planetscale.com Use withDDL for vreplication log queries 4a1ab946e3628ba8ef610ea4a158186a5fdd17ba a5fdd17ba +rohTEST@planetscale.com Add license file. Minor refactor fa9de690ce0d27a781befbc1866aca5cd447798f cd447798f +rohTEST@planetscale.com Added comments and refactored tests b6d39acb08939ba56e9e9587f34f3b8bcdcdc504 bcdcdc504 +rohTEST@planetscale.com Add logs for start and end of the copy phase 1cf72866ddfbd554700d6c9e32b9835ebb3b444c ebb3b444c +rohTEST@planetscale.com Fix test 0992d39c6d473b548679d012cfa5a889ffa448ef 9ffa448ef +rohTEST@planetscale.com Add test for vreplication log and fix string conversion bug b616143b14b75e7c23042c2eef4f6b27a275b0f7 7a275b0f7 +rohTEST@planetscale.com Ignore queries related to _vt.vreplication_log in tests e6926932c14da9a2213be246bc2de5f011668551 011668551 +rohTEST@planetscale.com Create log table. Util functions to insert logs. Insert logs in VReplicationExec and setMessage/State 37c09d3be83922a8ef936fbc028a5031f96b7dbf 1f96b7dbf +harshTEST@planetscale.com Merge pull request #7951 from vmg/vmg/vr-client-perf 7794c62651066970e1176181cb7000d385d0b327 172fac7dec8b11937a4efb26ebf4bedf1771f189 f1771f189 +alkin.tezuysTEST@gmail.com java: Bump SNAPSHOT version to 11.0.0-SNAPSHOT after Vitess release v10 7794c62651066970e1176181cb7000d385d0b327 385d0b327 +alkin.tezuysTEST@gmail.com Merge pull request #7964 from planetscale/10_0_RC1_release_notes 31d84d6ce8e233a053794ad0ffe5168d34d04450 b020dc71f5c7dc663d814563f1b6c97340f4411f 340f4411f +vTEST@strn.cat vstreamer: fix docs e7bf329da0029414c3b18e18e5cb2226b9a731a2 6b9a731a2 +amasTEST@slack-corp.com [workflow] extract migration targets from wrangler (#7934) 8bd5a7cb093369b50a0926bfa3a112b3b744e782 3b744e782 +alkin.tezuysTEST@gmail.com More spacing issues fixed 7509d47ba785e7a39b8726dc80f93955953ab98d 5953ab98d +alkin.tezuysTEST@gmail.com Minor spacing fixes d31362e76ac69fb2bc4083e22e7c87683099fecd 83099fecd +alkin.tezuysTEST@gmail.com Update 10_0_0_release_notes.md a7034bdf5d454a47738335ed2afc75f72bdbcf37 72bdbcf37 +alkin.tezuysTEST@gmail.com v10 GA Release Notes ad37320b2637620ee36d44d163399ecc2c1eea6c c2c1eea6c +andrTEST@planetscale.com Merge pull request #7912 from planetscale/show-databases-like 7e13d4bccca0325ca07a488334e77c4f2f964f6b 95eceb17d10c62d56f2e94e5478afb5a1b63e1c2 a1b63e1c2 +andrTEST@planetscale.com Merge pull request #7629 from planetscale/gen4-table-aliases 2e1b1e9322a6bfcfe792cca341b0d52860d3c66e 7ad14e3f3d26cb1780cdbf9c22029740e5aebde4 0e5aebde4 +andrTEST@planetscale.com Merge remote-tracking branch 'upstream/master' into show-databases-like 6b3ee1c31a939fc6628515f00087baa3e1e8acf7 2e1b1e9322a6bfcfe792cca341b0d52860d3c66e 860d3c66e +2607934+shlomi-noaTEST@users.noreply.github.com Merge pull request #7959 from Hellcatlk/master 6c826115937d28ef83f05a1f0d54db0fcb814db4 cdab3040aaaa11c51e291d6b1a7af6fadd83dedf add83dedf +zouy.fnTEST@cn.fujitsu.com Fix a gofmt warning 08038850a258d6de250cf9d864d6118616f5562c 616f5562c +vTEST@strn.cat mysql: allow reusing row storage when reading from a stream a2850bbf41100618cb1192067b16585ba7c6b0c7 ba7c6b0c7 +vTEST@strn.cat throttle: do not check for time constantly e0b90daebe9e6b98d969934a24899b41d25e3a68 1d25e3a68 +andrTEST@planetscale.com fix compilation error 18036f5fb5f58523dbf50726beb741cedac2baf8 edac2baf8 +andrTEST@planetscale.com better code comment c173c945cf0e75e8649e6fa621509b5fb4ebd6c9 fb4ebd6c9 +vTEST@strn.cat conn: do not let header escape to the heap d31fb23d8cb9463810ed9fc132df4060a6812f6e 0a6812f6e +vTEST@strn.cat vstreamer: do not allocate when filtering rows dafc1cb729d7be7dff2c05bd05a926005eb9a044 05eb9a044 +vTEST@strn.cat vstreamer: do not allocate when converting rows c5cd3067aeb9d952a2f45084c37634267e4f9062 67e4f9062 +andrTEST@planetscale.com Merge remote-tracking branch 'upstream/master' into gen4-table-aliases 8c01827ed8b748240f213d9476ee162306ab01eb b1f9000ddd166d49adda6581e7ca9e0aca10c252 aca10c252 +aquarapTEST@gmail.com Fix mysql80 docker build with dep. a28591577b8d432b9c5d78abf59ad494a0a943b0 4a0a943b0 +TEST@planetscale.com Revert "docker/lite/install_dependencies.sh: Upgrade MySQL 8 to 8.0.24" 7858ff46545cff749b3663c92ae90ef27a5dfbc2 27a5dfbc2 +TEST@planetscale.com docker/lite/install_dependencies.sh: Upgrade MySQL 8 to 8.0.24 c91d46782933292941a846fef2590ff1a6fa193f a6fa193f` + + prs, authorCommits, count, err := parseGitLog(in) + require.NoError(t, err) + assert.Equal(t, []string{"7629", "7831", "7912", "7943", "7951", "7959", "7964", "7968", "7970"}, prs) + assert.Equal(t, []string{"385d0b327", "3b744e782", "4a0a943b0", "538709da5", "616f5562c", "6b9a731a2", "e5242a88a", "edac2baf8"}, authorCommits) + assert.Equal(t, 28, count) +}