Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add commit count and authors to the release notes #7982

Merged
merged 3 commits into from
Apr 28, 2021
Merged

Add commit count and authors to the release notes #7982

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c9f6eda
add a little more info to the release notes
systay Apr 28, 2021
9924c98
load gh user name instead of email
systay Apr 28, 2021
d07d5b9
make gh user names clickable
systay Apr 28, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
200 changes: 162 additions & 38 deletions go/tools/release-notes/release_notes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,15 @@ type (
Name string `json:"name"`
}

author struct {
Login string `json:"login"`
}

prInfo struct {
Labels []label `json:"labels"`
Number int `json:"number"`
Title string `json:"title"`
Author author `json:"author"`
}

prsByComponent = map[string][]prInfo
Expand Down Expand Up @@ -69,79 +74,178 @@ const (
{{- end }}
`

prefixType = "Type: "
prefixComponent = "Component: "
prefixType = "Type: "
prefixComponent = "Component: "
numberOfThreads = 10
lengthOfSingleSHA = 40
)

func loadMergedPRs(from, to string) ([]string, error) {
cmd := exec.Command("git", "log", "--oneline", fmt.Sprintf("%s..%s", from, to))
out, err := cmd.Output()
func loadMergedPRs(from, to string) (prs []string, authors []string, commitCount int, err error) {
// load the git log with "author \t title \t parents"
out, err := execCmd("git", "log", `--pretty=format:%ae%x09%s%x09%P%x09%h`, fmt.Sprintf("%s..%s", from, to))

if err != nil {
execErr := err.(*exec.ExitError)
return nil, fmt.Errorf("%s:\nstderr: %s\nstdout: %s", err.Error(), execErr.Stderr, out)
return
}

var prs []string
rgx := regexp.MustCompile(`Merge pull request #(\d+)`)
lines := strings.Split(string(out), "\n")
return parseGitLog(string(out))
}

func parseGitLog(s string) (prs []string, authorCommits []string, commitCount int, err error) {
rx := regexp.MustCompile(`(.+)\t(.+)\t(.+)\t(.+)`)
mergePR := regexp.MustCompile(`Merge pull request #(\d+)`)
authMap := map[string]string{} // here we will store email <-> gh user mappings
lines := strings.Split(s, "\n")
for _, line := range lines {
lineInfo := rgx.FindStringSubmatch(line)
if len(lineInfo) == 2 {
prs = append(prs, lineInfo[1])
lineInfo := rx.FindStringSubmatch(line)
if len(lineInfo) != 5 {
log.Fatalf("failed to parse the output from git log: %s", line)
}
authorEmail := lineInfo[1]
title := lineInfo[2]
parents := lineInfo[3]
sha := lineInfo[4]
merged := mergePR.FindStringSubmatch(title)
if len(merged) == 2 {
// this is a merged PR. remember the PR #
prs = append(prs, merged[1])
continue
}

if len(parents) > lengthOfSingleSHA {
// if we have two parents, it means this is a merge commit. we only count non-merge commits
continue
}
commitCount++
if _, exists := authMap[authorEmail]; !exists {
authMap[authorEmail] = sha
}
}

for _, author := range authMap {
authorCommits = append(authorCommits, author)
}

sort.Strings(prs)
return prs, nil
sort.Strings(authorCommits) // not really needed, but makes testing easier

return
}

func loadPRinfo(pr string) (prInfo, error) {
cmd := exec.Command("gh", "pr", "view", pr, "--json", "title,number,labels")
out, err := cmd.Output()
func execCmd(name string, arg ...string) ([]byte, error) {
out, err := exec.Command(name, arg...).Output()
if err != nil {
execErr, ok := err.(*exec.ExitError)
if ok {
return prInfo{}, fmt.Errorf("%s:\nstderr: %s\nstdout: %s", err.Error(), execErr.Stderr, out)
return nil, fmt.Errorf("%s:\nstderr: %s\nstdout: %s", err.Error(), execErr.Stderr, out)
}
if strings.Contains(err.Error(), " executable file not found in") {
return prInfo{}, fmt.Errorf("the command `gh` seems to be missing. Please install it from https://github.com/cli/cli")
return nil, fmt.Errorf("the command `gh` seems to be missing. Please install it from https://github.com/cli/cli")
}
return nil, err
}
return out, nil
}

func loadPRInfo(pr string) (prInfo, error) {
out, err := execCmd("gh", "pr", "view", pr, "--json", "title,number,labels,author")
if err != nil {
return prInfo{}, err
}
var prInfo prInfo
err = json.Unmarshal(out, &prInfo)
return prInfo, err
}

func loadAllPRs(prs []string) ([]prInfo, error) {
func loadAuthorInfo(sha string) (string, error) {
out, err := execCmd("gh", "api", "/repos/vitessio/vitess/commits/"+sha)
if err != nil {
return "", err
}
var prInfo prInfo
err = json.Unmarshal(out, &prInfo)
if err != nil {
return "", err
}
return prInfo.Author.Login, nil
}

type req struct {
isPR bool
key string
}

func loadAllPRs(prs, authorCommits []string) ([]prInfo, []string, error) {
errChan := make(chan error)
wgDone := make(chan bool)
prChan := make(chan string, len(prs))
prChan := make(chan req, len(prs)+len(authorCommits))
// fill the work queue
for _, s := range prs {
prChan <- s
prChan <- req{isPR: true, key: s}
}
for _, s := range authorCommits {
prChan <- req{isPR: false, key: s}
}
close(prChan)

var prInfos []prInfo
var authors []string
fmt.Printf("Found %d merged PRs. Loading PR info", len(prs))
wg := sync.WaitGroup{}
mu := sync.Mutex{}
for i := 0; i < 10; i++ {

shouldLoad := func(in string) bool {
if in == "" {
return false
}
mu.Lock()
defer mu.Unlock()

for _, existing := range authors {
if existing == in {
return false
}
}
return true
}
addAuthor := func(in string) {
mu.Lock()
defer mu.Unlock()
authors = append(authors, in)
}
addPR := func(in prInfo) {
mu.Lock()
defer mu.Unlock()
prInfos = append(prInfos, in)
}

for i := 0; i < numberOfThreads; i++ {
wg.Add(1)
go func() {
// load meta data about PRs
defer wg.Done()

for b := range prChan {
fmt.Print(".")
prInfo, err := loadPRinfo(b)

if b.isPR {
prInfo, err := loadPRInfo(b.key)
if err != nil {
errChan <- err
break
}
addPR(prInfo)
continue
}
author, err := loadAuthorInfo(b.key)
if err != nil {
errChan <- err
break
}
mu.Lock()
prInfos = append(prInfos, prInfo)
mu.Unlock()
if shouldLoad(author) {
addAuthor(author)
}

}
}()
}
Expand All @@ -161,7 +265,10 @@ func loadAllPRs(prs []string) ([]prInfo, error) {
}

fmt.Println()
return prInfos, err

sort.Strings(authors)

return prInfos, authors, err
}

func groupPRs(prInfos []prInfo) prsByType {
Expand Down Expand Up @@ -226,15 +333,15 @@ func createSortedPrTypeSlice(prPerType prsByType) []sortedPRType {
return data
}

func writePrInfos(fileout string, prPerType prsByType) (err error) {
writeTo := os.Stdout
if fileout != "" {
writeTo, err = os.OpenFile(fileout, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
if err != nil {
return err
}
func getOutput(fileout string) (*os.File, error) {
if fileout == "" {
return os.Stdout, nil
}

return os.OpenFile(fileout, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
}

func writePrInfos(writeTo *os.File, prPerType prsByType) (err error) {
data := createSortedPrTypeSlice(prPerType)

t := template.Must(template.New("markdownTemplate").Parse(markdownTemplate))
Expand All @@ -252,19 +359,36 @@ func main() {

flag.Parse()

prs, err := loadMergedPRs(*from, *to)
prs, authorCommits, commits, err := loadMergedPRs(*from, *to)
if err != nil {
log.Fatal(err)
}

prInfos, err := loadAllPRs(prs)
prInfos, authors, err := loadAllPRs(prs, authorCommits)
if err != nil {
log.Fatal(err)
}

prPerType := groupPRs(prInfos)
out, err := getOutput(*fileout)
if err != nil {
log.Fatal(err)
}
defer func() {
_ = out.Close()
}()

err = writePrInfos(out, prPerType)
if err != nil {
log.Fatal(err)
}

_, err = out.WriteString(fmt.Sprintf("\n\nThe release includes %d commits (excluding merges)\n", commits))
if err != nil {
log.Fatal(err)
}

err = writePrInfos(*fileout, prPerType)
_, err = out.WriteString(fmt.Sprintf("Thanks to all our contributors: @%s\n", strings.Join(authors, ", @")))
if err != nil {
log.Fatal(err)
}
Expand Down
51 changes: 51 additions & 0 deletions go/tools/release-notes/release_notes_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ package main
import (
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"vitess.io/vitess/go/test/utils"
)

Expand Down Expand Up @@ -53,3 +56,51 @@ func Test_groupPRs(t *testing.T) {
})
}
}

func TestParseGitLogOutput(t *testing.T) {
in := `harshTEST@planetscale.com Merge pull request #7968 from planetscale/bump_java_snapshot_v11 7e8ebbb5b79b65d2d45fd6c838efb51bdafc7c0b 195a09df191d3e86a32ebcc7a1f1dde168fe819e 168fe819e
deeptTEST@planetscale.com Merge pull request #7970 from planetscale/vttestserver-default-charset 887be6914690b6d106aba001c72deea80a4d8dab ff8c750eda4b30787e772547a451ed1f50931150 f50931150
deeptTEST@planetscale.com Merge pull request #7943 from planetscale/fix-mysql80-container-image 01fb7e55ab92df7c3f300b85976fdf3fd5bd35b3 3cc94a10752014c9ce311d88af9e1aa18e7fa2d8 18e7fa2d8
57520317+rohit-nayak-TEST@users.noreply.github.com Merge pull request #7831 from planetscale/rn-vr-log2 37c09d3be83922a8ef936fbc028a5031f96b7dbf f57350c3ea1720496e5f1cec35d58f069e4df515 69e4df515
TEST@planetscale.com docker/vttestserver/run.sh: Add $CHARSET environment variable 482a7008117ee3215663aeb33cad981e5242a88a e5242a88a
rohTEST@planetscale.com Add ability to select from vreplication_log in VReplicationExec 427cac89cd6b143d3a1928ee682b3a9538709da5 538709da5
rohTEST@planetscale.com Use withDDL for vreplication log queries 4a1ab946e3628ba8ef610ea4a158186a5fdd17ba a5fdd17ba
rohTEST@planetscale.com Add license file. Minor refactor fa9de690ce0d27a781befbc1866aca5cd447798f cd447798f
rohTEST@planetscale.com Added comments and refactored tests b6d39acb08939ba56e9e9587f34f3b8bcdcdc504 bcdcdc504
rohTEST@planetscale.com Add logs for start and end of the copy phase 1cf72866ddfbd554700d6c9e32b9835ebb3b444c ebb3b444c
rohTEST@planetscale.com Fix test 0992d39c6d473b548679d012cfa5a889ffa448ef 9ffa448ef
rohTEST@planetscale.com Add test for vreplication log and fix string conversion bug b616143b14b75e7c23042c2eef4f6b27a275b0f7 7a275b0f7
rohTEST@planetscale.com Ignore queries related to _vt.vreplication_log in tests e6926932c14da9a2213be246bc2de5f011668551 011668551
rohTEST@planetscale.com Create log table. Util functions to insert logs. Insert logs in VReplicationExec and setMessage/State 37c09d3be83922a8ef936fbc028a5031f96b7dbf 1f96b7dbf
harshTEST@planetscale.com Merge pull request #7951 from vmg/vmg/vr-client-perf 7794c62651066970e1176181cb7000d385d0b327 172fac7dec8b11937a4efb26ebf4bedf1771f189 f1771f189
alkin.tezuysTEST@gmail.com java: Bump SNAPSHOT version to 11.0.0-SNAPSHOT after Vitess release v10 7794c62651066970e1176181cb7000d385d0b327 385d0b327
alkin.tezuysTEST@gmail.com Merge pull request #7964 from planetscale/10_0_RC1_release_notes 31d84d6ce8e233a053794ad0ffe5168d34d04450 b020dc71f5c7dc663d814563f1b6c97340f4411f 340f4411f
vTEST@strn.cat vstreamer: fix docs e7bf329da0029414c3b18e18e5cb2226b9a731a2 6b9a731a2
amasTEST@slack-corp.com [workflow] extract migration targets from wrangler (#7934) 8bd5a7cb093369b50a0926bfa3a112b3b744e782 3b744e782
alkin.tezuysTEST@gmail.com More spacing issues fixed 7509d47ba785e7a39b8726dc80f93955953ab98d 5953ab98d
alkin.tezuysTEST@gmail.com Minor spacing fixes d31362e76ac69fb2bc4083e22e7c87683099fecd 83099fecd
alkin.tezuysTEST@gmail.com Update 10_0_0_release_notes.md a7034bdf5d454a47738335ed2afc75f72bdbcf37 72bdbcf37
alkin.tezuysTEST@gmail.com v10 GA Release Notes ad37320b2637620ee36d44d163399ecc2c1eea6c c2c1eea6c
andrTEST@planetscale.com Merge pull request #7912 from planetscale/show-databases-like 7e13d4bccca0325ca07a488334e77c4f2f964f6b 95eceb17d10c62d56f2e94e5478afb5a1b63e1c2 a1b63e1c2
andrTEST@planetscale.com Merge pull request #7629 from planetscale/gen4-table-aliases 2e1b1e9322a6bfcfe792cca341b0d52860d3c66e 7ad14e3f3d26cb1780cdbf9c22029740e5aebde4 0e5aebde4
andrTEST@planetscale.com Merge remote-tracking branch 'upstream/master' into show-databases-like 6b3ee1c31a939fc6628515f00087baa3e1e8acf7 2e1b1e9322a6bfcfe792cca341b0d52860d3c66e 860d3c66e
2607934+shlomi-noaTEST@users.noreply.github.com Merge pull request #7959 from Hellcatlk/master 6c826115937d28ef83f05a1f0d54db0fcb814db4 cdab3040aaaa11c51e291d6b1a7af6fadd83dedf add83dedf
zouy.fnTEST@cn.fujitsu.com Fix a gofmt warning 08038850a258d6de250cf9d864d6118616f5562c 616f5562c
vTEST@strn.cat mysql: allow reusing row storage when reading from a stream a2850bbf41100618cb1192067b16585ba7c6b0c7 ba7c6b0c7
vTEST@strn.cat throttle: do not check for time constantly e0b90daebe9e6b98d969934a24899b41d25e3a68 1d25e3a68
andrTEST@planetscale.com fix compilation error 18036f5fb5f58523dbf50726beb741cedac2baf8 edac2baf8
andrTEST@planetscale.com better code comment c173c945cf0e75e8649e6fa621509b5fb4ebd6c9 fb4ebd6c9
vTEST@strn.cat conn: do not let header escape to the heap d31fb23d8cb9463810ed9fc132df4060a6812f6e 0a6812f6e
vTEST@strn.cat vstreamer: do not allocate when filtering rows dafc1cb729d7be7dff2c05bd05a926005eb9a044 05eb9a044
vTEST@strn.cat vstreamer: do not allocate when converting rows c5cd3067aeb9d952a2f45084c37634267e4f9062 67e4f9062
andrTEST@planetscale.com Merge remote-tracking branch 'upstream/master' into gen4-table-aliases 8c01827ed8b748240f213d9476ee162306ab01eb b1f9000ddd166d49adda6581e7ca9e0aca10c252 aca10c252
aquarapTEST@gmail.com Fix mysql80 docker build with dep. a28591577b8d432b9c5d78abf59ad494a0a943b0 4a0a943b0
TEST@planetscale.com Revert "docker/lite/install_dependencies.sh: Upgrade MySQL 8 to 8.0.24" 7858ff46545cff749b3663c92ae90ef27a5dfbc2 27a5dfbc2
TEST@planetscale.com docker/lite/install_dependencies.sh: Upgrade MySQL 8 to 8.0.24 c91d46782933292941a846fef2590ff1a6fa193f a6fa193f`

prs, authorCommits, count, err := parseGitLog(in)
require.NoError(t, err)
assert.Equal(t, []string{"7629", "7831", "7912", "7943", "7951", "7959", "7964", "7968", "7970"}, prs)
assert.Equal(t, []string{"385d0b327", "3b744e782", "4a0a943b0", "538709da5", "616f5562c", "6b9a731a2", "e5242a88a", "edac2baf8"}, authorCommits)
assert.Equal(t, 28, count)
}