From 8fa3bbc42450fe34cc0cee3de566b17fa131d1c6 Mon Sep 17 00:00:00 2001 From: GiteaBot Date: Sun, 20 Jun 2021 00:11:36 +0000 Subject: [PATCH 01/10] [skip ci] Updated translations via Crowdin --- options/locale/locale_de-DE.ini | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/options/locale/locale_de-DE.ini b/options/locale/locale_de-DE.ini index 0de4f5b5323e1..8528aa2cffdc0 100644 --- a/options/locale/locale_de-DE.ini +++ b/options/locale/locale_de-DE.ini @@ -91,8 +91,10 @@ loading=Laden… step1=Schritt 1: step2=Schritt 2: +error=Fehler error404=Die Seite, die du gerade versuchst aufzurufen, existiert entweder nicht oder du bist nicht berechtigt, diese anzusehen. +never=Niemals [error] occurred=Ein Fehler ist aufgetreten @@ -724,6 +726,7 @@ mirror_prune_desc=Entferne veraltete remote-tracking Referenzen mirror_interval=Spiegel-Intervall (gültige Zeiteinheiten sind 'h', 'm', 's'). 0 schaltet die automatische Synchronisierung aus. mirror_interval_invalid=Das Spiegel-Intervall ist ungültig. mirror_address=Klonen via URL +mirror_address_desc=Gib alle erforderlichen Anmeldedaten im Abschnitt "Authentifizierung" ein. mirror_address_url_invalid=Die angegebene URL ist ungültig. Achte darauf, alle URL-Komponenten korrekt zu maskieren. mirror_address_protocol_invalid=Die angegebene URL ist ungültig. Nur Pfade beginnend mit http(s):// oder git:// können gespiegelt werden. mirror_lfs=Großdatei-Speicher (LFS) @@ -731,6 +734,9 @@ mirror_lfs_desc=Mirroring von LFS-Dateien aktivieren. mirror_lfs_endpoint=LFS-Endpunkt mirror_lfs_endpoint_desc=Sync wird versuchen, die Klon-URL zu verwenden, um den LFS-Server zu bestimmen. Du kannst auch einen eigenen Endpunkt angeben, wenn die LFS-Dateien woanders gespeichert werden. mirror_last_synced=Zuletzt synchronisiert +mirror_password_placeholder=(unverändert) +mirror_password_blank_placeholder=(Nicht gesetzt) +mirror_password_help=Ändere den Benutzernamen, um ein gespeichertes Passwort zu löschen. watchers=Beobachter stargazers=Favorisiert von forks=Forks @@ -783,6 +789,7 @@ form.reach_limit_of_creation_n=Du hast bereits dein Limit von %d Repositories er form.name_reserved=Der Repository-Name „%s“ ist reserviert. form.name_pattern_not_allowed='%s' ist nicht erlaubt für Repository-Namen. +need_auth=Authentifizierung migrate_options=Migrationsoptionen migrate_service=Migrationsdienst migrate_options_mirror_helper=Dieses Repository wird ein Mirror sein @@ -816,11 +823,17 @@ migrated_from_fake=Migriert von %[1]s migrate.migrate=Migrieren von %s migrate.migrating=Migriere von %s ... migrate.migrating_failed=Migrieren von %s fehlgeschlagen. +migrate.migrating_failed.error=Fehler: %s migrate.github.description=Migriere Daten von Github.com oder Github Enterprise. migrate.git.description=Migriere oder spiegele git-Daten von Git-Services migrate.gitlab.description=Migriere Daten von GitLab.com oder einem selbst gehostetem gitlab Server. migrate.gitea.description=Migriere Daten von Gitea.com oder einem selbst gehostetem Gitea Server. migrate.gogs.description=Migriere Daten von notabug.org oder einem anderen, selbst gehosteten Gogs Server. +migrate.migrating_milestones=Meilensteine werden migriert +migrate.migrating_labels=Labels werden migriert +migrate.migrating_releases=Releases werden migriert +migrate.migrating_issues=Issues werden migriert +migrate.migrating_pulls=Pull Requests werden migriert mirror_from=Mirror von forked_from=geforkt von @@ -1315,6 +1328,9 @@ pulls.is_closed=Der Pull-Request wurde geschlossen. pulls.has_merged=Der Pull-Request wurde gemergt. pulls.title_wip_desc=`Beginne den Titel mit %s um zu verhindern, dass der Pull Request versehentlich gemergt wird.` pulls.cannot_merge_work_in_progress=Dieser Pull Request ist als Work in Progress markiert. +pulls.still_in_progress=Noch in Bearbeitung? +pulls.add_prefix=%s Präfix hinzufügen +pulls.remove_prefix=%s Präfix entfernen pulls.data_broken=Dieser Pull-Requests ist kaputt, da Fork-Informationen gelöscht wurden. pulls.files_conflicted=Dieser Pull-Request hat Änderungen, die im Widerspruch zum Ziel-Branch stehen. pulls.is_checking=Die Konfliktprüfung läuft noch. Bitte aktualisiere die Seite in wenigen Augenblicken. @@ -1540,6 +1556,11 @@ settings.hooks=Webhooks settings.githooks=Git-Hooks settings.basic_settings=Grundeinstellungen settings.mirror_settings=Mirror-Einstellungen +settings.mirror_settings.direction=Richtung +settings.mirror_settings.direction.pull=Pull +settings.mirror_settings.direction.push=Push +settings.mirror_settings.last_update=Letzte Aktualisierung +settings.mirror_settings.push_mirror.none=Keine Push-Mirrors konfiguriert settings.sync_mirror=Jetzt synchronisieren settings.mirror_sync_in_progress=Mirror-Synchronisierung wird zurzeit ausgeführt. Komm in ein paar Minuten zurück. settings.email_notifications.enable=E-Mail Benachrichtigungen aktivieren From 23358bc55de67be132e3858a5d40f25dbdd0a769 Mon Sep 17 00:00:00 2001 From: zeripath Date: Sun, 20 Jun 2021 23:00:46 +0100 Subject: [PATCH 02/10] Use git log name-status in get last commit (#16059) * Improve get last commit using git log --name-status git log --name-status -c provides information about the diff between a commit and its parents. Using this and adjusting the algorithm to use the first change to a path allows for a much faster generation of commit info. There is a subtle change in the results generated but this will cause the results to more closely match those from elsewhere. Signed-off-by: Andrew Thornton Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: techknowlogick Co-authored-by: Lauris BH --- go.mod | 2 + go.sum | 5 + modules/git/batch_reader.go | 111 ++--- modules/git/commit_info_nogogit.go | 237 +---------- modules/git/last_commit_cache_nogogit.go | 5 +- modules/git/log_name_status.go | 398 ++++++++++++++++++ modules/git/notes_nogogit.go | 2 +- modules/git/pipeline/lfs_nogogit.go | 6 + modules/git/repo_language_stats_nogogit.go | 3 + modules/indexer/code/bleve.go | 3 + modules/indexer/code/elastic_search.go | 3 + vendor/github.com/djherbis/buffer/.travis.yml | 20 + vendor/github.com/djherbis/buffer/LICENSE.txt | 20 + vendor/github.com/djherbis/buffer/README.md | 174 ++++++++ vendor/github.com/djherbis/buffer/buffer.go | 48 +++ vendor/github.com/djherbis/buffer/discard.go | 36 ++ vendor/github.com/djherbis/buffer/file.go | 72 ++++ vendor/github.com/djherbis/buffer/go.mod | 3 + .../github.com/djherbis/buffer/limio/limit.go | 31 ++ vendor/github.com/djherbis/buffer/list.go | 47 +++ vendor/github.com/djherbis/buffer/list_at.go | 47 +++ vendor/github.com/djherbis/buffer/mem.go | 82 ++++ vendor/github.com/djherbis/buffer/multi.go | 185 ++++++++ .../github.com/djherbis/buffer/partition.go | 101 +++++ .../djherbis/buffer/partition_at.go | 187 ++++++++ vendor/github.com/djherbis/buffer/pool.go | 111 +++++ vendor/github.com/djherbis/buffer/pool_at.go | 111 +++++ vendor/github.com/djherbis/buffer/ring.go | 58 +++ vendor/github.com/djherbis/buffer/spill.go | 41 ++ vendor/github.com/djherbis/buffer/swap.go | 99 +++++ .../djherbis/buffer/wrapio/limitwrap.go | 94 +++++ .../github.com/djherbis/buffer/wrapio/wrap.go | 139 ++++++ vendor/github.com/djherbis/nio/v3/.travis.yml | 22 + vendor/github.com/djherbis/nio/v3/LICENSE.txt | 20 + vendor/github.com/djherbis/nio/v3/README.md | 65 +++ vendor/github.com/djherbis/nio/v3/go.mod | 5 + vendor/github.com/djherbis/nio/v3/go.sum | 2 + vendor/github.com/djherbis/nio/v3/nio.go | 53 +++ vendor/github.com/djherbis/nio/v3/sync.go | 177 ++++++++ vendor/modules.txt | 8 + 40 files changed, 2538 insertions(+), 295 deletions(-) create mode 100644 modules/git/log_name_status.go create mode 100644 vendor/github.com/djherbis/buffer/.travis.yml create mode 100644 vendor/github.com/djherbis/buffer/LICENSE.txt create mode 100644 vendor/github.com/djherbis/buffer/README.md create mode 100644 vendor/github.com/djherbis/buffer/buffer.go create mode 100644 vendor/github.com/djherbis/buffer/discard.go create mode 100644 vendor/github.com/djherbis/buffer/file.go create mode 100644 vendor/github.com/djherbis/buffer/go.mod create mode 100644 vendor/github.com/djherbis/buffer/limio/limit.go create mode 100644 vendor/github.com/djherbis/buffer/list.go create mode 100644 vendor/github.com/djherbis/buffer/list_at.go create mode 100644 vendor/github.com/djherbis/buffer/mem.go create mode 100644 vendor/github.com/djherbis/buffer/multi.go create mode 100644 vendor/github.com/djherbis/buffer/partition.go create mode 100644 vendor/github.com/djherbis/buffer/partition_at.go create mode 100644 vendor/github.com/djherbis/buffer/pool.go create mode 100644 vendor/github.com/djherbis/buffer/pool_at.go create mode 100644 vendor/github.com/djherbis/buffer/ring.go create mode 100644 vendor/github.com/djherbis/buffer/spill.go create mode 100644 vendor/github.com/djherbis/buffer/swap.go create mode 100644 vendor/github.com/djherbis/buffer/wrapio/limitwrap.go create mode 100644 vendor/github.com/djherbis/buffer/wrapio/wrap.go create mode 100644 vendor/github.com/djherbis/nio/v3/.travis.yml create mode 100644 vendor/github.com/djherbis/nio/v3/LICENSE.txt create mode 100644 vendor/github.com/djherbis/nio/v3/README.md create mode 100644 vendor/github.com/djherbis/nio/v3/go.mod create mode 100644 vendor/github.com/djherbis/nio/v3/go.sum create mode 100644 vendor/github.com/djherbis/nio/v3/nio.go create mode 100644 vendor/github.com/djherbis/nio/v3/sync.go diff --git a/go.mod b/go.mod index c28db7b8edb63..0ac321f0e012c 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,8 @@ require ( github.com/couchbase/goutils v0.0.0-20210118111533-e33d3ffb5401 // indirect github.com/denisenkom/go-mssqldb v0.10.0 github.com/dgrijalva/jwt-go v3.2.0+incompatible + github.com/djherbis/buffer v1.2.0 + github.com/djherbis/nio/v3 v3.0.1 github.com/dustin/go-humanize v1.0.0 github.com/editorconfig/editorconfig-core-go/v2 v2.4.2 github.com/emirpasic/gods v1.12.0 diff --git a/go.sum b/go.sum index 65a9f0b363fe4..31c66d0c4a0f3 100644 --- a/go.sum +++ b/go.sum @@ -244,6 +244,11 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/djherbis/buffer v1.1.0/go.mod h1:VwN8VdFkMY0DCALdY8o00d3IZ6Amz/UNVMWcSaJT44o= +github.com/djherbis/buffer v1.2.0 h1:PH5Dd2ss0C7CRRhQCZ2u7MssF+No9ide8Ye71nPHcrQ= +github.com/djherbis/buffer v1.2.0/go.mod h1:fjnebbZjCUpPinBRD+TDwXSOeNQ7fPQWLfGQqiAiUyE= +github.com/djherbis/nio/v3 v3.0.1 h1:6wxhnuppteMa6RHA4L81Dq7ThkZH8SwnDzXDYy95vB4= +github.com/djherbis/nio/v3 v3.0.1/go.mod h1:Ng4h80pbZFMla1yKzm61cF0tqqilXZYrogmWgZxOcmg= github.com/dlclark/regexp2 v1.1.6/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E= diff --git a/modules/git/batch_reader.go b/modules/git/batch_reader.go index d6ee0ce8e04d9..678b184708786 100644 --- a/modules/git/batch_reader.go +++ b/modules/git/batch_reader.go @@ -11,6 +11,9 @@ import ( "math" "strconv" "strings" + + "github.com/djherbis/buffer" + "github.com/djherbis/nio/v3" ) // WriteCloserError wraps an io.WriteCloser with an additional CloseWithError function @@ -42,7 +45,7 @@ func CatFileBatchCheck(repoPath string) (WriteCloserError, *bufio.Reader, func() } }() - // For simplicities sake we'll us a buffered reader to read from the cat-file --batch + // For simplicities sake we'll use a buffered reader to read from the cat-file --batch-check batchReader := bufio.NewReader(batchStdoutReader) return batchStdinWriter, batchReader, cancel @@ -53,7 +56,7 @@ func CatFileBatch(repoPath string) (WriteCloserError, *bufio.Reader, func()) { // We often want to feed the commits in order into cat-file --batch, followed by their trees and sub trees as necessary. // so let's create a batch stdin and stdout batchStdinReader, batchStdinWriter := io.Pipe() - batchStdoutReader, batchStdoutWriter := io.Pipe() + batchStdoutReader, batchStdoutWriter := nio.Pipe(buffer.New(32 * 1024)) cancel := func() { _ = batchStdinReader.Close() _ = batchStdinWriter.Close() @@ -74,7 +77,7 @@ func CatFileBatch(repoPath string) (WriteCloserError, *bufio.Reader, func()) { }() // For simplicities sake we'll us a buffered reader to read from the cat-file --batch - batchReader := bufio.NewReader(batchStdoutReader) + batchReader := bufio.NewReaderSize(batchStdoutReader, 32*1024) return batchStdinWriter, batchReader, cancel } @@ -84,22 +87,31 @@ func CatFileBatch(repoPath string) (WriteCloserError, *bufio.Reader, func()) { // SP SP LF // sha is a 40byte not 20byte here func ReadBatchLine(rd *bufio.Reader) (sha []byte, typ string, size int64, err error) { - sha, err = rd.ReadBytes(' ') + typ, err = rd.ReadString('\n') if err != nil { return } - sha = sha[:len(sha)-1] - - typ, err = rd.ReadString('\n') - if err != nil { + if len(typ) == 1 { + typ, err = rd.ReadString('\n') + if err != nil { + return + } + } + idx := strings.IndexByte(typ, ' ') + if idx < 0 { + log("missing space typ: %s", typ) + err = ErrNotExist{ID: string(sha)} return } + sha = []byte(typ[:idx]) + typ = typ[idx+1:] - idx := strings.Index(typ, " ") + idx = strings.IndexByte(typ, ' ') if idx < 0 { err = ErrNotExist{ID: string(sha)} return } + sizeStr := typ[idx+1 : len(typ)-1] typ = typ[:idx] @@ -130,7 +142,7 @@ headerLoop: } // Discard the rest of the tag - discard := size - n + discard := size - n + 1 for discard > math.MaxInt32 { _, err := rd.Discard(math.MaxInt32) if err != nil { @@ -200,85 +212,42 @@ func To40ByteSHA(sha, out []byte) []byte { return out } -// ParseTreeLineSkipMode reads an entry from a tree in a cat-file --batch stream -// This simply skips the mode - saving a substantial amount of time and carefully avoids allocations - except where fnameBuf is too small. +// ParseTreeLine reads an entry from a tree in a cat-file --batch stream +// This carefully avoids allocations - except where fnameBuf is too small. // It is recommended therefore to pass in an fnameBuf large enough to avoid almost all allocations // // Each line is composed of: // SP NUL <20-byte SHA> // // We don't attempt to convert the 20-byte SHA to 40-byte SHA to save a lot of time -func ParseTreeLineSkipMode(rd *bufio.Reader, fnameBuf, shaBuf []byte) (fname, sha []byte, n int, err error) { +func ParseTreeLine(rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fname, sha []byte, n int, err error) { var readBytes []byte - // Skip the Mode - readBytes, err = rd.ReadSlice(' ') // NB: DOES NOT ALLOCATE SIMPLY RETURNS SLICE WITHIN READER BUFFER - if err != nil { - return - } - n += len(readBytes) - // Deal with the fname + // Read the Mode & fname readBytes, err = rd.ReadSlice('\x00') - copy(fnameBuf, readBytes) - if len(fnameBuf) > len(readBytes) { - fnameBuf = fnameBuf[:len(readBytes)] // cut the buf the correct size - } else { - fnameBuf = append(fnameBuf, readBytes[len(fnameBuf):]...) // extend the buf and copy in the missing bits - } - for err == bufio.ErrBufferFull { // Then we need to read more - readBytes, err = rd.ReadSlice('\x00') - fnameBuf = append(fnameBuf, readBytes...) // there is little point attempting to avoid allocations here so just extend - } - n += len(fnameBuf) if err != nil { return } - fnameBuf = fnameBuf[:len(fnameBuf)-1] // Drop the terminal NUL - fname = fnameBuf // set the returnable fname to the slice - - // Now deal with the 20-byte SHA - idx := 0 - for idx < 20 { - read := 0 - read, err = rd.Read(shaBuf[idx:20]) - n += read - if err != nil { - return - } - idx += read - } - sha = shaBuf - return -} - -// ParseTreeLine reads an entry from a tree in a cat-file --batch stream -// This carefully avoids allocations - except where fnameBuf is too small. -// It is recommended therefore to pass in an fnameBuf large enough to avoid almost all allocations -// -// Each line is composed of: -// SP NUL <20-byte SHA> -// -// We don't attempt to convert the 20-byte SHA to 40-byte SHA to save a lot of time -func ParseTreeLine(rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fname, sha []byte, n int, err error) { - var readBytes []byte + idx := bytes.IndexByte(readBytes, ' ') + if idx < 0 { + log("missing space in readBytes ParseTreeLine: %s", readBytes) - // Read the Mode - readBytes, err = rd.ReadSlice(' ') - if err != nil { + err = &ErrNotExist{} return } - n += len(readBytes) - copy(modeBuf, readBytes) - if len(modeBuf) > len(readBytes) { - modeBuf = modeBuf[:len(readBytes)] - } else { - modeBuf = append(modeBuf, readBytes[len(modeBuf):]...) + n += idx + 1 + copy(modeBuf, readBytes[:idx]) + if len(modeBuf) >= idx { + modeBuf = modeBuf[:idx] + } else { + modeBuf = append(modeBuf, readBytes[len(modeBuf):idx]...) } - mode = modeBuf[:len(modeBuf)-1] // Drop the SP + mode = modeBuf + + readBytes = readBytes[idx+1:] // Deal with the fname - readBytes, err = rd.ReadSlice('\x00') copy(fnameBuf, readBytes) if len(fnameBuf) > len(readBytes) { fnameBuf = fnameBuf[:len(readBytes)] @@ -297,7 +266,7 @@ func ParseTreeLine(rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fn fname = fnameBuf // Deal with the 20-byte SHA - idx := 0 + idx = 0 for idx < 20 { read := 0 read, err = rd.Read(shaBuf[idx:20]) diff --git a/modules/git/commit_info_nogogit.go b/modules/git/commit_info_nogogit.go index f34bef9f018c8..2283510d9635d 100644 --- a/modules/git/commit_info_nogogit.go +++ b/modules/git/commit_info_nogogit.go @@ -7,15 +7,11 @@ package git import ( - "bufio" - "bytes" "context" "fmt" "io" - "math" "path" "sort" - "strings" ) // GetCommitsInfo gets information of all commits that are corresponding to these entries @@ -43,21 +39,16 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath return nil, nil, err } - for i, found := range commits { - if err := cache.Put(commit.ID.String(), path.Join(treePath, unHitPaths[i]), found.ID.String()); err != nil { + for pth, found := range commits { + if err := cache.Put(commit.ID.String(), path.Join(treePath, pth), found.ID.String()); err != nil { return nil, nil, err } - revs[unHitPaths[i]] = found + revs[pth] = found } } } else { sort.Strings(entryPaths) - revs = map[string]*Commit{} - var foundCommits []*Commit - foundCommits, err = GetLastCommitForPaths(ctx, commit, treePath, entryPaths) - for i, found := range foundCommits { - revs[entryPaths[i]] = found - } + revs, err = GetLastCommitForPaths(ctx, commit, treePath, entryPaths) } if err != nil { return nil, nil, err @@ -86,6 +77,8 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String()) commitsInfo[i].SubModuleFile = subModuleFile } + } else { + log("missing commit for %s", entry.Name()) } } @@ -125,220 +118,24 @@ func getLastCommitForPathsByCache(ctx context.Context, commitID, treePath string } // GetLastCommitForPaths returns last commit information -func GetLastCommitForPaths(ctx context.Context, commit *Commit, treePath string, paths []string) ([]*Commit, error) { +func GetLastCommitForPaths(ctx context.Context, commit *Commit, treePath string, paths []string) (map[string]*Commit, error) { // We read backwards from the commit to obtain all of the commits - - // We'll do this by using rev-list to provide us with parent commits in order - revListReader, revListWriter := io.Pipe() - defer func() { - _ = revListWriter.Close() - _ = revListReader.Close() - }() - - go func() { - stderr := strings.Builder{} - err := NewCommand("rev-list", "--format=%T", commit.ID.String()).SetParentContext(ctx).RunInDirPipeline(commit.repo.Path, revListWriter, &stderr) - if err != nil { - _ = revListWriter.CloseWithError(ConcatenateError(err, (&stderr).String())) - } else { - _ = revListWriter.Close() - } - }() + revs, err := WalkGitLog(ctx, commit.repo, commit, treePath, paths...) + if err != nil { + return nil, err + } batchStdinWriter, batchReader, cancel := commit.repo.CatFileBatch() defer cancel() - mapsize := 4096 - if len(paths) > mapsize { - mapsize = len(paths) - } - - path2idx := make(map[string]int, mapsize) - for i, path := range paths { - path2idx[path] = i - } - - fnameBuf := make([]byte, 4096) - modeBuf := make([]byte, 40) - - allShaBuf := make([]byte, (len(paths)+1)*20) - shaBuf := make([]byte, 20) - tmpTreeID := make([]byte, 40) - - // commits is the returnable commits matching the paths provided - commits := make([]string, len(paths)) - // ids are the blob/tree ids for the paths - ids := make([][]byte, len(paths)) - - // We'll use a scanner for the revList because it's simpler than a bufio.Reader - scan := bufio.NewScanner(revListReader) -revListLoop: - for scan.Scan() { - // Get the next parent commit ID - commitID := scan.Text() - if !scan.Scan() { - break revListLoop - } - commitID = commitID[7:] - rootTreeID := scan.Text() - - // push the tree to the cat-file --batch process - _, err := batchStdinWriter.Write([]byte(rootTreeID + "\n")) - if err != nil { - return nil, err - } - - currentPath := "" - - // OK if the target tree path is "" and the "" is in the paths just set this now - if treePath == "" && paths[0] == "" { - // If this is the first time we see this set the id appropriate for this paths to this tree and set the last commit to curCommit - if len(ids[0]) == 0 { - ids[0] = []byte(rootTreeID) - commits[0] = string(commitID) - } else if bytes.Equal(ids[0], []byte(rootTreeID)) { - commits[0] = string(commitID) - } - } - - treeReadingLoop: - for { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - _, _, size, err := ReadBatchLine(batchReader) - if err != nil { - return nil, err - } - - // Handle trees - - // n is counter for file position in the tree file - var n int64 - - // Two options: currentPath is the targetTreepath - if treePath == currentPath { - // We are in the right directory - // Parse each tree line in turn. (don't care about mode here.) - for n < size { - fname, sha, count, err := ParseTreeLineSkipMode(batchReader, fnameBuf, shaBuf) - shaBuf = sha - if err != nil { - return nil, err - } - n += int64(count) - idx, ok := path2idx[string(fname)] - if ok { - // Now if this is the first time round set the initial Blob(ish) SHA ID and the commit - if len(ids[idx]) == 0 { - copy(allShaBuf[20*(idx+1):20*(idx+2)], shaBuf) - ids[idx] = allShaBuf[20*(idx+1) : 20*(idx+2)] - commits[idx] = string(commitID) - } else if bytes.Equal(ids[idx], shaBuf) { - commits[idx] = string(commitID) - } - } - // FIXME: is there any order to the way strings are emitted from cat-file? - // if there is - then we could skip once we've passed all of our data - } - if _, err := batchReader.Discard(1); err != nil { - return nil, err - } - - break treeReadingLoop - } - - var treeID []byte - - // We're in the wrong directory - // Find target directory in this directory - idx := len(currentPath) - if idx > 0 { - idx++ - } - target := strings.SplitN(treePath[idx:], "/", 2)[0] - - for n < size { - // Read each tree entry in turn - mode, fname, sha, count, err := ParseTreeLine(batchReader, modeBuf, fnameBuf, shaBuf) - if err != nil { - return nil, err - } - n += int64(count) - - // if we have found the target directory - if bytes.Equal(fname, []byte(target)) && bytes.Equal(mode, []byte("40000")) { - copy(tmpTreeID, sha) - treeID = tmpTreeID - break - } - } - - if n < size { - // Discard any remaining entries in the current tree - discard := size - n - for discard > math.MaxInt32 { - _, err := batchReader.Discard(math.MaxInt32) - if err != nil { - return nil, err - } - discard -= math.MaxInt32 - } - _, err := batchReader.Discard(int(discard)) - if err != nil { - return nil, err - } - } - if _, err := batchReader.Discard(1); err != nil { - return nil, err - } - - // if we haven't found a treeID for the target directory our search is over - if len(treeID) == 0 { - break treeReadingLoop - } - - // add the target to the current path - if idx > 0 { - currentPath += "/" - } - currentPath += target - - // if we've now found the current path check its sha id and commit status - if treePath == currentPath && paths[0] == "" { - if len(ids[0]) == 0 { - copy(allShaBuf[0:20], treeID) - ids[0] = allShaBuf[0:20] - commits[0] = string(commitID) - } else if bytes.Equal(ids[0], treeID) { - commits[0] = string(commitID) - } - } - treeID = To40ByteSHA(treeID, treeID) - _, err = batchStdinWriter.Write(treeID) - if err != nil { - return nil, err - } - _, err = batchStdinWriter.Write([]byte("\n")) - if err != nil { - return nil, err - } - } - } - if scan.Err() != nil { - return nil, scan.Err() - } - - commitsMap := make(map[string]*Commit, len(commits)) + commitsMap := map[string]*Commit{} commitsMap[commit.ID.String()] = commit - commitCommits := make([]*Commit, len(commits)) - for i, commitID := range commits { + commitCommits := map[string]*Commit{} + for path, commitID := range revs { c, ok := commitsMap[commitID] if ok { - commitCommits[i] = c + commitCommits[path] = c continue } @@ -364,8 +161,8 @@ revListLoop: if _, err := batchReader.Discard(1); err != nil { return nil, err } - commitCommits[i] = c + commitCommits[path] = c } - return commitCommits, scan.Err() + return commitCommits, nil } diff --git a/modules/git/last_commit_cache_nogogit.go b/modules/git/last_commit_cache_nogogit.go index 3cbb0cca32e05..84c8ee132c260 100644 --- a/modules/git/last_commit_cache_nogogit.go +++ b/modules/git/last_commit_cache_nogogit.go @@ -88,9 +88,8 @@ func (c *LastCommitCache) recursiveCache(ctx context.Context, commit *Commit, tr return err } - for i, entryCommit := range commits { - entry := entryPaths[i] - if err := c.Put(commit.ID.String(), path.Join(treePath, entryPaths[i]), entryCommit.ID.String()); err != nil { + for entry, entryCommit := range commits { + if err := c.Put(commit.ID.String(), path.Join(treePath, entry), entryCommit.ID.String()); err != nil { return err } if entryMap[entry].IsDir() { diff --git a/modules/git/log_name_status.go b/modules/git/log_name_status.go new file mode 100644 index 0000000000000..803d614d611a2 --- /dev/null +++ b/modules/git/log_name_status.go @@ -0,0 +1,398 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package git + +import ( + "bufio" + "bytes" + "context" + "io" + "path" + "sort" + "strings" + + "github.com/djherbis/buffer" + "github.com/djherbis/nio/v3" +) + +// LogNameStatusRepo opens git log --raw in the provided repo and returns a stdin pipe, a stdout reader and cancel function +func LogNameStatusRepo(repository, head, treepath string, paths ...string) (*bufio.Reader, func()) { + // We often want to feed the commits in order into cat-file --batch, followed by their trees and sub trees as necessary. + // so let's create a batch stdin and stdout + stdoutReader, stdoutWriter := nio.Pipe(buffer.New(32 * 1024)) + cancel := func() { + _ = stdoutReader.Close() + _ = stdoutWriter.Close() + } + + args := make([]string, 0, 8+len(paths)) + args = append(args, "log", "--name-status", "-c", "--format=commit%x00%H %P%x00", "--parents", "--no-renames", "-t", "-z", head, "--") + if len(paths) < 70 { + if treepath != "" { + args = append(args, treepath) + for _, pth := range paths { + if pth != "" { + args = append(args, path.Join(treepath, pth)) + } + } + } else { + for _, pth := range paths { + if pth != "" { + args = append(args, pth) + } + } + } + } else if treepath != "" { + args = append(args, treepath) + } + + go func() { + stderr := strings.Builder{} + err := NewCommand(args...).RunInDirFullPipeline(repository, stdoutWriter, &stderr, nil) + if err != nil { + _ = stdoutWriter.CloseWithError(ConcatenateError(err, (&stderr).String())) + } else { + _ = stdoutWriter.Close() + } + }() + + // For simplicities sake we'll us a buffered reader to read from the cat-file --batch + bufReader := bufio.NewReaderSize(stdoutReader, 32*1024) + + return bufReader, cancel +} + +// LogNameStatusRepoParser parses a git log raw output from LogRawRepo +type LogNameStatusRepoParser struct { + treepath string + paths []string + next []byte + buffull bool + rd *bufio.Reader + cancel func() +} + +// NewLogNameStatusRepoParser returns a new parser for a git log raw output +func NewLogNameStatusRepoParser(repository, head, treepath string, paths ...string) *LogNameStatusRepoParser { + rd, cancel := LogNameStatusRepo(repository, head, treepath, paths...) + return &LogNameStatusRepoParser{ + treepath: treepath, + paths: paths, + rd: rd, + cancel: cancel, + } +} + +// LogNameStatusCommitData represents a commit artefact from git log raw +type LogNameStatusCommitData struct { + CommitID string + ParentIDs []string + Paths []bool +} + +// Next returns the next LogStatusCommitData +func (g *LogNameStatusRepoParser) Next(treepath string, paths2ids map[string]int, changed []bool, maxpathlen int) (*LogNameStatusCommitData, error) { + var err error + if g.next == nil || len(g.next) == 0 { + g.buffull = false + g.next, err = g.rd.ReadSlice('\x00') + if err != nil { + if err == bufio.ErrBufferFull { + g.buffull = true + } else if err == io.EOF { + return nil, nil + } else { + return nil, err + } + } + } + + ret := LogNameStatusCommitData{} + if bytes.Equal(g.next, []byte("commit\000")) { + g.next, err = g.rd.ReadSlice('\x00') + if err != nil { + if err == bufio.ErrBufferFull { + g.buffull = true + } else if err == io.EOF { + return nil, nil + } else { + return nil, err + } + } + } + + // Our "line" must look like: SP ( SP) * NUL + ret.CommitID = string(g.next[0:40]) + parents := string(g.next[41:]) + if g.buffull { + more, err := g.rd.ReadString('\x00') + if err != nil { + return nil, err + } + parents += more + } + parents = parents[:len(parents)-1] + ret.ParentIDs = strings.Split(parents, " ") + + // now read the next "line" + g.buffull = false + g.next, err = g.rd.ReadSlice('\x00') + if err != nil { + if err == bufio.ErrBufferFull { + g.buffull = true + } else if err != io.EOF { + return nil, err + } + } + + if err == io.EOF || !(g.next[0] == '\n' || g.next[0] == '\000') { + return &ret, nil + } + + // Ok we have some changes. + // This line will look like: NL NUL + // + // Subsequent lines will not have the NL - so drop it here - g.bufffull must also be false at this point too. + if g.next[0] == '\n' { + g.next = g.next[1:] + } else { + g.buffull = false + g.next, err = g.rd.ReadSlice('\x00') + if err != nil { + if err == bufio.ErrBufferFull { + g.buffull = true + } else if err != io.EOF { + return nil, err + } + } + if g.next[0] == '\x00' { + g.buffull = false + g.next, err = g.rd.ReadSlice('\x00') + if err != nil { + if err == bufio.ErrBufferFull { + g.buffull = true + } else if err != io.EOF { + return nil, err + } + } + } + } + + fnameBuf := make([]byte, 4096) + +diffloop: + for { + if err == io.EOF || bytes.Equal(g.next, []byte("commit\000")) { + return &ret, nil + } + g.next, err = g.rd.ReadSlice('\x00') + if err != nil { + if err == bufio.ErrBufferFull { + g.buffull = true + } else if err == io.EOF { + return &ret, nil + } else { + return nil, err + } + } + copy(fnameBuf, g.next) + if len(fnameBuf) < len(g.next) { + fnameBuf = append(fnameBuf, g.next[len(fnameBuf):]...) + } else { + fnameBuf = fnameBuf[:len(g.next)] + } + if err != nil { + if err != bufio.ErrBufferFull { + return nil, err + } + more, err := g.rd.ReadBytes('\x00') + if err != nil { + return nil, err + } + fnameBuf = append(fnameBuf, more...) + } + + // read the next line + g.buffull = false + g.next, err = g.rd.ReadSlice('\x00') + if err != nil { + if err == bufio.ErrBufferFull { + g.buffull = true + } else if err != io.EOF { + return nil, err + } + } + + if treepath != "" { + if !bytes.HasPrefix(fnameBuf, []byte(treepath)) { + fnameBuf = fnameBuf[:cap(fnameBuf)] + continue diffloop + } + } + fnameBuf = fnameBuf[len(treepath) : len(fnameBuf)-1] + if len(fnameBuf) > maxpathlen { + fnameBuf = fnameBuf[:cap(fnameBuf)] + continue diffloop + } + if len(fnameBuf) > 0 { + if len(treepath) > 0 { + if fnameBuf[0] != '/' || bytes.IndexByte(fnameBuf[1:], '/') >= 0 { + fnameBuf = fnameBuf[:cap(fnameBuf)] + continue diffloop + } + fnameBuf = fnameBuf[1:] + } else if bytes.IndexByte(fnameBuf, '/') >= 0 { + fnameBuf = fnameBuf[:cap(fnameBuf)] + continue diffloop + } + } + + idx, ok := paths2ids[string(fnameBuf)] + if !ok { + fnameBuf = fnameBuf[:cap(fnameBuf)] + continue diffloop + } + if ret.Paths == nil { + ret.Paths = changed + } + changed[idx] = true + } +} + +// Close closes the parser +func (g *LogNameStatusRepoParser) Close() { + g.cancel() +} + +// WalkGitLog walks the git log --name-status for the head commit in the provided treepath and files +func WalkGitLog(ctx context.Context, repo *Repository, head *Commit, treepath string, paths ...string) (map[string]string, error) { + tree, err := head.SubTree(treepath) + if err != nil { + return nil, err + } + + entries, err := tree.ListEntries() + if err != nil { + return nil, err + } + + if len(paths) == 0 { + paths = make([]string, 0, len(entries)+1) + paths = append(paths, "") + for _, entry := range entries { + paths = append(paths, entry.Name()) + } + } else { + sort.Strings(paths) + if paths[0] != "" { + paths = append([]string{""}, paths...) + } + // remove duplicates + for i := len(paths) - 1; i > 0; i-- { + if paths[i] == paths[i-1] { + paths = append(paths[:i-1], paths[i:]...) + } + } + } + + path2idx := map[string]int{} + maxpathlen := len(treepath) + + for i := range paths { + path2idx[paths[i]] = i + pthlen := len(paths[i]) + len(treepath) + 1 + if pthlen > maxpathlen { + maxpathlen = pthlen + } + } + + g := NewLogNameStatusRepoParser(repo.Path, head.ID.String(), treepath, paths...) + defer g.Close() + + results := make([]string, len(paths)) + remaining := len(paths) + nextRestart := (len(paths) * 3) / 4 + if nextRestart > 70 { + nextRestart = 70 + } + lastEmptyParent := head.ID.String() + commitSinceLastEmptyParent := uint64(0) + commitSinceNextRestart := uint64(0) + parentRemaining := map[string]bool{} + + changed := make([]bool, len(paths)) + +heaploop: + for { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + current, err := g.Next(treepath, path2idx, changed, maxpathlen) + if err != nil { + g.Close() + return nil, err + } + if current == nil { + break heaploop + } + delete(parentRemaining, current.CommitID) + if current.Paths != nil { + for i, found := range current.Paths { + if !found { + continue + } + changed[i] = false + if results[i] == "" { + results[i] = current.CommitID + delete(path2idx, paths[i]) + remaining-- + if results[0] == "" { + results[0] = current.CommitID + delete(path2idx, "") + remaining-- + } + } + } + } + + if remaining <= 0 { + break heaploop + } + commitSinceLastEmptyParent++ + if len(parentRemaining) == 0 { + lastEmptyParent = current.CommitID + commitSinceLastEmptyParent = 0 + } + if remaining <= nextRestart { + commitSinceNextRestart++ + if 4*commitSinceNextRestart > 3*commitSinceLastEmptyParent { + g.Close() + remainingPaths := make([]string, 0, len(paths)) + for i, pth := range paths { + if results[i] == "" { + remainingPaths = append(remainingPaths, pth) + } + } + g = NewLogNameStatusRepoParser(repo.Path, lastEmptyParent, treepath, remainingPaths...) + parentRemaining = map[string]bool{} + nextRestart = (remaining * 3) / 4 + continue heaploop + } + } + for _, parent := range current.ParentIDs { + parentRemaining[parent] = true + } + } + g.Close() + + resultsMap := map[string]string{} + for i, pth := range paths { + resultsMap[pth] = results[i] + } + + return resultsMap, nil +} diff --git a/modules/git/notes_nogogit.go b/modules/git/notes_nogogit.go index 2b927249954a7..267087a86fafa 100644 --- a/modules/git/notes_nogogit.go +++ b/modules/git/notes_nogogit.go @@ -68,7 +68,7 @@ func GetNote(ctx context.Context, repo *Repository, commitID string, note *Note) if err != nil { return err } - note.Commit = lastCommits[0] + note.Commit = lastCommits[path] return nil } diff --git a/modules/git/pipeline/lfs_nogogit.go b/modules/git/pipeline/lfs_nogogit.go index e618dd04b7a3d..d3696fcda219c 100644 --- a/modules/git/pipeline/lfs_nogogit.go +++ b/modules/git/pipeline/lfs_nogogit.go @@ -116,6 +116,9 @@ func FindLFSFile(repo *git.Repository, hash git.SHA1) ([]*LFSResult, error) { if err != nil { return nil, err } + if _, err := batchReader.Discard(1); err != nil { + return nil, err + } _, err := batchStdinWriter.Write([]byte(curCommit.Tree.ID.String() + "\n")) if err != nil { @@ -146,6 +149,9 @@ func FindLFSFile(repo *git.Repository, hash git.SHA1) ([]*LFSResult, error) { paths = append(paths, curPath+string(fname)+"/") } } + if _, err := batchReader.Discard(1); err != nil { + return nil, err + } if len(trees) > 0 { _, err := batchStdinWriter.Write(trees[len(trees)-1]) if err != nil { diff --git a/modules/git/repo_language_stats_nogogit.go b/modules/git/repo_language_stats_nogogit.go index abbf5e943ba43..46b084cf01e37 100644 --- a/modules/git/repo_language_stats_nogogit.go +++ b/modules/git/repo_language_stats_nogogit.go @@ -49,6 +49,9 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err log("Unable to get commit for: %s. Err: %v", commitID, err) return nil, err } + if _, err = batchReader.Discard(1); err != nil { + return nil, err + } tree := commit.Tree diff --git a/modules/indexer/code/bleve.go b/modules/indexer/code/bleve.go index 17128052f4912..600789a284096 100644 --- a/modules/indexer/code/bleve.go +++ b/modules/indexer/code/bleve.go @@ -216,6 +216,9 @@ func (b *BleveIndexer) addUpdate(batchWriter git.WriteCloserError, batchReader * return nil } + if _, err = batchReader.Discard(1); err != nil { + return err + } id := filenameIndexerID(repo.ID, update.Filename) return batch.Index(id, &RepoIndexerData{ RepoID: repo.ID, diff --git a/modules/indexer/code/elastic_search.go b/modules/indexer/code/elastic_search.go index 16d4a1821a2f7..38a97ad888c05 100644 --- a/modules/indexer/code/elastic_search.go +++ b/modules/indexer/code/elastic_search.go @@ -215,6 +215,9 @@ func (b *ElasticSearchIndexer) addUpdate(batchWriter git.WriteCloserError, batch return nil, nil } + if _, err = batchReader.Discard(1); err != nil { + return nil, err + } id := filenameIndexerID(repo.ID, update.Filename) return []elastic.BulkableRequest{ diff --git a/vendor/github.com/djherbis/buffer/.travis.yml b/vendor/github.com/djherbis/buffer/.travis.yml new file mode 100644 index 0000000000000..7d03fb1ffb667 --- /dev/null +++ b/vendor/github.com/djherbis/buffer/.travis.yml @@ -0,0 +1,20 @@ +language: go +go: +- tip +before_install: + - go get golang.org/x/lint/golint + - go get github.com/axw/gocov/gocov + - go get github.com/mattn/goveralls + - if ! go get code.google.com/p/go.tools/cmd/cover; then go get golang.org/x/tools/cmd/cover; fi +script: + - '[ "${TRAVIS_PULL_REQUEST}" != "false" ] || $HOME/gopath/bin/goveralls -service=travis-ci -repotoken $COVERALLS_TOKEN' + - $HOME/gopath/bin/golint ./... + - go vet + - go test -v ./... +notifications: + email: + on_success: never + on_failure: change +env: + global: + secure: X2uEipzLOL7IDFQgiJdKQvA7gWw746gmU4HoLr73Au+mDZnIaYfpM7pR0r9S9DY23obmflOBFytB9IIyr6Ganhs8KDd6osBS3JSu5ydZKhoHDshSZHxW6GdCiR0Ya85JZ2k/CzwuZ95FcCTztXG59D8VhAoM+8gNW6VLK2mL60Y= diff --git a/vendor/github.com/djherbis/buffer/LICENSE.txt b/vendor/github.com/djherbis/buffer/LICENSE.txt new file mode 100644 index 0000000000000..f5daa194f7803 --- /dev/null +++ b/vendor/github.com/djherbis/buffer/LICENSE.txt @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2015 Dustin H + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/djherbis/buffer/README.md b/vendor/github.com/djherbis/buffer/README.md new file mode 100644 index 0000000000000..b953f8b9fe01a --- /dev/null +++ b/vendor/github.com/djherbis/buffer/README.md @@ -0,0 +1,174 @@ +Buffer +========== + +[![GoDoc](https://godoc.org/github.com/djherbis/buffer?status.svg)](https://godoc.org/github.com/djherbis/buffer) +[![Release](https://img.shields.io/github/release/djherbis/buffer.svg)](https://github.com/djherbis/buffer/releases/latest) +[![Software License](https://img.shields.io/badge/license-MIT-brightgreen.svg)](LICENSE.txt) +[![Build Status](https://travis-ci.org/djherbis/buffer.svg?branch=master)](https://travis-ci.org/djherbis/buffer) +[![Coverage Status](https://coveralls.io/repos/djherbis/buffer/badge.svg?branch=master)](https://coveralls.io/r/djherbis/buffer?branch=master) +[![Go Report Card](https://goreportcard.com/badge/github.com/djherbis/buffer)](https://goreportcard.com/report/github.com/djherbis/buffer) + +Usage +------------ + +The following buffers provide simple unique behaviours which when composed can create complex buffering strategies. For use with github.com/djherbis/nio for Buffered io.Pipe and io.Copy implementations. + +For example: + +```go +import ( + "github.com/djherbis/buffer" + "github.com/djherbis/nio" + + "io/ioutil" +) + +// Buffer 32KB to Memory, after that buffer to 100MB chunked files +buf := buffer.NewUnboundedBuffer(32*1024, 100*1024*1024) +nio.Copy(w, r, buf) // Reads from r, writes to buf, reads from buf writes to w (concurrently). + +// Buffer 32KB to Memory, discard overflow +buf = buffer.NewSpill(32*1024, ioutil.Discard) +nio.Copy(w, r, buf) +``` + +Supported Buffers +------------ + +#### Bounded Buffers #### + +Memory: Wrapper for bytes.Buffer + +File: File-based buffering. The file never exceeds Cap() in length, no matter how many times its written/read from. It accomplishes this by "wrapping" around the fixed max-length file when the data gets too long but there is available freed space at the beginning of the file. The caller is responsible for closing and deleting the file when done. + +```go +import ( + "ioutil" + "os" + + "github.com/djherbis/buffer" +) + +// Create a File-based Buffer with max size 100MB +file, err := ioutil.TempFile("", "buffer") +if err != nil { + return err +} +defer os.Remove(file.Name()) +defer file.Close() + +buf := buffer.NewFile(100*1024*1024, file) + +// A simpler way: +pool := buffer.NewFilePool(100*1024*1024, "") // "" -- use temp dir +buf, err := pool.Get() // allocate the buffer +if err != nil { + return err +} +defer pool.Put(buf) // close and remove the allocated file for the buffer + +``` + +Multi: A fixed length linked-list of buffers. Each buffer reads from the next buffer so that all the buffered data is shifted upwards in the list when reading. Writes are always written to the first buffer in the list whose Len() < Cap(). + +```go +import ( + "github.com/djherbis/buffer" +) + +mem := buffer.New(32*1024) +file := buffer.NewFile(100*1024*1024, someFileObj)) // you'll need to manage Open(), Close() and Delete someFileObj + +// Buffer composed of 32KB of memory, and 100MB of file. +buf := buffer.NewMulti(mem, file) +``` + +#### Unbounded Buffers #### + +Partition: A queue of buffers. Writes always go to the last buffer in the queue. If all buffers are full, a new buffer is "pushed" to the end of the queue (generated by a user-given function). Reads come from the first buffer, when the first buffer is emptied it is "popped" off the queue. + +```go +import ( + "github.com/djherbis/buffer" +) + +// Create 32 KB sized-chunks of memory as needed to expand/contract the buffer size. +buf := buffer.NewPartition(buffer.NewMemPool(32*1024)) + +// Create 100 MB sized-chunks of files as needed to expand/contract the buffer size. +buf = buffer.NewPartition(buffer.NewFilePool(100*1024*1024, "")) +``` + +Ring: A single buffer which begins overwriting the oldest buffered data when it reaches its capacity. + +```go +import ( + "github.com/djherbis/buffer" +) + +// Create a File-based Buffer with max size 100MB +file := buffer.NewFile(100*1024*1024, someFileObj) // you'll need to Open(), Close() and Delete someFileObj. + +// If buffered data exceeds 100MB, overwrite oldest data as new data comes in +buf := buffer.NewRing(file) // requires BufferAt interface. +``` + +Spill: A single buffer which when full, writes the overflow to a given io.Writer. +-> Note that it will actually "spill" whenever there is an error while writing, this should only be a "full" error. + +```go +import ( + "github.com/djherbis/buffer" + "github.com/djherbis/nio" + + "io/ioutil" +) + +// Buffer 32KB to Memory, discard overflow +buf := buffer.NewSpill(32*1024, ioutil.Discard) +nio.Copy(w, r, buf) +``` + +#### Empty Buffer #### + +Discard: Reads always return EOF, writes goto ioutil.Discard. + +```go +import ( + "github.com/djherbis/buffer" +) + +// Reads will return io.EOF, writes will return success (nil error, full write) but no data was written. +buf := buffer.Discard +``` + +Custom Buffers +------------ + +Feel free to implement your own buffer, just meet the required interface (Buffer/BufferAt) and compose away! + +```go + +// Buffer Interface used by Multi and Partition +type Buffer interface { + Len() int64 + Cap() int64 + io.Reader + io.Writer + Reset() +} + +// BufferAt interface used by Ring +type BufferAt interface { + Buffer + io.ReaderAt + io.WriterAt +} + +``` + +Installation +------------ +```sh +go get github.com/djherbis/buffer +``` diff --git a/vendor/github.com/djherbis/buffer/buffer.go b/vendor/github.com/djherbis/buffer/buffer.go new file mode 100644 index 0000000000000..34b0811dacf45 --- /dev/null +++ b/vendor/github.com/djherbis/buffer/buffer.go @@ -0,0 +1,48 @@ +// Package buffer implements a series of Buffers which can be composed to implement complicated buffering strategies +package buffer + +import ( + "io" + "os" +) + +// Buffer is used to Write() data which will be Read() later. +type Buffer interface { + Len() int64 // How much data is Buffered in bytes + Cap() int64 // How much data can be Buffered at once in bytes. + io.Reader // Read() will read from the top of the buffer [io.EOF if empty] + io.Writer // Write() will write to the end of the buffer [io.ErrShortWrite if not enough space] + Reset() // Truncates the buffer, Len() == 0. +} + +// BufferAt is a buffer which supports io.ReaderAt and io.WriterAt +type BufferAt interface { + Buffer + io.ReaderAt + io.WriterAt +} + +func len64(p []byte) int64 { + return int64(len(p)) +} + +// Gap returns buf.Cap() - buf.Len() +func Gap(buf Buffer) int64 { + return buf.Cap() - buf.Len() +} + +// Full returns true iff buf.Len() == buf.Cap() +func Full(buf Buffer) bool { + return buf.Len() == buf.Cap() +} + +// Empty returns false iff buf.Len() == 0 +func Empty(buf Buffer) bool { + return buf.Len() == 0 +} + +// NewUnboundedBuffer returns a Buffer which buffers "mem" bytes to memory +// and then creates file's of size "file" to buffer above "mem" bytes. +func NewUnboundedBuffer(mem, file int64) Buffer { + return NewMulti(New(mem), NewPartition(NewFilePool(file, os.TempDir()))) +} diff --git a/vendor/github.com/djherbis/buffer/discard.go b/vendor/github.com/djherbis/buffer/discard.go new file mode 100644 index 0000000000000..ecf44987bab52 --- /dev/null +++ b/vendor/github.com/djherbis/buffer/discard.go @@ -0,0 +1,36 @@ +package buffer + +import ( + "encoding/gob" + "io" + "io/ioutil" + "math" +) + +type discard struct{} + +// Discard is a Buffer which writes to ioutil.Discard and read's return 0, io.EOF. +// All of its methods are concurrent safe. +var Discard Buffer = discard{} + +func (buf discard) Len() int64 { + return 0 +} + +func (buf discard) Cap() int64 { + return math.MaxInt64 +} + +func (buf discard) Reset() {} + +func (buf discard) Read(p []byte) (n int, err error) { + return 0, io.EOF +} + +func (buf discard) Write(p []byte) (int, error) { + return ioutil.Discard.Write(p) +} + +func init() { + gob.Register(&discard{}) +} diff --git a/vendor/github.com/djherbis/buffer/file.go b/vendor/github.com/djherbis/buffer/file.go new file mode 100644 index 0000000000000..6eb77aabe6ec5 --- /dev/null +++ b/vendor/github.com/djherbis/buffer/file.go @@ -0,0 +1,72 @@ +package buffer + +import ( + "bytes" + "encoding/gob" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/djherbis/buffer/wrapio" +) + +// File is used as the backing resource for a the NewFile BufferAt. +type File interface { + Name() string + Stat() (fi os.FileInfo, err error) + io.ReaderAt + io.WriterAt + Close() error +} + +type fileBuffer struct { + file File + *wrapio.Wrapper +} + +// NewFile returns a new BufferAt backed by "file" with max-size N. +func NewFile(N int64, file File) BufferAt { + return &fileBuffer{ + file: file, + Wrapper: wrapio.NewWrapper(file, 0, 0, N), + } +} + +func init() { + gob.Register(&fileBuffer{}) +} + +func (buf *fileBuffer) MarshalBinary() ([]byte, error) { + fullpath, err := filepath.Abs(filepath.Dir(buf.file.Name())) + if err != nil { + return nil, err + } + base := filepath.Base(buf.file.Name()) + buf.file.Close() + + buffer := bytes.NewBuffer(nil) + fmt.Fprintln(buffer, filepath.Join(fullpath, base)) + fmt.Fprintln(buffer, buf.Wrapper.N, buf.Wrapper.L, buf.Wrapper.O) + return buffer.Bytes(), nil +} + +func (buf *fileBuffer) UnmarshalBinary(data []byte) error { + buffer := bytes.NewBuffer(data) + var filename string + var N, L, O int64 + _, err := fmt.Fscanln(buffer, &filename) + if err != nil { + return err + } + + file, err := os.Open(filename) + if err != nil { + return err + } + buf.file = file + + _, err = fmt.Fscanln(buffer, &N, &L, &O) + buf.Wrapper = wrapio.NewWrapper(file, L, O, N) + return err +} diff --git a/vendor/github.com/djherbis/buffer/go.mod b/vendor/github.com/djherbis/buffer/go.mod new file mode 100644 index 0000000000000..e0ef91a6a5aee --- /dev/null +++ b/vendor/github.com/djherbis/buffer/go.mod @@ -0,0 +1,3 @@ +module github.com/djherbis/buffer + +go 1.13 diff --git a/vendor/github.com/djherbis/buffer/limio/limit.go b/vendor/github.com/djherbis/buffer/limio/limit.go new file mode 100644 index 0000000000000..deb50fffc9c65 --- /dev/null +++ b/vendor/github.com/djherbis/buffer/limio/limit.go @@ -0,0 +1,31 @@ +package limio + +import "io" + +type limitedWriter struct { + W io.Writer + N int64 +} + +func (l *limitedWriter) Write(p []byte) (n int, err error) { + if l.N <= 0 { + return 0, io.ErrShortWrite + } + if int64(len(p)) > l.N { + p = p[0:l.N] + err = io.ErrShortWrite + } + n, er := l.W.Write(p) + if er != nil { + err = er + } + l.N -= int64(n) + return n, err +} + +// LimitWriter works like io.LimitReader. It writes at most n bytes +// to the underlying Writer. It returns io.ErrShortWrite if more than n +// bytes are attempted to be written. +func LimitWriter(w io.Writer, n int64) io.Writer { + return &limitedWriter{W: w, N: n} +} diff --git a/vendor/github.com/djherbis/buffer/list.go b/vendor/github.com/djherbis/buffer/list.go new file mode 100644 index 0000000000000..11ec6961c7af1 --- /dev/null +++ b/vendor/github.com/djherbis/buffer/list.go @@ -0,0 +1,47 @@ +package buffer + +import "math" + +// List is a slice of Buffers, it's the backing for NewPartition +type List []Buffer + +// Len is the sum of the Len()'s of the Buffers in the List. +func (l *List) Len() (n int64) { + for _, buffer := range *l { + if n > math.MaxInt64-buffer.Len() { + return math.MaxInt64 + } + n += buffer.Len() + } + return n +} + +// Cap is the sum of the Cap()'s of the Buffers in the List. +func (l *List) Cap() (n int64) { + for _, buffer := range *l { + if n > math.MaxInt64-buffer.Cap() { + return math.MaxInt64 + } + n += buffer.Cap() + } + return n +} + +// Reset calls Reset() on each of the Buffers in the list. +func (l *List) Reset() { + for _, buffer := range *l { + buffer.Reset() + } +} + +// Push adds a Buffer to the end of the List +func (l *List) Push(b Buffer) { + *l = append(*l, b) +} + +// Pop removes and returns a Buffer from the front of the List +func (l *List) Pop() (b Buffer) { + b = (*l)[0] + *l = (*l)[1:] + return b +} diff --git a/vendor/github.com/djherbis/buffer/list_at.go b/vendor/github.com/djherbis/buffer/list_at.go new file mode 100644 index 0000000000000..6893df8856a0d --- /dev/null +++ b/vendor/github.com/djherbis/buffer/list_at.go @@ -0,0 +1,47 @@ +package buffer + +import "math" + +// ListAt is a slice of BufferAt's, it's the backing for NewPartitionAt +type ListAt []BufferAt + +// Len is the sum of the Len()'s of the BufferAt's in the list. +func (l *ListAt) Len() (n int64) { + for _, buffer := range *l { + if n > math.MaxInt64-buffer.Len() { + return math.MaxInt64 + } + n += buffer.Len() + } + return n +} + +// Cap is the sum of the Cap()'s of the BufferAt's in the list. +func (l *ListAt) Cap() (n int64) { + for _, buffer := range *l { + if n > math.MaxInt64-buffer.Cap() { + return math.MaxInt64 + } + n += buffer.Cap() + } + return n +} + +// Reset calls Reset() on each of the BufferAt's in the list. +func (l *ListAt) Reset() { + for _, buffer := range *l { + buffer.Reset() + } +} + +// Push adds a BufferAt to the end of the list +func (l *ListAt) Push(b BufferAt) { + *l = append(*l, b) +} + +// Pop removes and returns a BufferAt from the front of the list +func (l *ListAt) Pop() (b BufferAt) { + b = (*l)[0] + *l = (*l)[1:] + return b +} diff --git a/vendor/github.com/djherbis/buffer/mem.go b/vendor/github.com/djherbis/buffer/mem.go new file mode 100644 index 0000000000000..8c7ef36461b57 --- /dev/null +++ b/vendor/github.com/djherbis/buffer/mem.go @@ -0,0 +1,82 @@ +package buffer + +import ( + "bytes" + "encoding/gob" + "fmt" + "io" + + "github.com/djherbis/buffer/limio" +) + +type memory struct { + N int64 + *bytes.Buffer +} + +// New returns a new in memory BufferAt with max size N. +// It's backed by a bytes.Buffer. +func New(n int64) BufferAt { + return &memory{ + N: n, + Buffer: bytes.NewBuffer(nil), + } +} + +func (buf *memory) Cap() int64 { + return buf.N +} + +func (buf *memory) Len() int64 { + return int64(buf.Buffer.Len()) +} + +func (buf *memory) Write(p []byte) (n int, err error) { + return limio.LimitWriter(buf.Buffer, Gap(buf)).Write(p) +} + +func (buf *memory) WriteAt(p []byte, off int64) (n int, err error) { + if off > buf.Len() { + return 0, io.ErrShortWrite + } else if len64(p)+off <= buf.Len() { + d := buf.Bytes()[off:] + return copy(d, p), nil + } else { + d := buf.Bytes()[off:] + n = copy(d, p) + m, err := buf.Write(p[n:]) + return n + m, err + } +} + +func (buf *memory) ReadAt(p []byte, off int64) (n int, err error) { + return bytes.NewReader(buf.Bytes()).ReadAt(p, off) +} + +func (buf *memory) Read(p []byte) (n int, err error) { + return io.LimitReader(buf.Buffer, buf.Len()).Read(p) +} + +func (buf *memory) ReadFrom(r io.Reader) (n int64, err error) { + return buf.Buffer.ReadFrom(io.LimitReader(r, Gap(buf))) +} + +func init() { + gob.Register(&memory{}) +} + +func (buf *memory) MarshalBinary() ([]byte, error) { + var b bytes.Buffer + fmt.Fprintln(&b, buf.N) + b.Write(buf.Bytes()) + return b.Bytes(), nil +} + +func (buf *memory) UnmarshalBinary(bindata []byte) error { + data := make([]byte, len(bindata)) + copy(data, bindata) + b := bytes.NewBuffer(data) + _, err := fmt.Fscanln(b, &buf.N) + buf.Buffer = bytes.NewBuffer(b.Bytes()) + return err +} diff --git a/vendor/github.com/djherbis/buffer/multi.go b/vendor/github.com/djherbis/buffer/multi.go new file mode 100644 index 0000000000000..a752483ee43be --- /dev/null +++ b/vendor/github.com/djherbis/buffer/multi.go @@ -0,0 +1,185 @@ +package buffer + +import ( + "bytes" + "encoding/gob" + "io" + "math" +) + +type chain struct { + Buf BufferAt + Next BufferAt +} + +type nopBufferAt struct { + Buffer +} + +func (buf *nopBufferAt) ReadAt(p []byte, off int64) (int, error) { + panic("ReadAt not implemented") +} + +func (buf *nopBufferAt) WriteAt(p []byte, off int64) (int, error) { + panic("WriteAt not implemented") +} + +// toBufferAt converts a Buffer to a BufferAt with nop ReadAt and WriteAt funcs +func toBufferAt(buf Buffer) BufferAt { + return &nopBufferAt{Buffer: buf} +} + +// NewMultiAt returns a BufferAt which is the logical concatenation of the passed BufferAts. +// The data in the buffers is shifted such that there is no non-empty buffer following +// a non-full buffer, this process is also run after every Read. +// If no buffers are passed, the returned Buffer is nil. +func NewMultiAt(buffers ...BufferAt) BufferAt { + if len(buffers) == 0 { + return nil + } else if len(buffers) == 1 { + return buffers[0] + } + + buf := &chain{ + Buf: buffers[0], + Next: NewMultiAt(buffers[1:]...), + } + + buf.Defrag() + + return buf +} + +// NewMulti returns a Buffer which is the logical concatenation of the passed buffers. +// The data in the buffers is shifted such that there is no non-empty buffer following +// a non-full buffer, this process is also run after every Read. +// If no buffers are passed, the returned Buffer is nil. +func NewMulti(buffers ...Buffer) Buffer { + bufAt := make([]BufferAt, len(buffers)) + for i, buf := range buffers { + bufAt[i] = toBufferAt(buf) + } + return NewMultiAt(bufAt...) +} + +func (buf *chain) Reset() { + buf.Next.Reset() + buf.Buf.Reset() +} + +func (buf *chain) Cap() (n int64) { + Next := buf.Next.Cap() + if buf.Buf.Cap() > math.MaxInt64-Next { + return math.MaxInt64 + } + return buf.Buf.Cap() + Next +} + +func (buf *chain) Len() (n int64) { + Next := buf.Next.Len() + if buf.Buf.Len() > math.MaxInt64-Next { + return math.MaxInt64 + } + return buf.Buf.Len() + Next +} + +func (buf *chain) Defrag() { + for !Full(buf.Buf) && !Empty(buf.Next) { + r := io.LimitReader(buf.Next, Gap(buf.Buf)) + if _, err := io.Copy(buf.Buf, r); err != nil && err != io.EOF { + return + } + } +} + +func (buf *chain) Read(p []byte) (n int, err error) { + n, err = buf.Buf.Read(p) + if len(p[n:]) > 0 && (err == nil || err == io.EOF) { + m, err := buf.Next.Read(p[n:]) + n += m + if err != nil { + return n, err + } + } + + buf.Defrag() + + return n, err +} + +func (buf *chain) ReadAt(p []byte, off int64) (n int, err error) { + if buf.Buf.Len() < off { + return buf.Next.ReadAt(p, off-buf.Buf.Len()) + } + + n, err = buf.Buf.ReadAt(p, off) + if len(p[n:]) > 0 && (err == nil || err == io.EOF) { + var m int + m, err = buf.Next.ReadAt(p[n:], 0) + n += m + } + return n, err +} + +func (buf *chain) Write(p []byte) (n int, err error) { + if n, err = buf.Buf.Write(p); err == io.ErrShortWrite { + err = nil + } + p = p[n:] + if len(p) > 0 && err == nil { + m, err := buf.Next.Write(p) + n += m + if err != nil { + return n, err + } + } + return n, err +} + +func (buf *chain) WriteAt(p []byte, off int64) (n int, err error) { + switch { + case buf.Buf.Cap() <= off: // past the end + return buf.Next.WriteAt(p, off-buf.Buf.Cap()) + + case buf.Buf.Cap() >= off+int64(len(p)): // fits in + return buf.Buf.WriteAt(p, off) + + default: // partial fit + n, err = buf.Buf.WriteAt(p, off) + if len(p[n:]) > 0 && (err == nil || err == io.ErrShortWrite) { + var m int + m, err = buf.Next.WriteAt(p[n:], 0) + n += m + } + return n, err + } +} + +func init() { + gob.Register(&chain{}) + gob.Register(&nopBufferAt{}) +} + +func (buf *chain) MarshalBinary() ([]byte, error) { + b := bytes.NewBuffer(nil) + enc := gob.NewEncoder(b) + if err := enc.Encode(&buf.Buf); err != nil { + return nil, err + } + if err := enc.Encode(&buf.Next); err != nil { + return nil, err + } + return b.Bytes(), nil +} + +func (buf *chain) UnmarshalBinary(data []byte) error { + b := bytes.NewBuffer(data) + dec := gob.NewDecoder(b) + if err := dec.Decode(&buf.Buf); err != nil { + return err + } + if err := dec.Decode(&buf.Next); err != nil { + return err + } + return nil +} diff --git a/vendor/github.com/djherbis/buffer/partition.go b/vendor/github.com/djherbis/buffer/partition.go new file mode 100644 index 0000000000000..1726d20f84409 --- /dev/null +++ b/vendor/github.com/djherbis/buffer/partition.go @@ -0,0 +1,101 @@ +package buffer + +import ( + "encoding/gob" + "io" + "math" +) + +type partition struct { + List + Pool +} + +// NewPartition returns a Buffer which uses a Pool to extend or shrink its size as needed. +// It automatically allocates new buffers with pool.Get() to extend is length, and +// pool.Put() to release unused buffers as it shrinks. +func NewPartition(pool Pool, buffers ...Buffer) Buffer { + return &partition{ + Pool: pool, + List: buffers, + } +} + +func (buf *partition) Cap() int64 { + return math.MaxInt64 +} + +func (buf *partition) Read(p []byte) (n int, err error) { + for len(p) > 0 { + + if len(buf.List) == 0 { + return n, io.EOF + } + + buffer := buf.List[0] + + if Empty(buffer) { + buf.Pool.Put(buf.Pop()) + continue + } + + m, er := buffer.Read(p) + n += m + p = p[m:] + + if er != nil && er != io.EOF { + return n, er + } + + } + return n, nil +} + +func (buf *partition) grow() error { + next, err := buf.Pool.Get() + if err != nil { + return err + } + buf.Push(next) + return nil +} + +func (buf *partition) Write(p []byte) (n int, err error) { + for len(p) > 0 { + + if len(buf.List) == 0 { + if err := buf.grow(); err != nil { + return n, err + } + } + + buffer := buf.List[len(buf.List)-1] + + if Full(buffer) { + if err := buf.grow(); err != nil { + return n, err + } + continue + } + + m, er := buffer.Write(p) + n += m + p = p[m:] + + if er != nil && er != io.ErrShortWrite { + return n, er + } + + } + return n, nil +} + +func (buf *partition) Reset() { + for len(buf.List) > 0 { + buf.Pool.Put(buf.Pop()) + } +} + +func init() { + gob.Register(&partition{}) +} diff --git a/vendor/github.com/djherbis/buffer/partition_at.go b/vendor/github.com/djherbis/buffer/partition_at.go new file mode 100644 index 0000000000000..56b44e8d27de3 --- /dev/null +++ b/vendor/github.com/djherbis/buffer/partition_at.go @@ -0,0 +1,187 @@ +package buffer + +import ( + "encoding/gob" + "errors" + "io" + "math" +) + +type partitionAt struct { + ListAt + PoolAt +} + +// NewPartitionAt returns a BufferAt which uses a PoolAt to extend or shrink its size as needed. +// It automatically allocates new buffers with pool.Get() to extend is length, and +// pool.Put() to release unused buffers as it shrinks. +func NewPartitionAt(pool PoolAt, buffers ...BufferAt) BufferAt { + return &partitionAt{ + PoolAt: pool, + ListAt: buffers, + } +} + +func (buf *partitionAt) Cap() int64 { + return math.MaxInt64 +} + +func (buf *partitionAt) Read(p []byte) (n int, err error) { + for len(p) > 0 { + + if len(buf.ListAt) == 0 { + return n, io.EOF + } + + buffer := buf.ListAt[0] + + if Empty(buffer) { + buf.PoolAt.Put(buf.Pop()) + continue + } + + m, er := buffer.Read(p) + n += m + p = p[m:] + + if er != nil && er != io.EOF { + return n, er + } + + } + return n, nil +} + +func (buf *partitionAt) ReadAt(p []byte, off int64) (n int, err error) { + if off < 0 { + return 0, errors.New("buffer.PartionAt.ReadAt: negative offset") + } + for _, buffer := range buf.ListAt { + // Find the buffer where this offset is found. + if buffer.Len() <= off { + off -= buffer.Len() + continue + } + + m, er := buffer.ReadAt(p, off) + n += m + p = p[m:] + + if er != nil && er != io.EOF { + return n, er + } + if len(p) == 0 { + return n, er + } + // We need to read more, starting from 0 in the next buffer. + off = 0 + } + if len(p) > 0 { + return n, io.EOF + } + return n, nil +} + +func (buf *partitionAt) grow() error { + next, err := buf.PoolAt.Get() + if err != nil { + return err + } + buf.Push(next) + return nil +} + +func (buf *partitionAt) Write(p []byte) (n int, err error) { + for len(p) > 0 { + + if len(buf.ListAt) == 0 { + if err := buf.grow(); err != nil { + return n, err + } + } + + buffer := buf.ListAt[len(buf.ListAt)-1] + + if Full(buffer) { + if err := buf.grow(); err != nil { + return n, err + } + continue + } + + m, er := buffer.Write(p) + n += m + p = p[m:] + + if er != nil && er != io.ErrShortWrite { + return n, er + } + + } + return n, nil +} + +func (buf *partitionAt) WriteAt(p []byte, off int64) (n int, err error) { + if off < 0 { + return 0, errors.New("buffer.PartionAt.WriteAt: negative offset") + } + if off == buf.Len() { // writing at the end special case + if err := buf.grow(); err != nil { + return 0, err + } + } + fitCheck := BufferAt.Len + for i := 0; i < len(buf.ListAt); i++ { + buffer := buf.ListAt[i] + + // Find the buffer where this offset is found. + if fitCheck(buffer) < off { + off -= fitCheck(buffer) + continue + } + + if i+1 == len(buf.ListAt) { + fitCheck = BufferAt.Cap + } + + endOff := off + int64(len(p)) + if fitCheck(buffer) >= endOff { + // Everything should fit. + return buffer.WriteAt(p, off) + } + + // Assume it won't all fit, only write what should fit. + canFit := int(fitCheck(buffer) - off) + if len(p[:canFit]) > 0 { + var m int + m, err = buffer.WriteAt(p[:canFit], off) + n += m + p = p[m:] + } + off = 0 // All writes are at offset 0 of following buffers now. + + if err != nil || len(p) == 0 { + return n, err + } + if i+1 == len(buf.ListAt) { + if err := buf.grow(); err != nil { + return 0, err + } + fitCheck = BufferAt.Cap + } + } + if len(p) > 0 { + err = io.ErrShortWrite + } + return n, err +} + +func (buf *partitionAt) Reset() { + for len(buf.ListAt) > 0 { + buf.PoolAt.Put(buf.Pop()) + } +} + +func init() { + gob.Register(&partitionAt{}) +} diff --git a/vendor/github.com/djherbis/buffer/pool.go b/vendor/github.com/djherbis/buffer/pool.go new file mode 100644 index 0000000000000..0e57de8406bf4 --- /dev/null +++ b/vendor/github.com/djherbis/buffer/pool.go @@ -0,0 +1,111 @@ +package buffer + +import ( + "bytes" + "encoding/binary" + "encoding/gob" + "io/ioutil" + "os" + "sync" +) + +// Pool provides a way to Allocate and Release Buffer objects +// Pools mut be concurrent-safe for calls to Get() and Put(). +type Pool interface { + Get() (Buffer, error) // Allocate a Buffer + Put(buf Buffer) error // Release or Reuse a Buffer +} + +type pool struct { + pool sync.Pool +} + +// NewPool returns a Pool(), it's backed by a sync.Pool so its safe for concurrent use. +// Get() and Put() errors will always be nil. +// It will not work with gob. +func NewPool(New func() Buffer) Pool { + return &pool{ + pool: sync.Pool{ + New: func() interface{} { + return New() + }, + }, + } +} + +func (p *pool) Get() (Buffer, error) { + return p.pool.Get().(Buffer), nil +} + +func (p *pool) Put(buf Buffer) error { + buf.Reset() + p.pool.Put(buf) + return nil +} + +type memPool struct { + N int64 + Pool +} + +// NewMemPool returns a Pool, Get() returns an in memory buffer of max size N. +// Put() returns the buffer to the pool after resetting it. +// Get() and Put() errors will always be nil. +func NewMemPool(N int64) Pool { + return &memPool{ + N: N, + Pool: NewPool(func() Buffer { + return New(N) + }), + } +} + +func (m *memPool) MarshalBinary() ([]byte, error) { + buf := bytes.NewBuffer(nil) + err := binary.Write(buf, binary.LittleEndian, m.N) + return buf.Bytes(), err +} + +func (m *memPool) UnmarshalBinary(data []byte) error { + buf := bytes.NewReader(data) + err := binary.Read(buf, binary.LittleEndian, &m.N) + m.Pool = NewPool(func() Buffer { + return New(m.N) + }) + return err +} + +type filePool struct { + N int64 + Directory string +} + +// NewFilePool returns a Pool, Get() returns a file-based buffer of max size N. +// Put() closes and deletes the underlying file for the buffer. +// Get() may return an error if it fails to create a file for the buffer. +// Put() may return an error if it fails to delete the file. +func NewFilePool(N int64, dir string) Pool { + return &filePool{N: N, Directory: dir} +} + +func (p *filePool) Get() (Buffer, error) { + file, err := ioutil.TempFile(p.Directory, "buffer") + if err != nil { + return nil, err + } + return NewFile(p.N, file), nil +} + +func (p *filePool) Put(buf Buffer) (err error) { + buf.Reset() + if fileBuf, ok := buf.(*fileBuffer); ok { + fileBuf.file.Close() + err = os.Remove(fileBuf.file.Name()) + } + return err +} + +func init() { + gob.Register(&memPool{}) + gob.Register(&filePool{}) +} diff --git a/vendor/github.com/djherbis/buffer/pool_at.go b/vendor/github.com/djherbis/buffer/pool_at.go new file mode 100644 index 0000000000000..23635ad128fcd --- /dev/null +++ b/vendor/github.com/djherbis/buffer/pool_at.go @@ -0,0 +1,111 @@ +package buffer + +import ( + "bytes" + "encoding/binary" + "encoding/gob" + "io/ioutil" + "os" + "sync" +) + +// PoolAt provides a way to Allocate and Release BufferAt objects +// PoolAt's mut be concurrent-safe for calls to Get() and Put(). +type PoolAt interface { + Get() (BufferAt, error) // Allocate a BufferAt + Put(buf BufferAt) error // Release or Reuse a BufferAt +} + +type poolAt struct { + poolAt sync.Pool +} + +// NewPoolAt returns a PoolAt(), it's backed by a sync.Pool so its safe for concurrent use. +// Get() and Put() errors will always be nil. +// It will not work with gob. +func NewPoolAt(New func() BufferAt) PoolAt { + return &poolAt{ + poolAt: sync.Pool{ + New: func() interface{} { + return New() + }, + }, + } +} + +func (p *poolAt) Get() (BufferAt, error) { + return p.poolAt.Get().(BufferAt), nil +} + +func (p *poolAt) Put(buf BufferAt) error { + buf.Reset() + p.poolAt.Put(buf) + return nil +} + +type memPoolAt struct { + N int64 + PoolAt +} + +// NewMemPoolAt returns a PoolAt, Get() returns an in memory buffer of max size N. +// Put() returns the buffer to the pool after resetting it. +// Get() and Put() errors will always be nil. +func NewMemPoolAt(N int64) PoolAt { + return &memPoolAt{ + N: N, + PoolAt: NewPoolAt(func() BufferAt { + return New(N) + }), + } +} + +func (m *memPoolAt) MarshalBinary() ([]byte, error) { + buf := bytes.NewBuffer(nil) + err := binary.Write(buf, binary.LittleEndian, m.N) + return buf.Bytes(), err +} + +func (m *memPoolAt) UnmarshalBinary(data []byte) error { + buf := bytes.NewReader(data) + err := binary.Read(buf, binary.LittleEndian, &m.N) + m.PoolAt = NewPoolAt(func() BufferAt { + return New(m.N) + }) + return err +} + +type filePoolAt struct { + N int64 + Directory string +} + +// NewFilePoolAt returns a PoolAt, Get() returns a file-based buffer of max size N. +// Put() closes and deletes the underlying file for the buffer. +// Get() may return an error if it fails to create a file for the buffer. +// Put() may return an error if it fails to delete the file. +func NewFilePoolAt(N int64, dir string) PoolAt { + return &filePoolAt{N: N, Directory: dir} +} + +func (p *filePoolAt) Get() (BufferAt, error) { + file, err := ioutil.TempFile(p.Directory, "buffer") + if err != nil { + return nil, err + } + return NewFile(p.N, file), nil +} + +func (p *filePoolAt) Put(buf BufferAt) (err error) { + buf.Reset() + if fileBuf, ok := buf.(*fileBuffer); ok { + fileBuf.file.Close() + err = os.Remove(fileBuf.file.Name()) + } + return err +} + +func init() { + gob.Register(&memPoolAt{}) + gob.Register(&filePoolAt{}) +} diff --git a/vendor/github.com/djherbis/buffer/ring.go b/vendor/github.com/djherbis/buffer/ring.go new file mode 100644 index 0000000000000..7da7f0a70b3a1 --- /dev/null +++ b/vendor/github.com/djherbis/buffer/ring.go @@ -0,0 +1,58 @@ +package buffer + +import ( + "io" + "math" + + "github.com/djherbis/buffer/wrapio" +) + +type ring struct { + BufferAt + L int64 + *wrapio.WrapReader + *wrapio.WrapWriter +} + +// NewRing returns a Ring Buffer from a BufferAt. +// It overwrites old data in the Buffer when needed (when its full). +func NewRing(buffer BufferAt) Buffer { + return &ring{ + BufferAt: buffer, + WrapReader: wrapio.NewWrapReader(buffer, 0, buffer.Cap()), + WrapWriter: wrapio.NewWrapWriter(buffer, 0, buffer.Cap()), + } +} + +func (buf *ring) Len() int64 { + return buf.L +} + +func (buf *ring) Cap() int64 { + return math.MaxInt64 +} + +func (buf *ring) Read(p []byte) (n int, err error) { + if buf.L == buf.BufferAt.Cap() { + buf.WrapReader.Seek(buf.WrapWriter.Offset(), 0) + } + n, err = io.LimitReader(buf.WrapReader, buf.L).Read(p) + buf.L -= int64(n) + return n, err +} + +func (buf *ring) Write(p []byte) (n int, err error) { + n, err = buf.WrapWriter.Write(p) + buf.L += int64(n) + if buf.L > buf.BufferAt.Cap() { + buf.L = buf.BufferAt.Cap() + } + return n, err +} + +func (buf *ring) Reset() { + buf.BufferAt.Reset() + buf.L = 0 + buf.WrapReader = wrapio.NewWrapReader(buf.BufferAt, 0, buf.BufferAt.Cap()) + buf.WrapWriter = wrapio.NewWrapWriter(buf.BufferAt, 0, buf.BufferAt.Cap()) +} diff --git a/vendor/github.com/djherbis/buffer/spill.go b/vendor/github.com/djherbis/buffer/spill.go new file mode 100644 index 0000000000000..44d618b549f7d --- /dev/null +++ b/vendor/github.com/djherbis/buffer/spill.go @@ -0,0 +1,41 @@ +package buffer + +import ( + "encoding/gob" + "io" + "io/ioutil" + "math" +) + +type spill struct { + Buffer + Spiller io.Writer +} + +// NewSpill returns a Buffer which writes data to w when there's an error +// writing to buf. Such as when buf is full, or the disk is full, etc. +func NewSpill(buf Buffer, w io.Writer) Buffer { + if w == nil { + w = ioutil.Discard + } + return &spill{ + Buffer: buf, + Spiller: w, + } +} + +func (buf *spill) Cap() int64 { + return math.MaxInt64 +} + +func (buf *spill) Write(p []byte) (n int, err error) { + if n, err = buf.Buffer.Write(p); err != nil { + m, err := buf.Spiller.Write(p[n:]) + return m + n, err + } + return len(p), nil +} + +func init() { + gob.Register(&spill{}) +} diff --git a/vendor/github.com/djherbis/buffer/swap.go b/vendor/github.com/djherbis/buffer/swap.go new file mode 100644 index 0000000000000..bdb11a461a599 --- /dev/null +++ b/vendor/github.com/djherbis/buffer/swap.go @@ -0,0 +1,99 @@ +package buffer + +import ( + "encoding/gob" + "io" +) + +type swap struct { + A BufferAt + B BufferAt +} + +// NewSwap creates a Buffer which writes to a until you write past a.Cap() +// then it io.Copy's from a to b and writes to b. +// Once the Buffer is empty again, it starts over writing to a. +// Note that if b.Cap() <= a.Cap() it will cause a panic, b is expected +// to be larger in order to accommodate writes past a.Cap(). +func NewSwap(a, b Buffer) Buffer { + return NewSwapAt(toBufferAt(a), toBufferAt(b)) +} + +// NewSwapAt creates a BufferAt which writes to a until you write past a.Cap() +// then it io.Copy's from a to b and writes to b. +// Once the Buffer is empty again, it starts over writing to a. +// Note that if b.Cap() <= a.Cap() it will cause a panic, b is expected +// to be larger in order to accommodate writes past a.Cap(). +func NewSwapAt(a, b BufferAt) BufferAt { + if b.Cap() <= a.Cap() { + panic("Buffer b must be larger than a.") + } + return &swap{A: a, B: b} +} + +func (buf *swap) Len() int64 { + return buf.A.Len() + buf.B.Len() +} + +func (buf *swap) Cap() int64 { + return buf.B.Cap() +} + +func (buf *swap) Read(p []byte) (n int, err error) { + if buf.A.Len() > 0 { + return buf.A.Read(p) + } + return buf.B.Read(p) +} + +func (buf *swap) ReadAt(p []byte, off int64) (n int, err error) { + if buf.A.Len() > 0 { + return buf.A.ReadAt(p, off) + } + return buf.B.ReadAt(p, off) +} + +func (buf *swap) Write(p []byte) (n int, err error) { + switch { + case buf.B.Len() > 0: + n, err = buf.B.Write(p) + + case buf.A.Len()+int64(len(p)) > buf.A.Cap(): + _, err = io.Copy(buf.B, buf.A) + if err == nil { + n, err = buf.B.Write(p) + } + + default: + n, err = buf.A.Write(p) + } + + return n, err +} + +func (buf *swap) WriteAt(p []byte, off int64) (n int, err error) { + switch { + case buf.B.Len() > 0: + n, err = buf.B.WriteAt(p, off) + + case off+int64(len(p)) > buf.A.Cap(): + _, err = io.Copy(buf.B, buf.A) + if err == nil { + n, err = buf.B.WriteAt(p, off) + } + + default: + n, err = buf.A.WriteAt(p, off) + } + + return n, err +} + +func (buf *swap) Reset() { + buf.A.Reset() + buf.B.Reset() +} + +func init() { + gob.Register(&swap{}) +} diff --git a/vendor/github.com/djherbis/buffer/wrapio/limitwrap.go b/vendor/github.com/djherbis/buffer/wrapio/limitwrap.go new file mode 100644 index 0000000000000..7af9750272270 --- /dev/null +++ b/vendor/github.com/djherbis/buffer/wrapio/limitwrap.go @@ -0,0 +1,94 @@ +package wrapio + +import ( + "encoding/gob" + "io" + + "github.com/djherbis/buffer/limio" +) + +// ReadWriterAt implements io.ReaderAt and io.WriterAt +type ReadWriterAt interface { + io.ReaderAt + io.WriterAt +} + +// Wrapper implements a io.ReadWriter and ReadWriterAt such that +// when reading/writing goes past N bytes, it "wraps" back to the beginning. +type Wrapper struct { + // N is the offset at which to "wrap" back to the start + N int64 + // L is the length of the data written + L int64 + // O is our offset in the data + O int64 + rwa ReadWriterAt +} + +// NewWrapper creates a Wrapper based on ReadWriterAt rwa. +// L is the current length, O is the current offset, and N is offset at which we "wrap". +func NewWrapper(rwa ReadWriterAt, L, O, N int64) *Wrapper { + return &Wrapper{ + L: L, + O: O, + N: N, + rwa: rwa, + } +} + +// Len returns the # of bytes in the Wrapper +func (wpr *Wrapper) Len() int64 { + return wpr.L +} + +// Cap returns the "wrap" offset (max # of bytes) +func (wpr *Wrapper) Cap() int64 { + return wpr.N +} + +// Reset seeks to the start (0 offset), and sets the length to 0. +func (wpr *Wrapper) Reset() { + wpr.O = 0 + wpr.L = 0 +} + +// SetReadWriterAt lets you switch the underlying Read/WriterAt +func (wpr *Wrapper) SetReadWriterAt(rwa ReadWriterAt) { + wpr.rwa = rwa +} + +// Read reads from the current offset into p, wrapping at Cap() +func (wpr *Wrapper) Read(p []byte) (n int, err error) { + n, err = wpr.ReadAt(p, 0) + wpr.L -= int64(n) + wpr.O += int64(n) + wpr.O %= wpr.N + return n, err +} + +// ReadAt reads from the current offset+off into p, wrapping at Cap() +func (wpr *Wrapper) ReadAt(p []byte, off int64) (n int, err error) { + wrap := NewWrapReader(wpr.rwa, wpr.O+off, wpr.N) + r := io.LimitReader(wrap, wpr.L-off) + return r.Read(p) +} + +// Write writes p to the end of the Wrapper (at Len()), wrapping at Cap() +func (wpr *Wrapper) Write(p []byte) (n int, err error) { + return wpr.WriteAt(p, wpr.L) +} + +// WriteAt writes p at the current offset+off, wrapping at Cap() +func (wpr *Wrapper) WriteAt(p []byte, off int64) (n int, err error) { + wrap := NewWrapWriter(wpr.rwa, wpr.O+off, wpr.N) + w := limio.LimitWriter(wrap, wpr.N-off) + n, err = w.Write(p) + if wpr.L < off+int64(n) { + wpr.L = int64(n) + off + } + return n, err +} + +func init() { + gob.Register(&Wrapper{}) +} diff --git a/vendor/github.com/djherbis/buffer/wrapio/wrap.go b/vendor/github.com/djherbis/buffer/wrapio/wrap.go new file mode 100644 index 0000000000000..b60a6b1392c2b --- /dev/null +++ b/vendor/github.com/djherbis/buffer/wrapio/wrap.go @@ -0,0 +1,139 @@ +package wrapio + +import "io" + +// DoerAt is a common interface for wrappers WriteAt or ReadAt functions +type DoerAt interface { + DoAt([]byte, int64) (int, error) +} + +// DoAtFunc is implemented by ReadAt/WriteAt +type DoAtFunc func([]byte, int64) (int, error) + +type wrapper struct { + off int64 + wrapAt int64 + doat DoAtFunc +} + +func (w *wrapper) Offset() int64 { + return w.off +} + +func (w *wrapper) Seek(offset int64, whence int) (int64, error) { + switch whence { + case 0: + w.off = offset + case 1: + w.off += offset + case 2: + w.off = (w.wrapAt + offset) + } + w.off %= w.wrapAt + return w.off, nil +} + +func (w *wrapper) DoAt(p []byte, off int64) (n int, err error) { + return w.doat(p, off) +} + +// WrapWriter wraps writes around a section of data. +type WrapWriter struct { + *wrapper +} + +// NewWrapWriter creates a WrapWriter starting at offset off, and wrapping at offset wrapAt. +func NewWrapWriter(w io.WriterAt, off int64, wrapAt int64) *WrapWriter { + return &WrapWriter{ + &wrapper{ + doat: w.WriteAt, + off: (off % wrapAt), + wrapAt: wrapAt, + }, + } +} + +// Write writes p starting at the current offset, wrapping when it reaches the end. +// The current offset is shifted forward by the amount written. +func (w *WrapWriter) Write(p []byte) (n int, err error) { + n, err = Wrap(w, p, w.off, w.wrapAt) + w.off = (w.off + int64(n)) % w.wrapAt + return n, err +} + +// WriteAt writes p starting at offset off, wrapping when it reaches the end. +func (w *WrapWriter) WriteAt(p []byte, off int64) (n int, err error) { + return Wrap(w, p, off, w.wrapAt) +} + +// WrapReader wraps reads around a section of data. +type WrapReader struct { + *wrapper +} + +// NewWrapReader creates a WrapReader starting at offset off, and wrapping at offset wrapAt. +func NewWrapReader(r io.ReaderAt, off int64, wrapAt int64) *WrapReader { + return &WrapReader{ + &wrapper{ + doat: r.ReadAt, + off: (off % wrapAt), + wrapAt: wrapAt, + }, + } +} + +// Read reads into p starting at the current offset, wrapping if it reaches the end. +// The current offset is shifted forward by the amount read. +func (r *WrapReader) Read(p []byte) (n int, err error) { + n, err = Wrap(r, p, r.off, r.wrapAt) + r.off = (r.off + int64(n)) % r.wrapAt + return n, err +} + +// ReadAt reads into p starting at the current offset, wrapping when it reaches the end. +func (r *WrapReader) ReadAt(p []byte, off int64) (n int, err error) { + return Wrap(r, p, off, r.wrapAt) +} + +// maxConsecutiveEmptyActions determines how many consecutive empty reads/writes can occur before giving up +const maxConsecutiveEmptyActions = 100 + +// Wrap causes an action on an array of bytes (like read/write) to be done from an offset off, +// wrapping at offset wrapAt. +func Wrap(w DoerAt, p []byte, off int64, wrapAt int64) (n int, err error) { + var m, fails int + + off %= wrapAt + + for len(p) > 0 { + + if off+int64(len(p)) < wrapAt { + m, err = w.DoAt(p, off) + } else { + space := wrapAt - off + m, err = w.DoAt(p[:space], off) + } + + if err != nil && err != io.EOF { + return n + m, err + } + + switch m { + case 0: + fails++ + default: + fails = 0 + } + + if fails > maxConsecutiveEmptyActions { + return n + m, io.ErrNoProgress + } + + n += m + p = p[m:] + off += int64(m) + off %= wrapAt + } + + return n, err +} diff --git a/vendor/github.com/djherbis/nio/v3/.travis.yml b/vendor/github.com/djherbis/nio/v3/.travis.yml new file mode 100644 index 0000000000000..06dc07b8958e3 --- /dev/null +++ b/vendor/github.com/djherbis/nio/v3/.travis.yml @@ -0,0 +1,22 @@ +language: go +go: +- tip +before_install: +- go get -u golang.org/x/lint/golint +- go get github.com/axw/gocov/gocov +- go get github.com/mattn/goveralls +- if ! go get code.google.com/p/go.tools/cmd/cover; then go get golang.org/x/tools/cmd/cover; + fi +script: +- '[ "${TRAVIS_PULL_REQUEST}" != "false" ] || $HOME/gopath/bin/goveralls -service=travis-ci + -repotoken $COVERALLS_TOKEN' +- "$HOME/gopath/bin/golint ./..." +- go vet +- go test -bench=.* -v ./... +notifications: + email: + on_success: never + on_failure: change +env: + global: + secure: gpKsimMN5YScLnbcoWvJPw8VL+qCpZgnC4i8mFn/lRX5Ta9FhDMROQre0Ko4bU9RX/u/IBL1fO/IyaVtVWQ0fhsDi+ovrh3LgzewwZBgz7FGiyFpagvf91Jwq5Yus15QQZ8MebrQ41H1YiWMdLOHlZdN6gNb0cswg3w4MRjbGb4= diff --git a/vendor/github.com/djherbis/nio/v3/LICENSE.txt b/vendor/github.com/djherbis/nio/v3/LICENSE.txt new file mode 100644 index 0000000000000..f5daa194f7803 --- /dev/null +++ b/vendor/github.com/djherbis/nio/v3/LICENSE.txt @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2015 Dustin H + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/djherbis/nio/v3/README.md b/vendor/github.com/djherbis/nio/v3/README.md new file mode 100644 index 0000000000000..8dc3165539265 --- /dev/null +++ b/vendor/github.com/djherbis/nio/v3/README.md @@ -0,0 +1,65 @@ +nio +========== + +[![GoDoc](https://godoc.org/github.com/djherbis/nio?status.svg)](https://godoc.org/github.com/djherbis/nio) +[![Release](https://img.shields.io/github/release/djherbis/nio.svg)](https://github.com/djherbis/nio/releases/latest) +[![Software License](https://img.shields.io/badge/license-MIT-brightgreen.svg)](LICENSE.txt) +[![Build Status](https://travis-ci.org/djherbis/nio.svg)](https://travis-ci.org/djherbis/nio) +[![Coverage Status](https://coveralls.io/repos/djherbis/nio/badge.svg?branch=master)](https://coveralls.io/r/djherbis/nio?branch=master) +[![Go Report Card](https://goreportcard.com/badge/github.com/djherbis/nio)](https://goreportcard.com/report/github.com/djherbis/nio) + +Usage +----- + +The Buffer interface: + +```go +type Buffer interface { + Len() int64 + Cap() int64 + io.ReadWriter +} + +``` + +nio's Copy method concurrently copies from an io.Reader to a supplied nio.Buffer, +then from the nio.Buffer to an io.Writer. This way, blocking writes don't slow the io.Reader. + +```go +import ( + "github.com/djherbis/buffer" + "github.com/djherbis/nio" +) + +buf := buffer.New(32*1024) // 32KB In memory Buffer +nio.Copy(w, r, buf) // Reads and Writes concurrently, buffering using buf. +``` + +nio's Pipe method is a buffered version of io.Pipe +The writer return once its data has been written to the Buffer. +The reader returns with data off the Buffer. + +```go +import ( + "gopkg.in/djherbis/buffer.v1" + "gopkg.in/djherbis/nio.v2" +) + +buf := buffer.New(32*1024) // 32KB In memory Buffer +r, w := nio.Pipe(buf) +``` + +Installation +------------ +```sh +go get gopkg.in/djherbis/nio.v2 +``` + +For some pre-built buffers grab: +```sh +go get gopkg.in/djherbis/buffer.v1 +``` + +Mentions +------------ +[GopherCon 2017: Peter Bourgon - Evolutionary Optimization with Go](https://www.youtube.com/watch?v=ha8gdZ27wMo&start=2077&end=2140) diff --git a/vendor/github.com/djherbis/nio/v3/go.mod b/vendor/github.com/djherbis/nio/v3/go.mod new file mode 100644 index 0000000000000..a5f602451e2ae --- /dev/null +++ b/vendor/github.com/djherbis/nio/v3/go.mod @@ -0,0 +1,5 @@ +module github.com/djherbis/nio/v3 + +go 1.16 + +require github.com/djherbis/buffer v1.1.0 diff --git a/vendor/github.com/djherbis/nio/v3/go.sum b/vendor/github.com/djherbis/nio/v3/go.sum new file mode 100644 index 0000000000000..f4578ade83cf8 --- /dev/null +++ b/vendor/github.com/djherbis/nio/v3/go.sum @@ -0,0 +1,2 @@ +github.com/djherbis/buffer v1.1.0 h1:uGQ+DZDAMlfC2z3khbBtLcAHC0wyoNrX9lpOml3g3fg= +github.com/djherbis/buffer v1.1.0/go.mod h1:VwN8VdFkMY0DCALdY8o00d3IZ6Amz/UNVMWcSaJT44o= diff --git a/vendor/github.com/djherbis/nio/v3/nio.go b/vendor/github.com/djherbis/nio/v3/nio.go new file mode 100644 index 0000000000000..ab476e62d2f99 --- /dev/null +++ b/vendor/github.com/djherbis/nio/v3/nio.go @@ -0,0 +1,53 @@ +// Package nio provides a few buffered io primitives. +package nio + +import "io" + +// Buffer is used to store bytes. +type Buffer interface { + // Len returns how many bytes are buffered + Len() int64 + + // Cap returns how many bytes can in the buffer at a time + Cap() int64 + + // ReadWriter writes are stored in the buffer, reads return the stored data + io.ReadWriter +} + +// Pipe creates a buffered pipe. +// It can be used to connect code expecting an io.Reader with code expecting an io.Writer. +// Reads on one end read from the supplied Buffer. Writes write to the supplied Buffer. +// It is safe to call Read and Write in parallel with each other or with Close. +// Close will complete once pending I/O is done, and may cancel blocking Read/Writes. +// Buffered data will still be available to Read after the Writer has been closed. +// Parallel calls to Read, and parallel calls to Write are also safe : +// the individual calls will be gated sequentially. +func Pipe(buf Buffer) (r *PipeReader, w *PipeWriter) { + p := newBufferedPipe(buf) + r = &PipeReader{bufpipe: p} + w = &PipeWriter{bufpipe: p} + return r, w +} + +// Copy copies from src to buf, and from buf to dst in parallel until +// either EOF is reached on src or an error occurs. It returns the number of bytes +// copied to dst and the first error encountered while copying, if any. +// EOF is not considered to be an error. If src implements WriterTo, it is used to +// write to the supplied Buffer. If dst implements ReaderFrom, it is used to read from +// the supplied Buffer. +func Copy(dst io.Writer, src io.Reader, buf Buffer) (n int64, err error) { + return io.Copy(dst, NewReader(src, buf)) +} + +// NewReader reads from the buffer which is concurrently filled with data from the passed src. +func NewReader(src io.Reader, buf Buffer) io.ReadCloser { + r, w := Pipe(buf) + + go func() { + _, err := io.Copy(w, src) + w.CloseWithError(err) + }() + + return r +} diff --git a/vendor/github.com/djherbis/nio/v3/sync.go b/vendor/github.com/djherbis/nio/v3/sync.go new file mode 100644 index 0000000000000..fec538381a60d --- /dev/null +++ b/vendor/github.com/djherbis/nio/v3/sync.go @@ -0,0 +1,177 @@ +package nio + +import ( + "io" + "sync" +) + +// PipeReader is the read half of the pipe. +type PipeReader struct { + *bufpipe +} + +// CloseWithError closes the reader; subsequent writes to the write half of the pipe will return the error err. +func (r *PipeReader) CloseWithError(err error) error { + if err == nil { + err = io.ErrClosedPipe + } + r.bufpipe.l.Lock() + defer r.bufpipe.l.Unlock() + if r.bufpipe.rerr == nil { + r.bufpipe.rerr = err + r.bufpipe.rwait.Signal() + r.bufpipe.wwait.Signal() + } + return nil +} + +// Close closes the reader; subsequent writes to the write half of the pipe will return the error io.ErrClosedPipe. +func (r *PipeReader) Close() error { + return r.CloseWithError(nil) +} + +// A PipeWriter is the write half of a pipe. +type PipeWriter struct { + *bufpipe +} + +// CloseWithError closes the writer; once the buffer is empty subsequent reads from the read half of the pipe will return +// no bytes and the error err, or io.EOF if err is nil. CloseWithError always returns nil. +func (w *PipeWriter) CloseWithError(err error) error { + if err == nil { + err = io.EOF + } + w.bufpipe.l.Lock() + defer w.bufpipe.l.Unlock() + if w.bufpipe.werr == nil { + w.bufpipe.werr = err + w.bufpipe.rwait.Signal() + w.bufpipe.wwait.Signal() + } + return nil +} + +// Close closes the writer; once the buffer is empty subsequent reads from the read half of the pipe will return +// no bytes and io.EOF after all the buffer has been read. CloseWithError always returns nil. +func (w *PipeWriter) Close() error { + return w.CloseWithError(nil) +} + +type bufpipe struct { + rl sync.Mutex + wl sync.Mutex + l sync.Mutex + rwait sync.Cond + wwait sync.Cond + b Buffer + rerr error // if reader closed, error to give writes + werr error // if writer closed, error to give reads +} + +func newBufferedPipe(buf Buffer) *bufpipe { + s := &bufpipe{ + b: buf, + } + s.rwait.L = &s.l + s.wwait.L = &s.l + return s +} + +func empty(buf Buffer) bool { + return buf.Len() == 0 +} + +func gap(buf Buffer) int64 { + return buf.Cap() - buf.Len() +} + +func (r *PipeReader) Read(p []byte) (n int, err error) { + r.rl.Lock() + defer r.rl.Unlock() + + r.l.Lock() + defer r.wwait.Signal() + defer r.l.Unlock() + + for empty(r.b) { + if r.rerr != nil { + return 0, io.ErrClosedPipe + } + + if r.werr != nil { + return 0, r.werr + } + + r.wwait.Signal() + r.rwait.Wait() + } + + n, err = r.b.Read(p) + if err == io.EOF { + err = nil + } + + return n, err +} + +func (w *PipeWriter) Write(p []byte) (int, error) { + var m int + var n, space int64 + var err error + sliceLen := int64(len(p)) + + w.wl.Lock() + defer w.wl.Unlock() + + w.l.Lock() + defer w.rwait.Signal() + defer w.l.Unlock() + + if w.werr != nil { + return 0, io.ErrClosedPipe + } + + // while there is data to write + for writeLen := sliceLen; writeLen > 0 && err == nil; writeLen = sliceLen - n { + + // wait for some buffer space to become available (while no errs) + for space = gap(w.b); space == 0 && w.rerr == nil && w.werr == nil; space = gap(w.b) { + w.rwait.Signal() + w.wwait.Wait() + } + + if w.rerr != nil { + err = w.rerr + break + } + + if w.werr != nil { + err = io.ErrClosedPipe + break + } + + // space > 0, and locked + + var nn int64 + if space < writeLen { + // => writeLen - space > 0 + // => (sliceLen - n) - space > 0 + // => sliceLen > n + space + // nn is safe to use for p[:nn] + nn = n + space + } else { + nn = sliceLen + } + + m, err = w.b.Write(p[n:nn]) + n += int64(m) + + // one of the following cases has occurred: + // 1. done writing -> writeLen == 0 + // 2. ran out of buffer space -> gap(w.b) == 0 + // 3. an error occurred err != nil + // all of these cases are handled at the top of this loop + } + + return int(n), err +} diff --git a/vendor/modules.txt b/vendor/modules.txt index c971bcd268328..6949fdb3f281a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -231,6 +231,14 @@ github.com/denisenkom/go-mssqldb/internal/querytext github.com/dgrijalva/jwt-go # github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f github.com/dgryski/go-rendezvous +# github.com/djherbis/buffer v1.2.0 +## explicit +github.com/djherbis/buffer +github.com/djherbis/buffer/limio +github.com/djherbis/buffer/wrapio +# github.com/djherbis/nio/v3 v3.0.1 +## explicit +github.com/djherbis/nio/v3 # github.com/dlclark/regexp2 v1.4.0 github.com/dlclark/regexp2 github.com/dlclark/regexp2/syntax From 196593e2e996aa4a59547629b870701f2b001d9b Mon Sep 17 00:00:00 2001 From: zeripath Date: Sun, 20 Jun 2021 23:39:12 +0100 Subject: [PATCH 03/10] More efficiently parse shas for shaPostProcessor (#16101) * More efficiently parse shas for shaPostProcessor The shaPostProcessor currently repeatedly calls git rev-parse --verify on both backends which is fine if there is only one thing that matches a sha - however if there are multiple things then this becomes wildly inefficient. This PR provides functions for both backends which are much faster to use. Fix #16092 * Add ShaExistCache to RenderContext Signed-off-by: Andrew Thornton Co-authored-by: 6543 <6543@obermui.de> --- modules/git/repo_branch_gogit.go | 24 ++++++++++++++++ modules/git/repo_branch_nogogit.go | 18 ++++++++++++ modules/markup/html.go | 28 ++++++++++++++++-- modules/markup/renderer.go | 46 +++++++++++++++++++++++++----- routers/web/org/home.go | 1 + routers/web/repo/issue.go | 5 ++++ routers/web/repo/milestone.go | 2 ++ routers/web/repo/projects.go | 2 ++ routers/web/repo/release.go | 2 ++ routers/web/repo/view.go | 3 ++ routers/web/user/profile.go | 1 + 11 files changed, 122 insertions(+), 10 deletions(-) diff --git a/modules/git/repo_branch_gogit.go b/modules/git/repo_branch_gogit.go index b00253f6ffd66..e8386b2dbd98f 100644 --- a/modules/git/repo_branch_gogit.go +++ b/modules/git/repo_branch_gogit.go @@ -13,6 +13,30 @@ import ( "github.com/go-git/go-git/v5/plumbing" ) +// IsObjectExist returns true if given reference exists in the repository. +func (repo *Repository) IsObjectExist(name string) bool { + if name == "" { + return false + } + + _, err := repo.gogitRepo.ResolveRevision(plumbing.Revision(name)) + + return err == nil +} + +// IsReferenceExist returns true if given reference exists in the repository. +func (repo *Repository) IsReferenceExist(name string) bool { + if name == "" { + return false + } + + reference, err := repo.gogitRepo.Reference(plumbing.ReferenceName(name), true) + if err != nil { + return false + } + return reference.Type() != plumbing.InvalidReference +} + // IsBranchExist returns true if given branch exists in current repository. func (repo *Repository) IsBranchExist(name string) bool { if name == "" { diff --git a/modules/git/repo_branch_nogogit.go b/modules/git/repo_branch_nogogit.go index 13ddcf06cf65e..dd34e48899034 100644 --- a/modules/git/repo_branch_nogogit.go +++ b/modules/git/repo_branch_nogogit.go @@ -9,10 +9,28 @@ package git import ( "bufio" + "bytes" "io" "strings" ) +// IsObjectExist returns true if given reference exists in the repository. +func (repo *Repository) IsObjectExist(name string) bool { + if name == "" { + return false + } + + wr, rd, cancel := repo.CatFileBatchCheck() + defer cancel() + _, err := wr.Write([]byte(name + "\n")) + if err != nil { + log("Error writing to CatFileBatchCheck %v", err) + return false + } + sha, _, _, err := ReadBatchLine(rd) + return err == nil && bytes.HasPrefix(sha, []byte(strings.TrimSpace(name))) +} + // IsReferenceExist returns true if given reference exists in the repository. func (repo *Repository) IsReferenceExist(name string) bool { if name == "" { diff --git a/modules/markup/html.go b/modules/markup/html.go index 2a83b8716e0fa..edf860da4510d 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -286,6 +286,7 @@ var tagCleaner = regexp.MustCompile(`<((?:/?\w+/\w+)|(?:/[\w ]+/)|(/?[hH][tT][mM var nulCleaner = strings.NewReplacer("\000", "") func postProcess(ctx *RenderContext, procs []processor, input io.Reader, output io.Writer) error { + defer ctx.Cancel() // FIXME: don't read all content to memory rawHTML, err := ioutil.ReadAll(input) if err != nil { @@ -996,6 +997,9 @@ func sha1CurrentPatternProcessor(ctx *RenderContext, node *html.Node) { start := 0 next := node.NextSibling + if ctx.ShaExistCache == nil { + ctx.ShaExistCache = make(map[string]bool) + } for node != nil && node != next && start < len(node.Data) { m := sha1CurrentPattern.FindStringSubmatchIndex(node.Data[start:]) if m == nil { @@ -1013,10 +1017,28 @@ func sha1CurrentPatternProcessor(ctx *RenderContext, node *html.Node) { // as used by git and github for linking and thus we have to do similar. // Because of this, we check to make sure that a matched hash is actually // a commit in the repository before making it a link. - if _, err := git.NewCommand("rev-parse", "--verify", hash).RunInDirBytes(ctx.Metas["repoPath"]); err != nil { - if !strings.Contains(err.Error(), "fatal: Needed a single revision") { - log.Debug("sha1CurrentPatternProcessor git rev-parse: %v", err) + + // check cache first + exist, inCache := ctx.ShaExistCache[hash] + if !inCache { + if ctx.GitRepo == nil { + var err error + ctx.GitRepo, err = git.OpenRepository(ctx.Metas["repoPath"]) + if err != nil { + log.Error("unable to open repository: %s Error: %v", ctx.Metas["repoPath"], err) + return + } + ctx.AddCancel(func() { + ctx.GitRepo.Close() + ctx.GitRepo = nil + }) } + + exist = ctx.GitRepo.IsObjectExist(hash) + ctx.ShaExistCache[hash] = exist + } + + if !exist { start = m[3] continue } diff --git a/modules/markup/renderer.go b/modules/markup/renderer.go index 5d35bd5a67715..d60c8ad71066b 100644 --- a/modules/markup/renderer.go +++ b/modules/markup/renderer.go @@ -13,6 +13,7 @@ import ( "strings" "sync" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/setting" ) @@ -35,13 +36,44 @@ func Init() { // RenderContext represents a render context type RenderContext struct { - Ctx context.Context - Filename string - Type string - IsWiki bool - URLPrefix string - Metas map[string]string - DefaultLink string + Ctx context.Context + Filename string + Type string + IsWiki bool + URLPrefix string + Metas map[string]string + DefaultLink string + GitRepo *git.Repository + ShaExistCache map[string]bool + cancelFn func() +} + +// Cancel runs any cleanup functions that have been registered for this Ctx +func (ctx *RenderContext) Cancel() { + if ctx == nil { + return + } + ctx.ShaExistCache = map[string]bool{} + if ctx.cancelFn == nil { + return + } + ctx.cancelFn() +} + +// AddCancel adds the provided fn as a Cleanup for this Ctx +func (ctx *RenderContext) AddCancel(fn func()) { + if ctx == nil { + return + } + oldCancelFn := ctx.cancelFn + if oldCancelFn == nil { + ctx.cancelFn = fn + return + } + ctx.cancelFn = func() { + defer oldCancelFn() + fn() + } } // Renderer defines an interface for rendering markup file to HTML diff --git a/routers/web/org/home.go b/routers/web/org/home.go index d84ae870ab6db..ad14f18454447 100644 --- a/routers/web/org/home.go +++ b/routers/web/org/home.go @@ -41,6 +41,7 @@ func Home(ctx *context.Context) { desc, err := markdown.RenderString(&markup.RenderContext{ URLPrefix: ctx.Repo.RepoLink, Metas: map[string]string{"mode": "document"}, + GitRepo: ctx.Repo.GitRepo, }, org.Description) if err != nil { ctx.ServerError("RenderString", err) diff --git a/routers/web/repo/issue.go b/routers/web/repo/issue.go index fd2877e7069d6..a7951b6bce180 100644 --- a/routers/web/repo/issue.go +++ b/routers/web/repo/issue.go @@ -1137,6 +1137,7 @@ func ViewIssue(ctx *context.Context) { issue.RenderedContent, err = markdown.RenderString(&markup.RenderContext{ URLPrefix: ctx.Repo.RepoLink, Metas: ctx.Repo.Repository.ComposeMetas(), + GitRepo: ctx.Repo.GitRepo, }, issue.Content) if err != nil { ctx.ServerError("RenderString", err) @@ -1301,6 +1302,7 @@ func ViewIssue(ctx *context.Context) { comment.RenderedContent, err = markdown.RenderString(&markup.RenderContext{ URLPrefix: ctx.Repo.RepoLink, Metas: ctx.Repo.Repository.ComposeMetas(), + GitRepo: ctx.Repo.GitRepo, }, comment.Content) if err != nil { ctx.ServerError("RenderString", err) @@ -1376,6 +1378,7 @@ func ViewIssue(ctx *context.Context) { comment.RenderedContent, err = markdown.RenderString(&markup.RenderContext{ URLPrefix: ctx.Repo.RepoLink, Metas: ctx.Repo.Repository.ComposeMetas(), + GitRepo: ctx.Repo.GitRepo, }, comment.Content) if err != nil { ctx.ServerError("RenderString", err) @@ -1734,6 +1737,7 @@ func UpdateIssueContent(ctx *context.Context) { content, err := markdown.RenderString(&markup.RenderContext{ URLPrefix: ctx.Query("context"), Metas: ctx.Repo.Repository.ComposeMetas(), + GitRepo: ctx.Repo.GitRepo, }, issue.Content) if err != nil { ctx.ServerError("RenderString", err) @@ -2161,6 +2165,7 @@ func UpdateCommentContent(ctx *context.Context) { content, err := markdown.RenderString(&markup.RenderContext{ URLPrefix: ctx.Query("context"), Metas: ctx.Repo.Repository.ComposeMetas(), + GitRepo: ctx.Repo.GitRepo, }, comment.Content) if err != nil { ctx.ServerError("RenderString", err) diff --git a/routers/web/repo/milestone.go b/routers/web/repo/milestone.go index bb6b310cbe8d4..4cdca38dd02b6 100644 --- a/routers/web/repo/milestone.go +++ b/routers/web/repo/milestone.go @@ -88,6 +88,7 @@ func Milestones(ctx *context.Context) { m.RenderedContent, err = markdown.RenderString(&markup.RenderContext{ URLPrefix: ctx.Repo.RepoLink, Metas: ctx.Repo.Repository.ComposeMetas(), + GitRepo: ctx.Repo.GitRepo, }, m.Content) if err != nil { ctx.ServerError("RenderString", err) @@ -280,6 +281,7 @@ func MilestoneIssuesAndPulls(ctx *context.Context) { milestone.RenderedContent, err = markdown.RenderString(&markup.RenderContext{ URLPrefix: ctx.Repo.RepoLink, Metas: ctx.Repo.Repository.ComposeMetas(), + GitRepo: ctx.Repo.GitRepo, }, milestone.Content) if err != nil { ctx.ServerError("RenderString", err) diff --git a/routers/web/repo/projects.go b/routers/web/repo/projects.go index eb0719995cb55..c7490893d5fe6 100644 --- a/routers/web/repo/projects.go +++ b/routers/web/repo/projects.go @@ -81,6 +81,7 @@ func Projects(ctx *context.Context) { projects[i].RenderedContent, err = markdown.RenderString(&markup.RenderContext{ URLPrefix: ctx.Repo.RepoLink, Metas: ctx.Repo.Repository.ComposeMetas(), + GitRepo: ctx.Repo.GitRepo, }, projects[i].Description) if err != nil { ctx.ServerError("RenderString", err) @@ -322,6 +323,7 @@ func ViewProject(ctx *context.Context) { project.RenderedContent, err = markdown.RenderString(&markup.RenderContext{ URLPrefix: ctx.Repo.RepoLink, Metas: ctx.Repo.Repository.ComposeMetas(), + GitRepo: ctx.Repo.GitRepo, }, project.Description) if err != nil { ctx.ServerError("RenderString", err) diff --git a/routers/web/repo/release.go b/routers/web/repo/release.go index b7730e4ee25e9..3b700e80160c5 100644 --- a/routers/web/repo/release.go +++ b/routers/web/repo/release.go @@ -145,6 +145,7 @@ func releasesOrTags(ctx *context.Context, isTagList bool) { r.Note, err = markdown.RenderString(&markup.RenderContext{ URLPrefix: ctx.Repo.RepoLink, Metas: ctx.Repo.Repository.ComposeMetas(), + GitRepo: ctx.Repo.GitRepo, }, r.Note) if err != nil { ctx.ServerError("RenderString", err) @@ -213,6 +214,7 @@ func SingleRelease(ctx *context.Context) { release.Note, err = markdown.RenderString(&markup.RenderContext{ URLPrefix: ctx.Repo.RepoLink, Metas: ctx.Repo.Repository.ComposeMetas(), + GitRepo: ctx.Repo.GitRepo, }, release.Note) if err != nil { ctx.ServerError("RenderString", err) diff --git a/routers/web/repo/view.go b/routers/web/repo/view.go index cd5b0f43edbc7..74e2a29597724 100644 --- a/routers/web/repo/view.go +++ b/routers/web/repo/view.go @@ -338,6 +338,7 @@ func renderDirectory(ctx *context.Context, treeLink string) { Filename: readmeFile.name, URLPrefix: readmeTreelink, Metas: ctx.Repo.Repository.ComposeDocumentMetas(), + GitRepo: ctx.Repo.GitRepo, }, rd, &result) if err != nil { log.Error("Render failed: %v then fallback", err) @@ -512,6 +513,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st Filename: blob.Name(), URLPrefix: path.Dir(treeLink), Metas: ctx.Repo.Repository.ComposeDocumentMetas(), + GitRepo: ctx.Repo.GitRepo, }, rd, &result) if err != nil { ctx.ServerError("Render", err) @@ -570,6 +572,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st Filename: blob.Name(), URLPrefix: path.Dir(treeLink), Metas: ctx.Repo.Repository.ComposeDocumentMetas(), + GitRepo: ctx.Repo.GitRepo, }, rd, &result) if err != nil { ctx.ServerError("Render", err) diff --git a/routers/web/user/profile.go b/routers/web/user/profile.go index e66820e1317bc..72d00666453e2 100644 --- a/routers/web/user/profile.go +++ b/routers/web/user/profile.go @@ -117,6 +117,7 @@ func Profile(ctx *context.Context) { content, err := markdown.RenderString(&markup.RenderContext{ URLPrefix: ctx.Repo.RepoLink, Metas: map[string]string{"mode": "document"}, + GitRepo: ctx.Repo.GitRepo, }, ctxUser.Description) if err != nil { ctx.ServerError("RenderString", err) From 8601440e81e0e609ab8049fb6dae8307db78fd6c Mon Sep 17 00:00:00 2001 From: GiteaBot Date: Mon, 21 Jun 2021 00:11:34 +0000 Subject: [PATCH 04/10] [skip ci] Updated translations via Crowdin --- options/locale/locale_de-DE.ini | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/options/locale/locale_de-DE.ini b/options/locale/locale_de-DE.ini index 8528aa2cffdc0..2cc2e7df78d00 100644 --- a/options/locale/locale_de-DE.ini +++ b/options/locale/locale_de-DE.ini @@ -829,6 +829,7 @@ migrate.git.description=Migriere oder spiegele git-Daten von Git-Services migrate.gitlab.description=Migriere Daten von GitLab.com oder einem selbst gehostetem gitlab Server. migrate.gitea.description=Migriere Daten von Gitea.com oder einem selbst gehostetem Gitea Server. migrate.gogs.description=Migriere Daten von notabug.org oder einem anderen, selbst gehosteten Gogs Server. +migrate.migrating_git=Git Daten werden migriert migrate.migrating_milestones=Meilensteine werden migriert migrate.migrating_labels=Labels werden migriert migrate.migrating_releases=Releases werden migriert @@ -1561,6 +1562,8 @@ settings.mirror_settings.direction.pull=Pull settings.mirror_settings.direction.push=Push settings.mirror_settings.last_update=Letzte Aktualisierung settings.mirror_settings.push_mirror.none=Keine Push-Mirrors konfiguriert +settings.mirror_settings.push_mirror.remote_url=URL zum Git-Remote-Repository +settings.mirror_settings.push_mirror.add=Push-Mirror hinzufügen settings.sync_mirror=Jetzt synchronisieren settings.mirror_sync_in_progress=Mirror-Synchronisierung wird zurzeit ausgeführt. Komm in ein paar Minuten zurück. settings.email_notifications.enable=E-Mail Benachrichtigungen aktivieren @@ -1626,6 +1629,7 @@ settings.transfer_form_title=Gib den Repository-Namen zur Bestätigung ein: settings.transfer_in_progress=Es gibt derzeit eine laufende Übertragung. Bitte brich diese ab, wenn du dieses Repository an einen anderen Benutzer übertragen möchtest. settings.transfer_notices_1=– Du wirst keinen Zugriff mehr haben, wenn der neue Besitzer ein individueller Benutzer ist. settings.transfer_notices_2=– Du wirst weiterhin Zugriff haben, wenn der neue Besitzer eine Organisation ist und du einer der Besitzer bist. +settings.transfer_notices_3=- Wenn das Repository privat ist und an einen einzelnen Benutzer übertragen wird, wird sichergestellt, dass der Benutzer mindestens Leserechte hat (und die Berechtigungen werden gegebenenfalls ändert). settings.transfer_owner=Neuer Besitzer settings.transfer_perform=Übertragung durchführen settings.transfer_started=Für dieses Repository wurde eine Übertragung eingeleitet und wartet nun auf die Bestätigung von "%s" From 4fcae3d06d1099ade682df96e41baf4f5ac56620 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 21 Jun 2021 04:12:19 +0200 Subject: [PATCH 05/10] Add tests for all webhooks (#16214) * Added tests for MS Teams. * Added tests for Dingtalk. * Added tests for Telegram. * Added tests for Feishu. * Added tests for Discord. * Added tests for closed issue and pullrequest comment. * Added tests for Matrix. * Trim all spaces. * Added tests for Slack. * Added JSONPayload tests. * Added general tests. * Replaced duplicated code. Co-authored-by: Lunny Xiao --- services/webhook/dingtalk.go | 123 ++------ services/webhook/dingtalk_test.go | 201 ++++++++++++- services/webhook/discord.go | 202 ++----------- services/webhook/discord_test.go | 245 +++++++++++++++ services/webhook/feishu.go | 23 +- services/webhook/feishu_test.go | 172 +++++++++++ services/webhook/general.go | 11 +- services/webhook/general_test.go | 464 +++++++++++++++++++++++++++-- services/webhook/matrix_test.go | 198 +++++++++--- services/webhook/msteams.go | 479 ++++++++---------------------- services/webhook/msteams_test.go | 374 +++++++++++++++++++++++ services/webhook/slack.go | 114 +++---- services/webhook/slack_test.go | 186 +++++++++--- services/webhook/telegram.go | 51 ++-- services/webhook/telegram_test.go | 158 +++++++++- 15 files changed, 2110 insertions(+), 891 deletions(-) create mode 100644 services/webhook/discord_test.go create mode 100644 services/webhook/feishu_test.go create mode 100644 services/webhook/msteams_test.go diff --git a/services/webhook/dingtalk.go b/services/webhook/dingtalk.go index 0401464a448a6..d781b8c87dbf6 100644 --- a/services/webhook/dingtalk.go +++ b/services/webhook/dingtalk.go @@ -44,16 +44,7 @@ func (d *DingtalkPayload) Create(p *api.CreatePayload) (api.Payloader, error) { refName := git.RefEndName(p.Ref) title := fmt.Sprintf("[%s] %s %s created", p.Repo.FullName, p.RefType, refName) - return &DingtalkPayload{ - MsgType: "actionCard", - ActionCard: dingtalk.ActionCard{ - Text: title, - Title: title, - HideAvatar: "0", - SingleTitle: fmt.Sprintf("view ref %s", refName), - SingleURL: p.Repo.HTMLURL + "/src/" + refName, - }, - }, nil + return createDingtalkPayload(title, title, fmt.Sprintf("view ref %s", refName), p.Repo.HTMLURL+"/src/"+refName), nil } // Delete implements PayloadConvertor Delete method @@ -62,32 +53,14 @@ func (d *DingtalkPayload) Delete(p *api.DeletePayload) (api.Payloader, error) { refName := git.RefEndName(p.Ref) title := fmt.Sprintf("[%s] %s %s deleted", p.Repo.FullName, p.RefType, refName) - return &DingtalkPayload{ - MsgType: "actionCard", - ActionCard: dingtalk.ActionCard{ - Text: title, - Title: title, - HideAvatar: "0", - SingleTitle: fmt.Sprintf("view ref %s", refName), - SingleURL: p.Repo.HTMLURL + "/src/" + refName, - }, - }, nil + return createDingtalkPayload(title, title, fmt.Sprintf("view ref %s", refName), p.Repo.HTMLURL+"/src/"+refName), nil } // Fork implements PayloadConvertor Fork method func (d *DingtalkPayload) Fork(p *api.ForkPayload) (api.Payloader, error) { title := fmt.Sprintf("%s is forked to %s", p.Forkee.FullName, p.Repo.FullName) - return &DingtalkPayload{ - MsgType: "actionCard", - ActionCard: dingtalk.ActionCard{ - Text: title, - Title: title, - HideAvatar: "0", - SingleTitle: fmt.Sprintf("view forked repo %s", p.Repo.FullName), - SingleURL: p.Repo.HTMLURL, - }, - }, nil + return createDingtalkPayload(title, title, fmt.Sprintf("view forked repo %s", p.Repo.FullName), p.Repo.HTMLURL), nil } // Push implements PayloadConvertor Push method @@ -124,70 +97,32 @@ func (d *DingtalkPayload) Push(p *api.PushPayload) (api.Payloader, error) { strings.TrimRight(commit.Message, "\r\n")) + authorName // add linebreak to each commit but the last if i < len(p.Commits)-1 { - text += "\n" + text += "\r\n" } } - return &DingtalkPayload{ - MsgType: "actionCard", - ActionCard: dingtalk.ActionCard{ - Text: text, - Title: title, - HideAvatar: "0", - SingleTitle: linkText, - SingleURL: titleLink, - }, - }, nil + return createDingtalkPayload(title, text, linkText, titleLink), nil } // Issue implements PayloadConvertor Issue method func (d *DingtalkPayload) Issue(p *api.IssuePayload) (api.Payloader, error) { text, issueTitle, attachmentText, _ := getIssuesPayloadInfo(p, noneLinkFormatter, true) - return &DingtalkPayload{ - MsgType: "actionCard", - ActionCard: dingtalk.ActionCard{ - Text: text + "\r\n\r\n" + attachmentText, - //Markdown: "# " + title + "\n" + text, - Title: issueTitle, - HideAvatar: "0", - SingleTitle: "view issue", - SingleURL: p.Issue.HTMLURL, - }, - }, nil + return createDingtalkPayload(issueTitle, text+"\r\n\r\n"+attachmentText, "view issue", p.Issue.HTMLURL), nil } // IssueComment implements PayloadConvertor IssueComment method func (d *DingtalkPayload) IssueComment(p *api.IssueCommentPayload) (api.Payloader, error) { text, issueTitle, _ := getIssueCommentPayloadInfo(p, noneLinkFormatter, true) - return &DingtalkPayload{ - MsgType: "actionCard", - ActionCard: dingtalk.ActionCard{ - Text: text + "\r\n\r\n" + p.Comment.Body, - Title: issueTitle, - HideAvatar: "0", - SingleTitle: "view issue comment", - SingleURL: p.Comment.HTMLURL, - }, - }, nil + return createDingtalkPayload(issueTitle, text+"\r\n\r\n"+p.Comment.Body, "view issue comment", p.Comment.HTMLURL), nil } // PullRequest implements PayloadConvertor PullRequest method func (d *DingtalkPayload) PullRequest(p *api.PullRequestPayload) (api.Payloader, error) { text, issueTitle, attachmentText, _ := getPullRequestPayloadInfo(p, noneLinkFormatter, true) - return &DingtalkPayload{ - MsgType: "actionCard", - ActionCard: dingtalk.ActionCard{ - Text: text + "\r\n\r\n" + attachmentText, - //Markdown: "# " + title + "\n" + text, - Title: issueTitle, - HideAvatar: "0", - SingleTitle: "view pull request", - SingleURL: p.PullRequest.HTMLURL, - }, - }, nil + return createDingtalkPayload(issueTitle, text+"\r\n\r\n"+attachmentText, "view pull request", p.PullRequest.HTMLURL), nil } // Review implements PayloadConvertor Review method @@ -205,37 +140,17 @@ func (d *DingtalkPayload) Review(p *api.PullRequestPayload, event models.HookEve } - return &DingtalkPayload{ - MsgType: "actionCard", - ActionCard: dingtalk.ActionCard{ - Text: title + "\r\n\r\n" + text, - Title: title, - HideAvatar: "0", - SingleTitle: "view pull request", - SingleURL: p.PullRequest.HTMLURL, - }, - }, nil + return createDingtalkPayload(title, title+"\r\n\r\n"+text, "view pull request", p.PullRequest.HTMLURL), nil } // Repository implements PayloadConvertor Repository method func (d *DingtalkPayload) Repository(p *api.RepositoryPayload) (api.Payloader, error) { - var title, url string switch p.Action { case api.HookRepoCreated: - title = fmt.Sprintf("[%s] Repository created", p.Repository.FullName) - url = p.Repository.HTMLURL - return &DingtalkPayload{ - MsgType: "actionCard", - ActionCard: dingtalk.ActionCard{ - Text: title, - Title: title, - HideAvatar: "0", - SingleTitle: "view repository", - SingleURL: url, - }, - }, nil + title := fmt.Sprintf("[%s] Repository created", p.Repository.FullName) + return createDingtalkPayload(title, title, "view repository", p.Repository.HTMLURL), nil case api.HookRepoDeleted: - title = fmt.Sprintf("[%s] Repository deleted", p.Repository.FullName) + title := fmt.Sprintf("[%s] Repository deleted", p.Repository.FullName) return &DingtalkPayload{ MsgType: "text", Text: struct { @@ -253,16 +168,20 @@ func (d *DingtalkPayload) Repository(p *api.RepositoryPayload) (api.Payloader, e func (d *DingtalkPayload) Release(p *api.ReleasePayload) (api.Payloader, error) { text, _ := getReleasePayloadInfo(p, noneLinkFormatter, true) + return createDingtalkPayload(text, text, "view release", p.Release.URL), nil +} + +func createDingtalkPayload(title, text, singleTitle, singleURL string) *DingtalkPayload { return &DingtalkPayload{ MsgType: "actionCard", ActionCard: dingtalk.ActionCard{ - Text: text, - Title: text, + Text: strings.TrimSpace(text), + Title: strings.TrimSpace(title), HideAvatar: "0", - SingleTitle: "view release", - SingleURL: p.Release.URL, + SingleTitle: singleTitle, + SingleURL: singleURL, }, - }, nil + } } // GetDingtalkPayload converts a ding talk webhook into a DingtalkPayload diff --git a/services/webhook/dingtalk_test.go b/services/webhook/dingtalk_test.go index e5aa0fca36abe..213ad1a284ec1 100644 --- a/services/webhook/dingtalk_test.go +++ b/services/webhook/dingtalk_test.go @@ -7,25 +7,202 @@ package webhook import ( "testing" + "code.gitea.io/gitea/models" api "code.gitea.io/gitea/modules/structs" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestGetDingTalkIssuesPayload(t *testing.T) { - p := issueTestPayload() - d := new(DingtalkPayload) - p.Action = api.HookIssueOpened - pl, err := d.Issue(p) +func TestDingTalkPayload(t *testing.T) { + t.Run("Create", func(t *testing.T) { + p := createTestPayload() + + d := new(DingtalkPayload) + pl, err := d.Create(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &DingtalkPayload{}, pl) + + assert.Equal(t, "[test/repo] branch test created", pl.(*DingtalkPayload).ActionCard.Text) + assert.Equal(t, "[test/repo] branch test created", pl.(*DingtalkPayload).ActionCard.Title) + assert.Equal(t, "view ref test", pl.(*DingtalkPayload).ActionCard.SingleTitle) + assert.Equal(t, "http://localhost:3000/test/repo/src/test", pl.(*DingtalkPayload).ActionCard.SingleURL) + }) + + t.Run("Delete", func(t *testing.T) { + p := deleteTestPayload() + + d := new(DingtalkPayload) + pl, err := d.Delete(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &DingtalkPayload{}, pl) + + assert.Equal(t, "[test/repo] branch test deleted", pl.(*DingtalkPayload).ActionCard.Text) + assert.Equal(t, "[test/repo] branch test deleted", pl.(*DingtalkPayload).ActionCard.Title) + assert.Equal(t, "view ref test", pl.(*DingtalkPayload).ActionCard.SingleTitle) + assert.Equal(t, "http://localhost:3000/test/repo/src/test", pl.(*DingtalkPayload).ActionCard.SingleURL) + }) + + t.Run("Fork", func(t *testing.T) { + p := forkTestPayload() + + d := new(DingtalkPayload) + pl, err := d.Fork(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &DingtalkPayload{}, pl) + + assert.Equal(t, "test/repo2 is forked to test/repo", pl.(*DingtalkPayload).ActionCard.Text) + assert.Equal(t, "test/repo2 is forked to test/repo", pl.(*DingtalkPayload).ActionCard.Title) + assert.Equal(t, "view forked repo test/repo", pl.(*DingtalkPayload).ActionCard.SingleTitle) + assert.Equal(t, "http://localhost:3000/test/repo", pl.(*DingtalkPayload).ActionCard.SingleURL) + }) + + t.Run("Push", func(t *testing.T) { + p := pushTestPayload() + + d := new(DingtalkPayload) + pl, err := d.Push(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &DingtalkPayload{}, pl) + + assert.Equal(t, "[2020558](http://localhost:3000/test/repo/commit/2020558fe2e34debb818a514715839cabd25e778) commit message - user1\r\n[2020558](http://localhost:3000/test/repo/commit/2020558fe2e34debb818a514715839cabd25e778) commit message - user1", pl.(*DingtalkPayload).ActionCard.Text) + assert.Equal(t, "[test/repo:test] 2 new commits", pl.(*DingtalkPayload).ActionCard.Title) + assert.Equal(t, "view commit 2020558...2020558", pl.(*DingtalkPayload).ActionCard.SingleTitle) + assert.Equal(t, "http://localhost:3000/test/repo/src/test", pl.(*DingtalkPayload).ActionCard.SingleURL) + }) + + t.Run("Issue", func(t *testing.T) { + p := issueTestPayload() + + d := new(DingtalkPayload) + p.Action = api.HookIssueOpened + pl, err := d.Issue(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &DingtalkPayload{}, pl) + + assert.Equal(t, "[test/repo] Issue opened: #2 crash by user1\r\n\r\nissue body", pl.(*DingtalkPayload).ActionCard.Text) + assert.Equal(t, "#2 crash", pl.(*DingtalkPayload).ActionCard.Title) + assert.Equal(t, "view issue", pl.(*DingtalkPayload).ActionCard.SingleTitle) + assert.Equal(t, "http://localhost:3000/test/repo/issues/2", pl.(*DingtalkPayload).ActionCard.SingleURL) + + p.Action = api.HookIssueClosed + pl, err = d.Issue(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &DingtalkPayload{}, pl) + + assert.Equal(t, "[test/repo] Issue closed: #2 crash by user1", pl.(*DingtalkPayload).ActionCard.Text) + assert.Equal(t, "#2 crash", pl.(*DingtalkPayload).ActionCard.Title) + assert.Equal(t, "view issue", pl.(*DingtalkPayload).ActionCard.SingleTitle) + assert.Equal(t, "http://localhost:3000/test/repo/issues/2", pl.(*DingtalkPayload).ActionCard.SingleURL) + }) + + t.Run("IssueComment", func(t *testing.T) { + p := issueCommentTestPayload() + + d := new(DingtalkPayload) + pl, err := d.IssueComment(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &DingtalkPayload{}, pl) + + assert.Equal(t, "[test/repo] New comment on issue #2 crash by user1\r\n\r\nmore info needed", pl.(*DingtalkPayload).ActionCard.Text) + assert.Equal(t, "#2 crash", pl.(*DingtalkPayload).ActionCard.Title) + assert.Equal(t, "view issue comment", pl.(*DingtalkPayload).ActionCard.SingleTitle) + assert.Equal(t, "http://localhost:3000/test/repo/issues/2#issuecomment-4", pl.(*DingtalkPayload).ActionCard.SingleURL) + }) + + t.Run("PullRequest", func(t *testing.T) { + p := pullRequestTestPayload() + + d := new(DingtalkPayload) + pl, err := d.PullRequest(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &DingtalkPayload{}, pl) + + assert.Equal(t, "[test/repo] Pull request opened: #12 Fix bug by user1\r\n\r\nfixes bug #2", pl.(*DingtalkPayload).ActionCard.Text) + assert.Equal(t, "#12 Fix bug", pl.(*DingtalkPayload).ActionCard.Title) + assert.Equal(t, "view pull request", pl.(*DingtalkPayload).ActionCard.SingleTitle) + assert.Equal(t, "http://localhost:3000/test/repo/pulls/12", pl.(*DingtalkPayload).ActionCard.SingleURL) + }) + + t.Run("PullRequestComment", func(t *testing.T) { + p := pullRequestCommentTestPayload() + + d := new(DingtalkPayload) + pl, err := d.IssueComment(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &DingtalkPayload{}, pl) + + assert.Equal(t, "[test/repo] New comment on pull request #12 Fix bug by user1\r\n\r\nchanges requested", pl.(*DingtalkPayload).ActionCard.Text) + assert.Equal(t, "#12 Fix bug", pl.(*DingtalkPayload).ActionCard.Title) + assert.Equal(t, "view issue comment", pl.(*DingtalkPayload).ActionCard.SingleTitle) + assert.Equal(t, "http://localhost:3000/test/repo/pulls/12#issuecomment-4", pl.(*DingtalkPayload).ActionCard.SingleURL) + }) + + t.Run("Review", func(t *testing.T) { + p := pullRequestTestPayload() + p.Action = api.HookIssueReviewed + + d := new(DingtalkPayload) + pl, err := d.Review(p, models.HookEventPullRequestReviewApproved) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &DingtalkPayload{}, pl) + + assert.Equal(t, "[test/repo] Pull request review approved : #12 Fix bug\r\n\r\ngood job", pl.(*DingtalkPayload).ActionCard.Text) + assert.Equal(t, "[test/repo] Pull request review approved : #12 Fix bug", pl.(*DingtalkPayload).ActionCard.Title) + assert.Equal(t, "view pull request", pl.(*DingtalkPayload).ActionCard.SingleTitle) + assert.Equal(t, "http://localhost:3000/test/repo/pulls/12", pl.(*DingtalkPayload).ActionCard.SingleURL) + }) + + t.Run("Repository", func(t *testing.T) { + p := repositoryTestPayload() + + d := new(DingtalkPayload) + pl, err := d.Repository(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &DingtalkPayload{}, pl) + + assert.Equal(t, "[test/repo] Repository created", pl.(*DingtalkPayload).ActionCard.Text) + assert.Equal(t, "[test/repo] Repository created", pl.(*DingtalkPayload).ActionCard.Title) + assert.Equal(t, "view repository", pl.(*DingtalkPayload).ActionCard.SingleTitle) + assert.Equal(t, "http://localhost:3000/test/repo", pl.(*DingtalkPayload).ActionCard.SingleURL) + }) + + t.Run("Release", func(t *testing.T) { + p := pullReleaseTestPayload() + + d := new(DingtalkPayload) + pl, err := d.Release(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &DingtalkPayload{}, pl) + + assert.Equal(t, "[test/repo] Release created: v1.0 by user1", pl.(*DingtalkPayload).ActionCard.Text) + assert.Equal(t, "[test/repo] Release created: v1.0 by user1", pl.(*DingtalkPayload).ActionCard.Title) + assert.Equal(t, "view release", pl.(*DingtalkPayload).ActionCard.SingleTitle) + assert.Equal(t, "http://localhost:3000/api/v1/repos/test/repo/releases/2", pl.(*DingtalkPayload).ActionCard.SingleURL) + }) +} + +func TestDingTalkJSONPayload(t *testing.T) { + p := pushTestPayload() + + pl, err := new(DingtalkPayload).Push(p) require.NoError(t, err) require.NotNil(t, pl) - assert.Equal(t, "#2 crash", pl.(*DingtalkPayload).ActionCard.Title) - assert.Equal(t, "[test/repo] Issue opened: #2 crash by user1\r\n\r\n", pl.(*DingtalkPayload).ActionCard.Text) + require.IsType(t, &DingtalkPayload{}, pl) - p.Action = api.HookIssueClosed - pl, err = d.Issue(p) + json, err := pl.JSONPayload() require.NoError(t, err) - require.NotNil(t, pl) - assert.Equal(t, "#2 crash", pl.(*DingtalkPayload).ActionCard.Title) - assert.Equal(t, "[test/repo] Issue closed: #2 crash by user1\r\n\r\n", pl.(*DingtalkPayload).ActionCard.Text) + assert.NotEmpty(t, json) } diff --git a/services/webhook/discord.go b/services/webhook/discord.go index d28904715f618..378d9ff725db6 100644 --- a/services/webhook/discord.go +++ b/services/webhook/discord.go @@ -120,22 +120,7 @@ func (d *DiscordPayload) Create(p *api.CreatePayload) (api.Payloader, error) { refName := git.RefEndName(p.Ref) title := fmt.Sprintf("[%s] %s %s created", p.Repo.FullName, p.RefType, refName) - return &DiscordPayload{ - Username: d.Username, - AvatarURL: d.AvatarURL, - Embeds: []DiscordEmbed{ - { - Title: title, - URL: p.Repo.HTMLURL + "/src/" + refName, - Color: greenColor, - Author: DiscordEmbedAuthor{ - Name: p.Sender.UserName, - URL: setting.AppURL + p.Sender.UserName, - IconURL: p.Sender.AvatarURL, - }, - }, - }, - }, nil + return d.createPayload(p.Sender, title, "", p.Repo.HTMLURL+"/src/"+refName, greenColor), nil } // Delete implements PayloadConvertor Delete method @@ -144,44 +129,14 @@ func (d *DiscordPayload) Delete(p *api.DeletePayload) (api.Payloader, error) { refName := git.RefEndName(p.Ref) title := fmt.Sprintf("[%s] %s %s deleted", p.Repo.FullName, p.RefType, refName) - return &DiscordPayload{ - Username: d.Username, - AvatarURL: d.AvatarURL, - Embeds: []DiscordEmbed{ - { - Title: title, - URL: p.Repo.HTMLURL + "/src/" + refName, - Color: redColor, - Author: DiscordEmbedAuthor{ - Name: p.Sender.UserName, - URL: setting.AppURL + p.Sender.UserName, - IconURL: p.Sender.AvatarURL, - }, - }, - }, - }, nil + return d.createPayload(p.Sender, title, "", p.Repo.HTMLURL+"/src/"+refName, redColor), nil } // Fork implements PayloadConvertor Fork method func (d *DiscordPayload) Fork(p *api.ForkPayload) (api.Payloader, error) { title := fmt.Sprintf("%s is forked to %s", p.Forkee.FullName, p.Repo.FullName) - return &DiscordPayload{ - Username: d.Username, - AvatarURL: d.AvatarURL, - Embeds: []DiscordEmbed{ - { - Title: title, - URL: p.Repo.HTMLURL, - Color: greenColor, - Author: DiscordEmbedAuthor{ - Name: p.Sender.UserName, - URL: setting.AppURL + p.Sender.UserName, - IconURL: p.Sender.AvatarURL, - }, - }, - }, - }, nil + return d.createPayload(p.Sender, title, "", p.Repo.HTMLURL, greenColor), nil } // Push implements PayloadConvertor Push method @@ -216,92 +171,28 @@ func (d *DiscordPayload) Push(p *api.PushPayload) (api.Payloader, error) { } } - return &DiscordPayload{ - Username: d.Username, - AvatarURL: d.AvatarURL, - Embeds: []DiscordEmbed{ - { - Title: title, - Description: text, - URL: titleLink, - Color: greenColor, - Author: DiscordEmbedAuthor{ - Name: p.Sender.UserName, - URL: setting.AppURL + p.Sender.UserName, - IconURL: p.Sender.AvatarURL, - }, - }, - }, - }, nil + return d.createPayload(p.Sender, title, text, titleLink, greenColor), nil } // Issue implements PayloadConvertor Issue method func (d *DiscordPayload) Issue(p *api.IssuePayload) (api.Payloader, error) { - text, _, attachmentText, color := getIssuesPayloadInfo(p, noneLinkFormatter, false) + title, _, text, color := getIssuesPayloadInfo(p, noneLinkFormatter, false) - return &DiscordPayload{ - Username: d.Username, - AvatarURL: d.AvatarURL, - Embeds: []DiscordEmbed{ - { - Title: text, - Description: attachmentText, - URL: p.Issue.HTMLURL, - Color: color, - Author: DiscordEmbedAuthor{ - Name: p.Sender.UserName, - URL: setting.AppURL + p.Sender.UserName, - IconURL: p.Sender.AvatarURL, - }, - }, - }, - }, nil + return d.createPayload(p.Sender, title, text, p.Issue.HTMLURL, color), nil } // IssueComment implements PayloadConvertor IssueComment method func (d *DiscordPayload) IssueComment(p *api.IssueCommentPayload) (api.Payloader, error) { - text, _, color := getIssueCommentPayloadInfo(p, noneLinkFormatter, false) + title, _, color := getIssueCommentPayloadInfo(p, noneLinkFormatter, false) - return &DiscordPayload{ - Username: d.Username, - AvatarURL: d.AvatarURL, - Embeds: []DiscordEmbed{ - { - Title: text, - Description: p.Comment.Body, - URL: p.Comment.HTMLURL, - Color: color, - Author: DiscordEmbedAuthor{ - Name: p.Sender.UserName, - URL: setting.AppURL + p.Sender.UserName, - IconURL: p.Sender.AvatarURL, - }, - }, - }, - }, nil + return d.createPayload(p.Sender, title, p.Comment.Body, p.Comment.HTMLURL, color), nil } // PullRequest implements PayloadConvertor PullRequest method func (d *DiscordPayload) PullRequest(p *api.PullRequestPayload) (api.Payloader, error) { - text, _, attachmentText, color := getPullRequestPayloadInfo(p, noneLinkFormatter, false) + title, _, text, color := getPullRequestPayloadInfo(p, noneLinkFormatter, false) - return &DiscordPayload{ - Username: d.Username, - AvatarURL: d.AvatarURL, - Embeds: []DiscordEmbed{ - { - Title: text, - Description: attachmentText, - URL: p.PullRequest.HTMLURL, - Color: color, - Author: DiscordEmbedAuthor{ - Name: p.Sender.UserName, - URL: setting.AppURL + p.Sender.UserName, - IconURL: p.Sender.AvatarURL, - }, - }, - }, - }, nil + return d.createPayload(p.Sender, title, text, p.PullRequest.HTMLURL, color), nil } // Review implements PayloadConvertor Review method @@ -330,23 +221,7 @@ func (d *DiscordPayload) Review(p *api.PullRequestPayload, event models.HookEven } } - return &DiscordPayload{ - Username: d.Username, - AvatarURL: d.AvatarURL, - Embeds: []DiscordEmbed{ - { - Title: title, - Description: text, - URL: p.PullRequest.HTMLURL, - Color: color, - Author: DiscordEmbedAuthor{ - Name: p.Sender.UserName, - URL: setting.AppURL + p.Sender.UserName, - IconURL: p.Sender.AvatarURL, - }, - }, - }, - }, nil + return d.createPayload(p.Sender, title, text, p.PullRequest.HTMLURL, color), nil } // Repository implements PayloadConvertor Repository method @@ -363,45 +238,14 @@ func (d *DiscordPayload) Repository(p *api.RepositoryPayload) (api.Payloader, er color = redColor } - return &DiscordPayload{ - Username: d.Username, - AvatarURL: d.AvatarURL, - Embeds: []DiscordEmbed{ - { - Title: title, - URL: url, - Color: color, - Author: DiscordEmbedAuthor{ - Name: p.Sender.UserName, - URL: setting.AppURL + p.Sender.UserName, - IconURL: p.Sender.AvatarURL, - }, - }, - }, - }, nil + return d.createPayload(p.Sender, title, "", url, color), nil } // Release implements PayloadConvertor Release method func (d *DiscordPayload) Release(p *api.ReleasePayload) (api.Payloader, error) { text, color := getReleasePayloadInfo(p, noneLinkFormatter, false) - return &DiscordPayload{ - Username: d.Username, - AvatarURL: d.AvatarURL, - Embeds: []DiscordEmbed{ - { - Title: text, - Description: p.Release.Note, - URL: p.Release.URL, - Color: color, - Author: DiscordEmbedAuthor{ - Name: p.Sender.UserName, - URL: setting.AppURL + p.Sender.UserName, - IconURL: p.Sender.AvatarURL, - }, - }, - }, - }, nil + return d.createPayload(p.Sender, text, p.Release.Note, p.Release.URL, color), nil } // GetDiscordPayload converts a discord webhook into a DiscordPayload @@ -433,3 +277,23 @@ func parseHookPullRequestEventType(event models.HookEventType) (string, error) { return "", errors.New("unknown event type") } } + +func (d *DiscordPayload) createPayload(s *api.User, title, text, url string, color int) *DiscordPayload { + return &DiscordPayload{ + Username: d.Username, + AvatarURL: d.AvatarURL, + Embeds: []DiscordEmbed{ + { + Title: title, + Description: text, + URL: url, + Color: color, + Author: DiscordEmbedAuthor{ + Name: s.UserName, + URL: setting.AppURL + s.UserName, + IconURL: s.AvatarURL, + }, + }, + }, + } +} diff --git a/services/webhook/discord_test.go b/services/webhook/discord_test.go new file mode 100644 index 0000000000000..fd7d2856c78b0 --- /dev/null +++ b/services/webhook/discord_test.go @@ -0,0 +1,245 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package webhook + +import ( + "testing" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/setting" + api "code.gitea.io/gitea/modules/structs" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDiscordPayload(t *testing.T) { + t.Run("Create", func(t *testing.T) { + p := createTestPayload() + + d := new(DiscordPayload) + pl, err := d.Create(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &DiscordPayload{}, pl) + + assert.Len(t, pl.(*DiscordPayload).Embeds, 1) + assert.Equal(t, "[test/repo] branch test created", pl.(*DiscordPayload).Embeds[0].Title) + assert.Empty(t, pl.(*DiscordPayload).Embeds[0].Description) + assert.Equal(t, "http://localhost:3000/test/repo/src/test", pl.(*DiscordPayload).Embeds[0].URL) + assert.Equal(t, p.Sender.UserName, pl.(*DiscordPayload).Embeds[0].Author.Name) + assert.Equal(t, setting.AppURL+p.Sender.UserName, pl.(*DiscordPayload).Embeds[0].Author.URL) + assert.Equal(t, p.Sender.AvatarURL, pl.(*DiscordPayload).Embeds[0].Author.IconURL) + }) + + t.Run("Delete", func(t *testing.T) { + p := deleteTestPayload() + + d := new(DiscordPayload) + pl, err := d.Delete(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &DiscordPayload{}, pl) + + assert.Len(t, pl.(*DiscordPayload).Embeds, 1) + assert.Equal(t, "[test/repo] branch test deleted", pl.(*DiscordPayload).Embeds[0].Title) + assert.Empty(t, pl.(*DiscordPayload).Embeds[0].Description) + assert.Equal(t, "http://localhost:3000/test/repo/src/test", pl.(*DiscordPayload).Embeds[0].URL) + assert.Equal(t, p.Sender.UserName, pl.(*DiscordPayload).Embeds[0].Author.Name) + assert.Equal(t, setting.AppURL+p.Sender.UserName, pl.(*DiscordPayload).Embeds[0].Author.URL) + assert.Equal(t, p.Sender.AvatarURL, pl.(*DiscordPayload).Embeds[0].Author.IconURL) + }) + + t.Run("Fork", func(t *testing.T) { + p := forkTestPayload() + + d := new(DiscordPayload) + pl, err := d.Fork(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &DiscordPayload{}, pl) + + assert.Len(t, pl.(*DiscordPayload).Embeds, 1) + assert.Equal(t, "test/repo2 is forked to test/repo", pl.(*DiscordPayload).Embeds[0].Title) + assert.Empty(t, pl.(*DiscordPayload).Embeds[0].Description) + assert.Equal(t, "http://localhost:3000/test/repo", pl.(*DiscordPayload).Embeds[0].URL) + assert.Equal(t, p.Sender.UserName, pl.(*DiscordPayload).Embeds[0].Author.Name) + assert.Equal(t, setting.AppURL+p.Sender.UserName, pl.(*DiscordPayload).Embeds[0].Author.URL) + assert.Equal(t, p.Sender.AvatarURL, pl.(*DiscordPayload).Embeds[0].Author.IconURL) + }) + + t.Run("Push", func(t *testing.T) { + p := pushTestPayload() + + d := new(DiscordPayload) + pl, err := d.Push(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &DiscordPayload{}, pl) + + assert.Len(t, pl.(*DiscordPayload).Embeds, 1) + assert.Equal(t, "[test/repo:test] 2 new commits", pl.(*DiscordPayload).Embeds[0].Title) + assert.Equal(t, "[2020558](http://localhost:3000/test/repo/commit/2020558fe2e34debb818a514715839cabd25e778) commit message - user1\n[2020558](http://localhost:3000/test/repo/commit/2020558fe2e34debb818a514715839cabd25e778) commit message - user1", pl.(*DiscordPayload).Embeds[0].Description) + assert.Equal(t, "http://localhost:3000/test/repo/src/test", pl.(*DiscordPayload).Embeds[0].URL) + assert.Equal(t, p.Sender.UserName, pl.(*DiscordPayload).Embeds[0].Author.Name) + assert.Equal(t, setting.AppURL+p.Sender.UserName, pl.(*DiscordPayload).Embeds[0].Author.URL) + assert.Equal(t, p.Sender.AvatarURL, pl.(*DiscordPayload).Embeds[0].Author.IconURL) + }) + + t.Run("Issue", func(t *testing.T) { + p := issueTestPayload() + + d := new(DiscordPayload) + p.Action = api.HookIssueOpened + pl, err := d.Issue(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &DiscordPayload{}, pl) + + assert.Len(t, pl.(*DiscordPayload).Embeds, 1) + assert.Equal(t, "[test/repo] Issue opened: #2 crash", pl.(*DiscordPayload).Embeds[0].Title) + assert.Equal(t, "issue body", pl.(*DiscordPayload).Embeds[0].Description) + assert.Equal(t, "http://localhost:3000/test/repo/issues/2", pl.(*DiscordPayload).Embeds[0].URL) + assert.Equal(t, p.Sender.UserName, pl.(*DiscordPayload).Embeds[0].Author.Name) + assert.Equal(t, setting.AppURL+p.Sender.UserName, pl.(*DiscordPayload).Embeds[0].Author.URL) + assert.Equal(t, p.Sender.AvatarURL, pl.(*DiscordPayload).Embeds[0].Author.IconURL) + + p.Action = api.HookIssueClosed + pl, err = d.Issue(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &DiscordPayload{}, pl) + + assert.Len(t, pl.(*DiscordPayload).Embeds, 1) + assert.Equal(t, "[test/repo] Issue closed: #2 crash", pl.(*DiscordPayload).Embeds[0].Title) + assert.Empty(t, pl.(*DiscordPayload).Embeds[0].Description) + assert.Equal(t, "http://localhost:3000/test/repo/issues/2", pl.(*DiscordPayload).Embeds[0].URL) + assert.Equal(t, p.Sender.UserName, pl.(*DiscordPayload).Embeds[0].Author.Name) + assert.Equal(t, setting.AppURL+p.Sender.UserName, pl.(*DiscordPayload).Embeds[0].Author.URL) + assert.Equal(t, p.Sender.AvatarURL, pl.(*DiscordPayload).Embeds[0].Author.IconURL) + }) + + t.Run("IssueComment", func(t *testing.T) { + p := issueCommentTestPayload() + + d := new(DiscordPayload) + pl, err := d.IssueComment(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &DiscordPayload{}, pl) + + assert.Len(t, pl.(*DiscordPayload).Embeds, 1) + assert.Equal(t, "[test/repo] New comment on issue #2 crash", pl.(*DiscordPayload).Embeds[0].Title) + assert.Equal(t, "more info needed", pl.(*DiscordPayload).Embeds[0].Description) + assert.Equal(t, "http://localhost:3000/test/repo/issues/2#issuecomment-4", pl.(*DiscordPayload).Embeds[0].URL) + assert.Equal(t, p.Sender.UserName, pl.(*DiscordPayload).Embeds[0].Author.Name) + assert.Equal(t, setting.AppURL+p.Sender.UserName, pl.(*DiscordPayload).Embeds[0].Author.URL) + assert.Equal(t, p.Sender.AvatarURL, pl.(*DiscordPayload).Embeds[0].Author.IconURL) + }) + + t.Run("PullRequest", func(t *testing.T) { + p := pullRequestTestPayload() + + d := new(DiscordPayload) + pl, err := d.PullRequest(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &DiscordPayload{}, pl) + + assert.Len(t, pl.(*DiscordPayload).Embeds, 1) + assert.Equal(t, "[test/repo] Pull request opened: #12 Fix bug", pl.(*DiscordPayload).Embeds[0].Title) + assert.Equal(t, "fixes bug #2", pl.(*DiscordPayload).Embeds[0].Description) + assert.Equal(t, "http://localhost:3000/test/repo/pulls/12", pl.(*DiscordPayload).Embeds[0].URL) + assert.Equal(t, p.Sender.UserName, pl.(*DiscordPayload).Embeds[0].Author.Name) + assert.Equal(t, setting.AppURL+p.Sender.UserName, pl.(*DiscordPayload).Embeds[0].Author.URL) + assert.Equal(t, p.Sender.AvatarURL, pl.(*DiscordPayload).Embeds[0].Author.IconURL) + }) + + t.Run("PullRequestComment", func(t *testing.T) { + p := pullRequestCommentTestPayload() + + d := new(DiscordPayload) + pl, err := d.IssueComment(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &DiscordPayload{}, pl) + + assert.Len(t, pl.(*DiscordPayload).Embeds, 1) + assert.Equal(t, "[test/repo] New comment on pull request #12 Fix bug", pl.(*DiscordPayload).Embeds[0].Title) + assert.Equal(t, "changes requested", pl.(*DiscordPayload).Embeds[0].Description) + assert.Equal(t, "http://localhost:3000/test/repo/pulls/12#issuecomment-4", pl.(*DiscordPayload).Embeds[0].URL) + assert.Equal(t, p.Sender.UserName, pl.(*DiscordPayload).Embeds[0].Author.Name) + assert.Equal(t, setting.AppURL+p.Sender.UserName, pl.(*DiscordPayload).Embeds[0].Author.URL) + assert.Equal(t, p.Sender.AvatarURL, pl.(*DiscordPayload).Embeds[0].Author.IconURL) + }) + + t.Run("Review", func(t *testing.T) { + p := pullRequestTestPayload() + p.Action = api.HookIssueReviewed + + d := new(DiscordPayload) + pl, err := d.Review(p, models.HookEventPullRequestReviewApproved) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &DiscordPayload{}, pl) + + assert.Len(t, pl.(*DiscordPayload).Embeds, 1) + assert.Equal(t, "[test/repo] Pull request review approved: #12 Fix bug", pl.(*DiscordPayload).Embeds[0].Title) + assert.Equal(t, "good job", pl.(*DiscordPayload).Embeds[0].Description) + assert.Equal(t, "http://localhost:3000/test/repo/pulls/12", pl.(*DiscordPayload).Embeds[0].URL) + assert.Equal(t, p.Sender.UserName, pl.(*DiscordPayload).Embeds[0].Author.Name) + assert.Equal(t, setting.AppURL+p.Sender.UserName, pl.(*DiscordPayload).Embeds[0].Author.URL) + assert.Equal(t, p.Sender.AvatarURL, pl.(*DiscordPayload).Embeds[0].Author.IconURL) + }) + + t.Run("Repository", func(t *testing.T) { + p := repositoryTestPayload() + + d := new(DiscordPayload) + pl, err := d.Repository(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &DiscordPayload{}, pl) + + assert.Len(t, pl.(*DiscordPayload).Embeds, 1) + assert.Equal(t, "[test/repo] Repository created", pl.(*DiscordPayload).Embeds[0].Title) + assert.Empty(t, pl.(*DiscordPayload).Embeds[0].Description) + assert.Equal(t, "http://localhost:3000/test/repo", pl.(*DiscordPayload).Embeds[0].URL) + assert.Equal(t, p.Sender.UserName, pl.(*DiscordPayload).Embeds[0].Author.Name) + assert.Equal(t, setting.AppURL+p.Sender.UserName, pl.(*DiscordPayload).Embeds[0].Author.URL) + assert.Equal(t, p.Sender.AvatarURL, pl.(*DiscordPayload).Embeds[0].Author.IconURL) + }) + + t.Run("Release", func(t *testing.T) { + p := pullReleaseTestPayload() + + d := new(DiscordPayload) + pl, err := d.Release(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &DiscordPayload{}, pl) + + assert.Len(t, pl.(*DiscordPayload).Embeds, 1) + assert.Equal(t, "[test/repo] Release created: v1.0", pl.(*DiscordPayload).Embeds[0].Title) + assert.Equal(t, "Note of first stable release", pl.(*DiscordPayload).Embeds[0].Description) + assert.Equal(t, "http://localhost:3000/api/v1/repos/test/repo/releases/2", pl.(*DiscordPayload).Embeds[0].URL) + assert.Equal(t, p.Sender.UserName, pl.(*DiscordPayload).Embeds[0].Author.Name) + assert.Equal(t, setting.AppURL+p.Sender.UserName, pl.(*DiscordPayload).Embeds[0].Author.URL) + assert.Equal(t, p.Sender.AvatarURL, pl.(*DiscordPayload).Embeds[0].Author.IconURL) + }) +} + +func TestDiscordJSONPayload(t *testing.T) { + p := pushTestPayload() + + pl, err := new(DiscordPayload).Push(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &DiscordPayload{}, pl) + + json, err := pl.JSONPayload() + require.NoError(t, err) + assert.NotEmpty(t, json) +} diff --git a/services/webhook/feishu.go b/services/webhook/feishu.go index 847a991f366c3..5c80efb820db7 100644 --- a/services/webhook/feishu.go +++ b/services/webhook/feishu.go @@ -30,7 +30,7 @@ func newFeishuTextPayload(text string) *FeishuPayload { Content: struct { Text string `json:"text"` }{ - Text: text, + Text: strings.TrimSpace(text), }, } } @@ -84,7 +84,7 @@ func (f *FeishuPayload) Push(p *api.PushPayload) (api.Payloader, error) { commitDesc string ) - var text = fmt.Sprintf("[%s:%s] %s\n", p.Repo.FullName, branchName, commitDesc) + var text = fmt.Sprintf("[%s:%s] %s\r\n", p.Repo.FullName, branchName, commitDesc) // for each commit, generate attachment text for i, commit := range p.Commits { var authorName string @@ -95,7 +95,7 @@ func (f *FeishuPayload) Push(p *api.PushPayload) (api.Payloader, error) { strings.TrimRight(commit.Message, "\r\n")) + authorName // add linebreak to each commit but the last if i < len(p.Commits)-1 { - text += "\n" + text += "\r\n" } } @@ -125,19 +125,14 @@ func (f *FeishuPayload) PullRequest(p *api.PullRequestPayload) (api.Payloader, e // Review implements PayloadConvertor Review method func (f *FeishuPayload) Review(p *api.PullRequestPayload, event models.HookEventType) (api.Payloader, error) { - var text, title string - switch p.Action { - case api.HookIssueSynchronized: - action, err := parseHookPullRequestEventType(event) - if err != nil { - return nil, err - } - - title = fmt.Sprintf("[%s] Pull request review %s : #%d %s", p.Repository.FullName, action, p.Index, p.PullRequest.Title) - text = p.Review.Content - + action, err := parseHookPullRequestEventType(event) + if err != nil { + return nil, err } + title := fmt.Sprintf("[%s] Pull request review %s : #%d %s", p.Repository.FullName, action, p.Index, p.PullRequest.Title) + text := p.Review.Content + return newFeishuTextPayload(title + "\r\n\r\n" + text), nil } diff --git a/services/webhook/feishu_test.go b/services/webhook/feishu_test.go new file mode 100644 index 0000000000000..7f3508c145d5a --- /dev/null +++ b/services/webhook/feishu_test.go @@ -0,0 +1,172 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package webhook + +import ( + "testing" + + "code.gitea.io/gitea/models" + api "code.gitea.io/gitea/modules/structs" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFeishuPayload(t *testing.T) { + t.Run("Create", func(t *testing.T) { + p := createTestPayload() + + d := new(FeishuPayload) + pl, err := d.Create(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &FeishuPayload{}, pl) + + assert.Equal(t, `[test/repo] branch test created`, pl.(*FeishuPayload).Content.Text) + }) + + t.Run("Delete", func(t *testing.T) { + p := deleteTestPayload() + + d := new(FeishuPayload) + pl, err := d.Delete(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &FeishuPayload{}, pl) + + assert.Equal(t, `[test/repo] branch test deleted`, pl.(*FeishuPayload).Content.Text) + }) + + t.Run("Fork", func(t *testing.T) { + p := forkTestPayload() + + d := new(FeishuPayload) + pl, err := d.Fork(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &FeishuPayload{}, pl) + + assert.Equal(t, `test/repo2 is forked to test/repo`, pl.(*FeishuPayload).Content.Text) + }) + + t.Run("Push", func(t *testing.T) { + p := pushTestPayload() + + d := new(FeishuPayload) + pl, err := d.Push(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &FeishuPayload{}, pl) + + assert.Equal(t, "[test/repo:test] \r\n[2020558](http://localhost:3000/test/repo/commit/2020558fe2e34debb818a514715839cabd25e778) commit message - user1\r\n[2020558](http://localhost:3000/test/repo/commit/2020558fe2e34debb818a514715839cabd25e778) commit message - user1", pl.(*FeishuPayload).Content.Text) + }) + + t.Run("Issue", func(t *testing.T) { + p := issueTestPayload() + + d := new(FeishuPayload) + p.Action = api.HookIssueOpened + pl, err := d.Issue(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &FeishuPayload{}, pl) + + assert.Equal(t, "#2 crash\r\n[test/repo] Issue opened: #2 crash by user1\r\n\r\nissue body", pl.(*FeishuPayload).Content.Text) + + p.Action = api.HookIssueClosed + pl, err = d.Issue(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &FeishuPayload{}, pl) + + assert.Equal(t, "#2 crash\r\n[test/repo] Issue closed: #2 crash by user1", pl.(*FeishuPayload).Content.Text) + }) + + t.Run("IssueComment", func(t *testing.T) { + p := issueCommentTestPayload() + + d := new(FeishuPayload) + pl, err := d.IssueComment(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &FeishuPayload{}, pl) + + assert.Equal(t, "#2 crash\r\n[test/repo] New comment on issue #2 crash by user1\r\n\r\nmore info needed", pl.(*FeishuPayload).Content.Text) + }) + + t.Run("PullRequest", func(t *testing.T) { + p := pullRequestTestPayload() + + d := new(FeishuPayload) + pl, err := d.PullRequest(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &FeishuPayload{}, pl) + + assert.Equal(t, "#12 Fix bug\r\n[test/repo] Pull request opened: #12 Fix bug by user1\r\n\r\nfixes bug #2", pl.(*FeishuPayload).Content.Text) + }) + + t.Run("PullRequestComment", func(t *testing.T) { + p := pullRequestCommentTestPayload() + + d := new(FeishuPayload) + pl, err := d.IssueComment(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &FeishuPayload{}, pl) + + assert.Equal(t, "#12 Fix bug\r\n[test/repo] New comment on pull request #12 Fix bug by user1\r\n\r\nchanges requested", pl.(*FeishuPayload).Content.Text) + }) + + t.Run("Review", func(t *testing.T) { + p := pullRequestTestPayload() + p.Action = api.HookIssueReviewed + + d := new(FeishuPayload) + pl, err := d.Review(p, models.HookEventPullRequestReviewApproved) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &FeishuPayload{}, pl) + + assert.Equal(t, "[test/repo] Pull request review approved : #12 Fix bug\r\n\r\ngood job", pl.(*FeishuPayload).Content.Text) + }) + + t.Run("Repository", func(t *testing.T) { + p := repositoryTestPayload() + + d := new(FeishuPayload) + pl, err := d.Repository(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &FeishuPayload{}, pl) + + assert.Equal(t, "[test/repo] Repository created", pl.(*FeishuPayload).Content.Text) + }) + + t.Run("Release", func(t *testing.T) { + p := pullReleaseTestPayload() + + d := new(FeishuPayload) + pl, err := d.Release(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &FeishuPayload{}, pl) + + assert.Equal(t, "[test/repo] Release created: v1.0 by user1", pl.(*FeishuPayload).Content.Text) + }) +} + +func TestFeishuJSONPayload(t *testing.T) { + p := pushTestPayload() + + pl, err := new(FeishuPayload).Push(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &FeishuPayload{}, pl) + + json, err := pl.JSONPayload() + require.NoError(t, err) + assert.NotEmpty(t, json) +} diff --git a/services/webhook/general.go b/services/webhook/general.go index ec247a24109be..777ae086b5e96 100644 --- a/services/webhook/general.go +++ b/services/webhook/general.go @@ -44,8 +44,11 @@ func getIssuesPayloadInfo(p *api.IssuePayload, linkFormatter linkFormatter, with case api.HookIssueEdited: text = fmt.Sprintf("[%s] Issue edited: %s", repoLink, titleLink) case api.HookIssueAssigned: - text = fmt.Sprintf("[%s] Issue assigned to %s: %s", repoLink, - linkFormatter(setting.AppURL+p.Issue.Assignee.UserName, p.Issue.Assignee.UserName), titleLink) + list := make([]string, len(p.Issue.Assignees)) + for i, user := range p.Issue.Assignees { + list[i] = linkFormatter(setting.AppURL+user.UserName, user.UserName) + } + text = fmt.Sprintf("[%s] Issue assigned to %s: %s", repoLink, strings.Join(list, ", "), titleLink) color = greenColor case api.HookIssueUnassigned: text = fmt.Sprintf("[%s] Issue unassigned: %s", repoLink, titleLink) @@ -102,7 +105,7 @@ func getPullRequestPayloadInfo(p *api.PullRequestPayload, linkFormatter linkForm for i, user := range p.PullRequest.Assignees { list[i] = linkFormatter(setting.AppURL+user.UserName, user.UserName) } - text = fmt.Sprintf("[%s] Pull request assigned: %s to %s", repoLink, + text = fmt.Sprintf("[%s] Pull request assigned to %s: %s", repoLink, strings.Join(list, ", "), titleLink) color = greenColor case api.HookIssueUnassigned: @@ -115,7 +118,7 @@ func getPullRequestPayloadInfo(p *api.PullRequestPayload, linkFormatter linkForm text = fmt.Sprintf("[%s] Pull request synchronized: %s", repoLink, titleLink) case api.HookIssueMilestoned: mileStoneLink := fmt.Sprintf("%s/milestone/%d", p.Repository.HTMLURL, p.PullRequest.Milestone.ID) - text = fmt.Sprintf("[%s] Pull request milestoned: %s to %s", repoLink, + text = fmt.Sprintf("[%s] Pull request milestoned to %s: %s", repoLink, linkFormatter(mileStoneLink, p.PullRequest.Milestone.Title), titleLink) case api.HookIssueDemilestoned: text = fmt.Sprintf("[%s] Pull request milestone cleared: %s", repoLink, titleLink) diff --git a/services/webhook/general_test.go b/services/webhook/general_test.go index 3033b578805b6..4d73afe060a68 100644 --- a/services/webhook/general_test.go +++ b/services/webhook/general_test.go @@ -5,14 +5,111 @@ package webhook import ( + "testing" + api "code.gitea.io/gitea/modules/structs" + + "github.com/stretchr/testify/assert" ) +func createTestPayload() *api.CreatePayload { + return &api.CreatePayload{ + Sha: "2020558fe2e34debb818a514715839cabd25e777", + Ref: "refs/heads/test", + RefType: "branch", + Repo: &api.Repository{ + HTMLURL: "http://localhost:3000/test/repo", + Name: "repo", + FullName: "test/repo", + }, + Sender: &api.User{ + UserName: "user1", + AvatarURL: "http://localhost:3000/user1/avatar", + }, + } +} + +func deleteTestPayload() *api.DeletePayload { + return &api.DeletePayload{ + Ref: "refs/heads/test", + RefType: "branch", + Repo: &api.Repository{ + HTMLURL: "http://localhost:3000/test/repo", + Name: "repo", + FullName: "test/repo", + }, + Sender: &api.User{ + UserName: "user1", + AvatarURL: "http://localhost:3000/user1/avatar", + }, + } +} + +func forkTestPayload() *api.ForkPayload { + return &api.ForkPayload{ + Forkee: &api.Repository{ + HTMLURL: "http://localhost:3000/test/repo2", + Name: "repo2", + FullName: "test/repo2", + }, + Repo: &api.Repository{ + HTMLURL: "http://localhost:3000/test/repo", + Name: "repo", + FullName: "test/repo", + }, + Sender: &api.User{ + UserName: "user1", + AvatarURL: "http://localhost:3000/user1/avatar", + }, + } +} + +func pushTestPayload() *api.PushPayload { + commit := &api.PayloadCommit{ + ID: "2020558fe2e34debb818a514715839cabd25e778", + Message: "commit message", + URL: "http://localhost:3000/test/repo/commit/2020558fe2e34debb818a514715839cabd25e778", + Author: &api.PayloadUser{ + Name: "user1", + Email: "user1@localhost", + UserName: "user1", + }, + Committer: &api.PayloadUser{ + Name: "user1", + Email: "user1@localhost", + UserName: "user1", + }, + } + + return &api.PushPayload{ + Ref: "refs/heads/test", + Before: "2020558fe2e34debb818a514715839cabd25e777", + After: "2020558fe2e34debb818a514715839cabd25e778", + CompareURL: "", + HeadCommit: commit, + Commits: []*api.PayloadCommit{commit, commit}, + Repo: &api.Repository{ + HTMLURL: "http://localhost:3000/test/repo", + Name: "repo", + FullName: "test/repo", + }, + Pusher: &api.User{ + UserName: "user1", + AvatarURL: "http://localhost:3000/user1/avatar", + }, + Sender: &api.User{ + UserName: "user1", + AvatarURL: "http://localhost:3000/user1/avatar", + }, + } +} + func issueTestPayload() *api.IssuePayload { return &api.IssuePayload{ Index: 2, Sender: &api.User{ - UserName: "user1", + UserName: "user1", + AvatarURL: "http://localhost:3000/user1/avatar", }, Repository: &api.Repository{ HTMLURL: "http://localhost:3000/test/repo", @@ -20,10 +117,23 @@ func issueTestPayload() *api.IssuePayload { FullName: "test/repo", }, Issue: &api.Issue{ - ID: 2, - Index: 2, - URL: "http://localhost:3000/api/v1/repos/test/repo/issues/2", - Title: "crash", + ID: 2, + Index: 2, + URL: "http://localhost:3000/api/v1/repos/test/repo/issues/2", + HTMLURL: "http://localhost:3000/test/repo/issues/2", + Title: "crash", + Body: "issue body", + Assignees: []*api.User{ + { + UserName: "user1", + AvatarURL: "http://localhost:3000/user1/avatar", + }, + }, + Milestone: &api.Milestone{ + ID: 1, + Title: "Milestone Title", + Description: "Milestone Description", + }, }, } } @@ -32,7 +142,8 @@ func issueCommentTestPayload() *api.IssueCommentPayload { return &api.IssueCommentPayload{ Action: api.HookIssueCommentCreated, Sender: &api.User{ - UserName: "user1", + UserName: "user1", + AvatarURL: "http://localhost:3000/user1/avatar", }, Repository: &api.Repository{ HTMLURL: "http://localhost:3000/test/repo", @@ -45,11 +156,12 @@ func issueCommentTestPayload() *api.IssueCommentPayload { Body: "more info needed", }, Issue: &api.Issue{ - ID: 2, - Index: 2, - URL: "http://localhost:3000/api/v1/repos/test/repo/issues/2", - Title: "crash", - Body: "this happened", + ID: 2, + Index: 2, + URL: "http://localhost:3000/api/v1/repos/test/repo/issues/2", + HTMLURL: "http://localhost:3000/test/repo/issues/2", + Title: "crash", + Body: "this happened", }, } } @@ -58,7 +170,8 @@ func pullRequestCommentTestPayload() *api.IssueCommentPayload { return &api.IssueCommentPayload{ Action: api.HookIssueCommentCreated, Sender: &api.User{ - UserName: "user1", + UserName: "user1", + AvatarURL: "http://localhost:3000/user1/avatar", }, Repository: &api.Repository{ HTMLURL: "http://localhost:3000/test/repo", @@ -66,16 +179,17 @@ func pullRequestCommentTestPayload() *api.IssueCommentPayload { FullName: "test/repo", }, Comment: &api.Comment{ - HTMLURL: "http://localhost:3000/test/repo/pulls/2#issuecomment-4", - PRURL: "http://localhost:3000/test/repo/pulls/2", + HTMLURL: "http://localhost:3000/test/repo/pulls/12#issuecomment-4", + PRURL: "http://localhost:3000/test/repo/pulls/12", Body: "changes requested", }, Issue: &api.Issue{ - ID: 2, - Index: 2, - URL: "http://localhost:3000/api/v1/repos/test/repo/issues/2", - Title: "Fix bug", - Body: "fixes bug #2", + ID: 12, + Index: 12, + URL: "http://localhost:3000/api/v1/repos/test/repo/pulls/12", + HTMLURL: "http://localhost:3000/test/repo/pulls/12", + Title: "Fix bug", + Body: "fixes bug #2", }, IsPull: true, } @@ -85,7 +199,8 @@ func pullReleaseTestPayload() *api.ReleasePayload { return &api.ReleasePayload{ Action: api.HookReleasePublished, Sender: &api.User{ - UserName: "user1", + UserName: "user1", + AvatarURL: "http://localhost:3000/user1/avatar", }, Repository: &api.Repository{ HTMLURL: "http://localhost:3000/test/repo", @@ -96,6 +211,7 @@ func pullReleaseTestPayload() *api.ReleasePayload { TagName: "v1.0", Target: "master", Title: "First stable release", + Note: "Note of first stable release", URL: "http://localhost:3000/api/v1/repos/test/repo/releases/2", }, } @@ -104,9 +220,10 @@ func pullReleaseTestPayload() *api.ReleasePayload { func pullRequestTestPayload() *api.PullRequestPayload { return &api.PullRequestPayload{ Action: api.HookIssueOpened, - Index: 2, + Index: 12, Sender: &api.User{ - UserName: "user1", + UserName: "user1", + AvatarURL: "http://localhost:3000/user1/avatar", }, Repository: &api.Repository{ HTMLURL: "http://localhost:3000/test/repo", @@ -114,12 +231,311 @@ func pullRequestTestPayload() *api.PullRequestPayload { FullName: "test/repo", }, PullRequest: &api.PullRequest{ - ID: 2, - Index: 2, + ID: 12, + Index: 12, URL: "http://localhost:3000/test/repo/pulls/12", + HTMLURL: "http://localhost:3000/test/repo/pulls/12", Title: "Fix bug", Body: "fixes bug #2", Mergeable: true, + Assignees: []*api.User{ + { + UserName: "user1", + AvatarURL: "http://localhost:3000/user1/avatar", + }, + }, + Milestone: &api.Milestone{ + ID: 1, + Title: "Milestone Title", + Description: "Milestone Description", + }, + }, + Review: &api.ReviewPayload{ + Content: "good job", + }, + } +} + +func repositoryTestPayload() *api.RepositoryPayload { + return &api.RepositoryPayload{ + Action: api.HookRepoCreated, + Sender: &api.User{ + UserName: "user1", + AvatarURL: "http://localhost:3000/user1/avatar", }, + Repository: &api.Repository{ + HTMLURL: "http://localhost:3000/test/repo", + Name: "repo", + FullName: "test/repo", + }, + } +} + +func TestGetIssuesPayloadInfo(t *testing.T) { + p := issueTestPayload() + + cases := []struct { + action api.HookIssueAction + text string + issueTitle string + attachmentText string + color int + }{ + { + api.HookIssueOpened, + "[test/repo] Issue opened: #2 crash by user1", + "#2 crash", + "issue body", + orangeColor, + }, + { + api.HookIssueClosed, + "[test/repo] Issue closed: #2 crash by user1", + "#2 crash", + "", + redColor, + }, + { + api.HookIssueReOpened, + "[test/repo] Issue re-opened: #2 crash by user1", + "#2 crash", + "", + yellowColor, + }, + { + api.HookIssueEdited, + "[test/repo] Issue edited: #2 crash by user1", + "#2 crash", + "issue body", + yellowColor, + }, + { + api.HookIssueAssigned, + "[test/repo] Issue assigned to user1: #2 crash by user1", + "#2 crash", + "", + greenColor, + }, + { + api.HookIssueUnassigned, + "[test/repo] Issue unassigned: #2 crash by user1", + "#2 crash", + "", + yellowColor, + }, + { + api.HookIssueLabelUpdated, + "[test/repo] Issue labels updated: #2 crash by user1", + "#2 crash", + "", + yellowColor, + }, + { + api.HookIssueLabelCleared, + "[test/repo] Issue labels cleared: #2 crash by user1", + "#2 crash", + "", + yellowColor, + }, + { + api.HookIssueSynchronized, + "[test/repo] Issue synchronized: #2 crash by user1", + "#2 crash", + "", + yellowColor, + }, + { + api.HookIssueMilestoned, + "[test/repo] Issue milestoned to Milestone Title: #2 crash by user1", + "#2 crash", + "", + yellowColor, + }, + { + api.HookIssueDemilestoned, + "[test/repo] Issue milestone cleared: #2 crash by user1", + "#2 crash", + "", + yellowColor, + }, + } + + for i, c := range cases { + p.Action = c.action + text, issueTitle, attachmentText, color := getIssuesPayloadInfo(p, noneLinkFormatter, true) + assert.Equal(t, c.text, text, "case %d", i) + assert.Equal(t, c.issueTitle, issueTitle, "case %d", i) + assert.Equal(t, c.attachmentText, attachmentText, "case %d", i) + assert.Equal(t, c.color, color, "case %d", i) + } +} + +func TestGetPullRequestPayloadInfo(t *testing.T) { + p := pullRequestTestPayload() + + cases := []struct { + action api.HookIssueAction + text string + issueTitle string + attachmentText string + color int + }{ + { + api.HookIssueOpened, + "[test/repo] Pull request opened: #12 Fix bug by user1", + "#12 Fix bug", + "fixes bug #2", + greenColor, + }, + { + api.HookIssueClosed, + "[test/repo] Pull request closed: #12 Fix bug by user1", + "#12 Fix bug", + "", + redColor, + }, + { + api.HookIssueReOpened, + "[test/repo] Pull request re-opened: #12 Fix bug by user1", + "#12 Fix bug", + "", + yellowColor, + }, + { + api.HookIssueEdited, + "[test/repo] Pull request edited: #12 Fix bug by user1", + "#12 Fix bug", + "fixes bug #2", + yellowColor, + }, + { + api.HookIssueAssigned, + "[test/repo] Pull request assigned to user1: #12 Fix bug by user1", + "#12 Fix bug", + "", + greenColor, + }, + { + api.HookIssueUnassigned, + "[test/repo] Pull request unassigned: #12 Fix bug by user1", + "#12 Fix bug", + "", + yellowColor, + }, + { + api.HookIssueLabelUpdated, + "[test/repo] Pull request labels updated: #12 Fix bug by user1", + "#12 Fix bug", + "", + yellowColor, + }, + { + api.HookIssueLabelCleared, + "[test/repo] Pull request labels cleared: #12 Fix bug by user1", + "#12 Fix bug", + "", + yellowColor, + }, + { + api.HookIssueSynchronized, + "[test/repo] Pull request synchronized: #12 Fix bug by user1", + "#12 Fix bug", + "", + yellowColor, + }, + { + api.HookIssueMilestoned, + "[test/repo] Pull request milestoned to Milestone Title: #12 Fix bug by user1", + "#12 Fix bug", + "", + yellowColor, + }, + { + api.HookIssueDemilestoned, + "[test/repo] Pull request milestone cleared: #12 Fix bug by user1", + "#12 Fix bug", + "", + yellowColor, + }, + } + + for i, c := range cases { + p.Action = c.action + text, issueTitle, attachmentText, color := getPullRequestPayloadInfo(p, noneLinkFormatter, true) + assert.Equal(t, c.text, text, "case %d", i) + assert.Equal(t, c.issueTitle, issueTitle, "case %d", i) + assert.Equal(t, c.attachmentText, attachmentText, "case %d", i) + assert.Equal(t, c.color, color, "case %d", i) + } +} + +func TestGetReleasePayloadInfo(t *testing.T) { + p := pullReleaseTestPayload() + + cases := []struct { + action api.HookReleaseAction + text string + color int + }{ + { + api.HookReleasePublished, + "[test/repo] Release created: v1.0 by user1", + greenColor, + }, + { + api.HookReleaseUpdated, + "[test/repo] Release updated: v1.0 by user1", + yellowColor, + }, + { + api.HookReleaseDeleted, + "[test/repo] Release deleted: v1.0 by user1", + redColor, + }, + } + + for i, c := range cases { + p.Action = c.action + text, color := getReleasePayloadInfo(p, noneLinkFormatter, true) + assert.Equal(t, c.text, text, "case %d", i) + assert.Equal(t, c.color, color, "case %d", i) + } +} + +func TestGetIssueCommentPayloadInfo(t *testing.T) { + p := pullRequestCommentTestPayload() + + cases := []struct { + action api.HookIssueCommentAction + text string + issueTitle string + color int + }{ + { + api.HookIssueCommentCreated, + "[test/repo] New comment on pull request #12 Fix bug by user1", + "#12 Fix bug", + greenColorLight, + }, + { + api.HookIssueCommentEdited, + "[test/repo] Comment edited on pull request #12 Fix bug by user1", + "#12 Fix bug", + yellowColor, + }, + { + api.HookIssueCommentDeleted, + "[test/repo] Comment deleted on pull request #12 Fix bug by user1", + "#12 Fix bug", + redColor, + }, + } + + for i, c := range cases { + p.Action = c.action + text, issueTitle, color := getIssueCommentPayloadInfo(p, noneLinkFormatter, true) + assert.Equal(t, c.text, text, "case %d", i) + assert.Equal(t, c.issueTitle, issueTitle, "case %d", i) + assert.Equal(t, c.color, color, "case %d", i) } } diff --git a/services/webhook/matrix_test.go b/services/webhook/matrix_test.go index 771146f2f30fe..7b10e21cfa4a3 100644 --- a/services/webhook/matrix_test.go +++ b/services/webhook/matrix_test.go @@ -14,71 +14,173 @@ import ( "github.com/stretchr/testify/require" ) -func TestMatrixIssuesPayloadOpened(t *testing.T) { - p := issueTestPayload() - m := new(MatrixPayloadUnsafe) +func TestMatrixPayload(t *testing.T) { + t.Run("Create", func(t *testing.T) { + p := createTestPayload() - p.Action = api.HookIssueOpened - pl, err := m.Issue(p) - require.NoError(t, err) - require.NotNil(t, pl) - assert.Equal(t, "[[test/repo](http://localhost:3000/test/repo)] Issue opened: [#2 crash](http://localhost:3000/test/repo/issues/2) by [user1](https://try.gitea.io/user1)", pl.(*MatrixPayloadUnsafe).Body) - assert.Equal(t, "[test/repo] Issue opened: #2 crash by user1", pl.(*MatrixPayloadUnsafe).FormattedBody) + d := new(MatrixPayloadUnsafe) + pl, err := d.Create(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &MatrixPayloadUnsafe{}, pl) - p.Action = api.HookIssueClosed - pl, err = m.Issue(p) - require.NoError(t, err) - require.NotNil(t, pl) - assert.Equal(t, "[[test/repo](http://localhost:3000/test/repo)] Issue closed: [#2 crash](http://localhost:3000/test/repo/issues/2) by [user1](https://try.gitea.io/user1)", pl.(*MatrixPayloadUnsafe).Body) - assert.Equal(t, "[test/repo] Issue closed: #2 crash by user1", pl.(*MatrixPayloadUnsafe).FormattedBody) -} + assert.Equal(t, "[[test/repo](http://localhost:3000/test/repo):[test](http://localhost:3000/test/repo/src/branch/test)] branch created by user1", pl.(*MatrixPayloadUnsafe).Body) + assert.Equal(t, `[test/repo:test] branch created by user1`, pl.(*MatrixPayloadUnsafe).FormattedBody) + }) -func TestMatrixIssueCommentPayload(t *testing.T) { - p := issueCommentTestPayload() - m := new(MatrixPayloadUnsafe) + t.Run("Delete", func(t *testing.T) { + p := deleteTestPayload() - pl, err := m.IssueComment(p) - require.NoError(t, err) - require.NotNil(t, pl) + d := new(MatrixPayloadUnsafe) + pl, err := d.Delete(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &MatrixPayloadUnsafe{}, pl) - assert.Equal(t, "[[test/repo](http://localhost:3000/test/repo)] New comment on issue [#2 crash](http://localhost:3000/test/repo/issues/2) by [user1](https://try.gitea.io/user1)", pl.(*MatrixPayloadUnsafe).Body) - assert.Equal(t, "[test/repo] New comment on issue #2 crash by user1", pl.(*MatrixPayloadUnsafe).FormattedBody) -} + assert.Equal(t, "[[test/repo](http://localhost:3000/test/repo):test] branch deleted by user1", pl.(*MatrixPayloadUnsafe).Body) + assert.Equal(t, `[test/repo:test] branch deleted by user1`, pl.(*MatrixPayloadUnsafe).FormattedBody) + }) -func TestMatrixPullRequestCommentPayload(t *testing.T) { - p := pullRequestCommentTestPayload() - m := new(MatrixPayloadUnsafe) + t.Run("Fork", func(t *testing.T) { + p := forkTestPayload() - pl, err := m.IssueComment(p) - require.NoError(t, err) - require.NotNil(t, pl) + d := new(MatrixPayloadUnsafe) + pl, err := d.Fork(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &MatrixPayloadUnsafe{}, pl) - assert.Equal(t, "[[test/repo](http://localhost:3000/test/repo)] New comment on pull request [#2 Fix bug](http://localhost:3000/test/repo/pulls/2) by [user1](https://try.gitea.io/user1)", pl.(*MatrixPayloadUnsafe).Body) - assert.Equal(t, "[test/repo] New comment on pull request #2 Fix bug by user1", pl.(*MatrixPayloadUnsafe).FormattedBody) -} + assert.Equal(t, "[test/repo2](http://localhost:3000/test/repo2) is forked to [test/repo](http://localhost:3000/test/repo)", pl.(*MatrixPayloadUnsafe).Body) + assert.Equal(t, `test/repo2 is forked to test/repo`, pl.(*MatrixPayloadUnsafe).FormattedBody) + }) -func TestMatrixReleasePayload(t *testing.T) { - p := pullReleaseTestPayload() - m := new(MatrixPayloadUnsafe) + t.Run("Push", func(t *testing.T) { + p := pushTestPayload() - pl, err := m.Release(p) - require.NoError(t, err) - require.NotNil(t, pl) + d := new(MatrixPayloadUnsafe) + pl, err := d.Push(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &MatrixPayloadUnsafe{}, pl) + + assert.Equal(t, "[[test/repo](http://localhost:3000/test/repo)] user1 pushed 2 commits to [test](http://localhost:3000/test/repo/src/branch/test):\n[2020558](http://localhost:3000/test/repo/commit/2020558fe2e34debb818a514715839cabd25e778): commit message - user1\n[2020558](http://localhost:3000/test/repo/commit/2020558fe2e34debb818a514715839cabd25e778): commit message - user1", pl.(*MatrixPayloadUnsafe).Body) + assert.Equal(t, `[test/repo] user1 pushed 2 commits to test:
2020558: commit message - user1
2020558: commit message - user1`, pl.(*MatrixPayloadUnsafe).FormattedBody) + }) + + t.Run("Issue", func(t *testing.T) { + p := issueTestPayload() + + d := new(MatrixPayloadUnsafe) + p.Action = api.HookIssueOpened + pl, err := d.Issue(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &MatrixPayloadUnsafe{}, pl) + + assert.Equal(t, "[[test/repo](http://localhost:3000/test/repo)] Issue opened: [#2 crash](http://localhost:3000/test/repo/issues/2) by [user1](https://try.gitea.io/user1)", pl.(*MatrixPayloadUnsafe).Body) + assert.Equal(t, `[test/repo] Issue opened: #2 crash by user1`, pl.(*MatrixPayloadUnsafe).FormattedBody) + + p.Action = api.HookIssueClosed + pl, err = d.Issue(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &MatrixPayloadUnsafe{}, pl) + + assert.Equal(t, "[[test/repo](http://localhost:3000/test/repo)] Issue closed: [#2 crash](http://localhost:3000/test/repo/issues/2) by [user1](https://try.gitea.io/user1)", pl.(*MatrixPayloadUnsafe).Body) + assert.Equal(t, `[test/repo] Issue closed: #2 crash by user1`, pl.(*MatrixPayloadUnsafe).FormattedBody) + }) - assert.Equal(t, "[[test/repo](http://localhost:3000/test/repo)] Release created: [v1.0](http://localhost:3000/test/repo/src/v1.0) by [user1](https://try.gitea.io/user1)", pl.(*MatrixPayloadUnsafe).Body) - assert.Equal(t, "[test/repo] Release created: v1.0 by user1", pl.(*MatrixPayloadUnsafe).FormattedBody) + t.Run("IssueComment", func(t *testing.T) { + p := issueCommentTestPayload() + + d := new(MatrixPayloadUnsafe) + pl, err := d.IssueComment(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &MatrixPayloadUnsafe{}, pl) + + assert.Equal(t, "[[test/repo](http://localhost:3000/test/repo)] New comment on issue [#2 crash](http://localhost:3000/test/repo/issues/2) by [user1](https://try.gitea.io/user1)", pl.(*MatrixPayloadUnsafe).Body) + assert.Equal(t, `[test/repo] New comment on issue #2 crash by user1`, pl.(*MatrixPayloadUnsafe).FormattedBody) + }) + + t.Run("PullRequest", func(t *testing.T) { + p := pullRequestTestPayload() + + d := new(MatrixPayloadUnsafe) + pl, err := d.PullRequest(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &MatrixPayloadUnsafe{}, pl) + + assert.Equal(t, "[[test/repo](http://localhost:3000/test/repo)] Pull request opened: [#12 Fix bug](http://localhost:3000/test/repo/pulls/12) by [user1](https://try.gitea.io/user1)", pl.(*MatrixPayloadUnsafe).Body) + assert.Equal(t, `[test/repo] Pull request opened: #12 Fix bug by user1`, pl.(*MatrixPayloadUnsafe).FormattedBody) + }) + + t.Run("PullRequestComment", func(t *testing.T) { + p := pullRequestCommentTestPayload() + + d := new(MatrixPayloadUnsafe) + pl, err := d.IssueComment(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &MatrixPayloadUnsafe{}, pl) + + assert.Equal(t, "[[test/repo](http://localhost:3000/test/repo)] New comment on pull request [#12 Fix bug](http://localhost:3000/test/repo/pulls/12) by [user1](https://try.gitea.io/user1)", pl.(*MatrixPayloadUnsafe).Body) + assert.Equal(t, `[test/repo] New comment on pull request #12 Fix bug by user1`, pl.(*MatrixPayloadUnsafe).FormattedBody) + }) + + t.Run("Review", func(t *testing.T) { + p := pullRequestTestPayload() + p.Action = api.HookIssueReviewed + + d := new(MatrixPayloadUnsafe) + pl, err := d.Review(p, models.HookEventPullRequestReviewApproved) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &MatrixPayloadUnsafe{}, pl) + + assert.Equal(t, "[[test/repo](http://localhost:3000/test/repo)] Pull request review approved: [#12 Fix bug](http://localhost:3000/test/repo/pulls/12) by [user1](https://try.gitea.io/user1)", pl.(*MatrixPayloadUnsafe).Body) + assert.Equal(t, `[test/repo] Pull request review approved: [#12 Fix bug](http://localhost:3000/test/repo/pulls/12) by user1`, pl.(*MatrixPayloadUnsafe).FormattedBody) + }) + + t.Run("Repository", func(t *testing.T) { + p := repositoryTestPayload() + + d := new(MatrixPayloadUnsafe) + pl, err := d.Repository(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &MatrixPayloadUnsafe{}, pl) + + assert.Equal(t, `[[test/repo](http://localhost:3000/test/repo)] Repository created by [user1](https://try.gitea.io/user1)`, pl.(*MatrixPayloadUnsafe).Body) + assert.Equal(t, `[test/repo] Repository created by user1`, pl.(*MatrixPayloadUnsafe).FormattedBody) + }) + + t.Run("Release", func(t *testing.T) { + p := pullReleaseTestPayload() + + d := new(MatrixPayloadUnsafe) + pl, err := d.Release(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &MatrixPayloadUnsafe{}, pl) + + assert.Equal(t, "[[test/repo](http://localhost:3000/test/repo)] Release created: [v1.0](http://localhost:3000/test/repo/src/v1.0) by [user1](https://try.gitea.io/user1)", pl.(*MatrixPayloadUnsafe).Body) + assert.Equal(t, `[test/repo] Release created: v1.0 by user1`, pl.(*MatrixPayloadUnsafe).FormattedBody) + }) } -func TestMatrixPullRequestPayload(t *testing.T) { - p := pullRequestTestPayload() - m := new(MatrixPayloadUnsafe) +func TestMatrixJSONPayload(t *testing.T) { + p := pushTestPayload() - pl, err := m.PullRequest(p) + pl, err := new(MatrixPayloadUnsafe).Push(p) require.NoError(t, err) require.NotNil(t, pl) + require.IsType(t, &MatrixPayloadUnsafe{}, pl) - assert.Equal(t, "[[test/repo](http://localhost:3000/test/repo)] Pull request opened: [#2 Fix bug](http://localhost:3000/test/repo/pulls/12) by [user1](https://try.gitea.io/user1)", pl.(*MatrixPayloadUnsafe).Body) - assert.Equal(t, "[test/repo] Pull request opened: #2 Fix bug by user1", pl.(*MatrixPayloadUnsafe).FormattedBody) + json, err := pl.JSONPayload() + require.NoError(t, err) + assert.NotEmpty(t, json) } func TestMatrixHookRequest(t *testing.T) { diff --git a/services/webhook/msteams.go b/services/webhook/msteams.go index dc83a47c8d801..51bb28a36181d 100644 --- a/services/webhook/msteams.go +++ b/services/webhook/msteams.go @@ -78,42 +78,15 @@ func (m *MSTeamsPayload) Create(p *api.CreatePayload) (api.Payloader, error) { refName := git.RefEndName(p.Ref) title := fmt.Sprintf("[%s] %s %s created", p.Repo.FullName, p.RefType, refName) - return &MSTeamsPayload{ - Type: "MessageCard", - Context: "https://schema.org/extensions", - ThemeColor: fmt.Sprintf("%x", greenColor), - Title: title, - Summary: title, - Sections: []MSTeamsSection{ - { - ActivityTitle: p.Sender.FullName, - ActivitySubtitle: p.Sender.UserName, - ActivityImage: p.Sender.AvatarURL, - Facts: []MSTeamsFact{ - { - Name: "Repository:", - Value: p.Repo.FullName, - }, - { - Name: fmt.Sprintf("%s:", p.RefType), - Value: refName, - }, - }, - }, - }, - PotentialAction: []MSTeamsAction{ - { - Type: "OpenUri", - Name: "View in Gitea", - Targets: []MSTeamsActionTarget{ - { - Os: "default", - URI: p.Repo.HTMLURL + "/src/" + refName, - }, - }, - }, - }, - }, nil + return createMSTeamsPayload( + p.Repo, + p.Sender, + title, + "", + p.Repo.HTMLURL+"/src/"+refName, + greenColor, + &MSTeamsFact{fmt.Sprintf("%s:", p.RefType), refName}, + ), nil } // Delete implements PayloadConvertor Delete method @@ -122,84 +95,30 @@ func (m *MSTeamsPayload) Delete(p *api.DeletePayload) (api.Payloader, error) { refName := git.RefEndName(p.Ref) title := fmt.Sprintf("[%s] %s %s deleted", p.Repo.FullName, p.RefType, refName) - return &MSTeamsPayload{ - Type: "MessageCard", - Context: "https://schema.org/extensions", - ThemeColor: fmt.Sprintf("%x", yellowColor), - Title: title, - Summary: title, - Sections: []MSTeamsSection{ - { - ActivityTitle: p.Sender.FullName, - ActivitySubtitle: p.Sender.UserName, - ActivityImage: p.Sender.AvatarURL, - Facts: []MSTeamsFact{ - { - Name: "Repository:", - Value: p.Repo.FullName, - }, - { - Name: fmt.Sprintf("%s:", p.RefType), - Value: refName, - }, - }, - }, - }, - PotentialAction: []MSTeamsAction{ - { - Type: "OpenUri", - Name: "View in Gitea", - Targets: []MSTeamsActionTarget{ - { - Os: "default", - URI: p.Repo.HTMLURL + "/src/" + refName, - }, - }, - }, - }, - }, nil + return createMSTeamsPayload( + p.Repo, + p.Sender, + title, + "", + p.Repo.HTMLURL+"/src/"+refName, + yellowColor, + &MSTeamsFact{fmt.Sprintf("%s:", p.RefType), refName}, + ), nil } // Fork implements PayloadConvertor Fork method func (m *MSTeamsPayload) Fork(p *api.ForkPayload) (api.Payloader, error) { title := fmt.Sprintf("%s is forked to %s", p.Forkee.FullName, p.Repo.FullName) - return &MSTeamsPayload{ - Type: "MessageCard", - Context: "https://schema.org/extensions", - ThemeColor: fmt.Sprintf("%x", greenColor), - Title: title, - Summary: title, - Sections: []MSTeamsSection{ - { - ActivityTitle: p.Sender.FullName, - ActivitySubtitle: p.Sender.UserName, - ActivityImage: p.Sender.AvatarURL, - Facts: []MSTeamsFact{ - { - Name: "Forkee:", - Value: p.Forkee.FullName, - }, - { - Name: "Repository:", - Value: p.Repo.FullName, - }, - }, - }, - }, - PotentialAction: []MSTeamsAction{ - { - Type: "OpenUri", - Name: "View in Gitea", - Targets: []MSTeamsActionTarget{ - { - Os: "default", - URI: p.Repo.HTMLURL, - }, - }, - }, - }, - }, nil + return createMSTeamsPayload( + p.Repo, + p.Sender, + title, + "", + p.Repo.HTMLURL, + greenColor, + &MSTeamsFact{"Forkee:", p.Forkee.FullName}, + ), nil } // Push implements PayloadConvertor Push method @@ -234,172 +153,60 @@ func (m *MSTeamsPayload) Push(p *api.PushPayload) (api.Payloader, error) { } } - return &MSTeamsPayload{ - Type: "MessageCard", - Context: "https://schema.org/extensions", - ThemeColor: fmt.Sprintf("%x", greenColor), - Title: title, - Summary: title, - Sections: []MSTeamsSection{ - { - ActivityTitle: p.Sender.FullName, - ActivitySubtitle: p.Sender.UserName, - ActivityImage: p.Sender.AvatarURL, - Text: text, - Facts: []MSTeamsFact{ - { - Name: "Repository:", - Value: p.Repo.FullName, - }, - { - Name: "Commit count:", - Value: fmt.Sprintf("%d", len(p.Commits)), - }, - }, - }, - }, - PotentialAction: []MSTeamsAction{ - { - Type: "OpenUri", - Name: "View in Gitea", - Targets: []MSTeamsActionTarget{ - { - Os: "default", - URI: titleLink, - }, - }, - }, - }, - }, nil + return createMSTeamsPayload( + p.Repo, + p.Sender, + title, + text, + titleLink, + greenColor, + &MSTeamsFact{"Commit count:", fmt.Sprintf("%d", len(p.Commits))}, + ), nil } // Issue implements PayloadConvertor Issue method func (m *MSTeamsPayload) Issue(p *api.IssuePayload) (api.Payloader, error) { - text, _, attachmentText, color := getIssuesPayloadInfo(p, noneLinkFormatter, false) - - return &MSTeamsPayload{ - Type: "MessageCard", - Context: "https://schema.org/extensions", - ThemeColor: fmt.Sprintf("%x", color), - Title: text, - Summary: text, - Sections: []MSTeamsSection{ - { - ActivityTitle: p.Sender.FullName, - ActivitySubtitle: p.Sender.UserName, - ActivityImage: p.Sender.AvatarURL, - Text: attachmentText, - Facts: []MSTeamsFact{ - { - Name: "Repository:", - Value: p.Repository.FullName, - }, - { - Name: "Issue #:", - Value: fmt.Sprintf("%d", p.Issue.ID), - }, - }, - }, - }, - PotentialAction: []MSTeamsAction{ - { - Type: "OpenUri", - Name: "View in Gitea", - Targets: []MSTeamsActionTarget{ - { - Os: "default", - URI: p.Issue.HTMLURL, - }, - }, - }, - }, - }, nil + title, _, attachmentText, color := getIssuesPayloadInfo(p, noneLinkFormatter, false) + + return createMSTeamsPayload( + p.Repository, + p.Sender, + title, + attachmentText, + p.Issue.HTMLURL, + color, + &MSTeamsFact{"Issue #:", fmt.Sprintf("%d", p.Issue.ID)}, + ), nil } // IssueComment implements PayloadConvertor IssueComment method func (m *MSTeamsPayload) IssueComment(p *api.IssueCommentPayload) (api.Payloader, error) { - text, _, color := getIssueCommentPayloadInfo(p, noneLinkFormatter, false) - - return &MSTeamsPayload{ - Type: "MessageCard", - Context: "https://schema.org/extensions", - ThemeColor: fmt.Sprintf("%x", color), - Title: text, - Summary: text, - Sections: []MSTeamsSection{ - { - ActivityTitle: p.Sender.FullName, - ActivitySubtitle: p.Sender.UserName, - ActivityImage: p.Sender.AvatarURL, - Text: p.Comment.Body, - Facts: []MSTeamsFact{ - { - Name: "Repository:", - Value: p.Repository.FullName, - }, - { - Name: "Issue #:", - Value: fmt.Sprintf("%d", p.Issue.ID), - }, - }, - }, - }, - PotentialAction: []MSTeamsAction{ - { - Type: "OpenUri", - Name: "View in Gitea", - Targets: []MSTeamsActionTarget{ - { - Os: "default", - URI: p.Comment.HTMLURL, - }, - }, - }, - }, - }, nil + title, _, color := getIssueCommentPayloadInfo(p, noneLinkFormatter, false) + + return createMSTeamsPayload( + p.Repository, + p.Sender, + title, + p.Comment.Body, + p.Comment.HTMLURL, + color, + &MSTeamsFact{"Issue #:", fmt.Sprintf("%d", p.Issue.ID)}, + ), nil } // PullRequest implements PayloadConvertor PullRequest method func (m *MSTeamsPayload) PullRequest(p *api.PullRequestPayload) (api.Payloader, error) { - text, _, attachmentText, color := getPullRequestPayloadInfo(p, noneLinkFormatter, false) - - return &MSTeamsPayload{ - Type: "MessageCard", - Context: "https://schema.org/extensions", - ThemeColor: fmt.Sprintf("%x", color), - Title: text, - Summary: text, - Sections: []MSTeamsSection{ - { - ActivityTitle: p.Sender.FullName, - ActivitySubtitle: p.Sender.UserName, - ActivityImage: p.Sender.AvatarURL, - Text: attachmentText, - Facts: []MSTeamsFact{ - { - Name: "Repository:", - Value: p.Repository.FullName, - }, - { - Name: "Pull request #:", - Value: fmt.Sprintf("%d", p.PullRequest.ID), - }, - }, - }, - }, - PotentialAction: []MSTeamsAction{ - { - Type: "OpenUri", - Name: "View in Gitea", - Targets: []MSTeamsActionTarget{ - { - Os: "default", - URI: p.PullRequest.HTMLURL, - }, - }, - }, - }, - }, nil + title, _, attachmentText, color := getPullRequestPayloadInfo(p, noneLinkFormatter, false) + + return createMSTeamsPayload( + p.Repository, + p.Sender, + title, + attachmentText, + p.PullRequest.HTMLURL, + color, + &MSTeamsFact{"Pull request #:", fmt.Sprintf("%d", p.PullRequest.ID)}, + ), nil } // Review implements PayloadConvertor Review method @@ -428,43 +235,15 @@ func (m *MSTeamsPayload) Review(p *api.PullRequestPayload, event models.HookEven } } - return &MSTeamsPayload{ - Type: "MessageCard", - Context: "https://schema.org/extensions", - ThemeColor: fmt.Sprintf("%x", color), - Title: title, - Summary: title, - Sections: []MSTeamsSection{ - { - ActivityTitle: p.Sender.FullName, - ActivitySubtitle: p.Sender.UserName, - ActivityImage: p.Sender.AvatarURL, - Text: text, - Facts: []MSTeamsFact{ - { - Name: "Repository:", - Value: p.Repository.FullName, - }, - { - Name: "Pull request #:", - Value: fmt.Sprintf("%d", p.PullRequest.ID), - }, - }, - }, - }, - PotentialAction: []MSTeamsAction{ - { - Type: "OpenUri", - Name: "View in Gitea", - Targets: []MSTeamsActionTarget{ - { - Os: "default", - URI: p.PullRequest.HTMLURL, - }, - }, - }, - }, - }, nil + return createMSTeamsPayload( + p.Repository, + p.Sender, + title, + text, + p.PullRequest.HTMLURL, + color, + &MSTeamsFact{"Pull request #:", fmt.Sprintf("%d", p.PullRequest.ID)}, + ), nil } // Repository implements PayloadConvertor Repository method @@ -481,66 +260,61 @@ func (m *MSTeamsPayload) Repository(p *api.RepositoryPayload) (api.Payloader, er color = yellowColor } - return &MSTeamsPayload{ - Type: "MessageCard", - Context: "https://schema.org/extensions", - ThemeColor: fmt.Sprintf("%x", color), - Title: title, - Summary: title, - Sections: []MSTeamsSection{ - { - ActivityTitle: p.Sender.FullName, - ActivitySubtitle: p.Sender.UserName, - ActivityImage: p.Sender.AvatarURL, - Facts: []MSTeamsFact{ - { - Name: "Repository:", - Value: p.Repository.FullName, - }, - }, - }, - }, - PotentialAction: []MSTeamsAction{ - { - Type: "OpenUri", - Name: "View in Gitea", - Targets: []MSTeamsActionTarget{ - { - Os: "default", - URI: url, - }, - }, - }, - }, - }, nil + return createMSTeamsPayload( + p.Repository, + p.Sender, + title, + "", + url, + color, + nil, + ), nil } // Release implements PayloadConvertor Release method func (m *MSTeamsPayload) Release(p *api.ReleasePayload) (api.Payloader, error) { - text, color := getReleasePayloadInfo(p, noneLinkFormatter, false) + title, color := getReleasePayloadInfo(p, noneLinkFormatter, false) + + return createMSTeamsPayload( + p.Repository, + p.Sender, + title, + "", + p.Release.URL, + color, + &MSTeamsFact{"Tag:", p.Release.TagName}, + ), nil +} + +// GetMSTeamsPayload converts a MSTeams webhook into a MSTeamsPayload +func GetMSTeamsPayload(p api.Payloader, event models.HookEventType, meta string) (api.Payloader, error) { + return convertPayloader(new(MSTeamsPayload), p, event) +} + +func createMSTeamsPayload(r *api.Repository, s *api.User, title, text, actionTarget string, color int, fact *MSTeamsFact) *MSTeamsPayload { + facts := []MSTeamsFact{ + { + Name: "Repository:", + Value: r.FullName, + }, + } + if fact != nil { + facts = append(facts, *fact) + } return &MSTeamsPayload{ Type: "MessageCard", Context: "https://schema.org/extensions", ThemeColor: fmt.Sprintf("%x", color), - Title: text, - Summary: text, + Title: title, + Summary: title, Sections: []MSTeamsSection{ { - ActivityTitle: p.Sender.FullName, - ActivitySubtitle: p.Sender.UserName, - ActivityImage: p.Sender.AvatarURL, - Text: p.Release.Note, - Facts: []MSTeamsFact{ - { - Name: "Repository:", - Value: p.Repository.FullName, - }, - { - Name: "Tag:", - Value: p.Release.TagName, - }, - }, + ActivityTitle: s.FullName, + ActivitySubtitle: s.UserName, + ActivityImage: s.AvatarURL, + Text: text, + Facts: facts, }, }, PotentialAction: []MSTeamsAction{ @@ -550,15 +324,10 @@ func (m *MSTeamsPayload) Release(p *api.ReleasePayload) (api.Payloader, error) { Targets: []MSTeamsActionTarget{ { Os: "default", - URI: p.Release.URL, + URI: actionTarget, }, }, }, }, - }, nil -} - -// GetMSTeamsPayload converts a MSTeams webhook into a MSTeamsPayload -func GetMSTeamsPayload(p api.Payloader, event models.HookEventType, meta string) (api.Payloader, error) { - return convertPayloader(new(MSTeamsPayload), p, event) + } } diff --git a/services/webhook/msteams_test.go b/services/webhook/msteams_test.go new file mode 100644 index 0000000000000..2f54c39d396b3 --- /dev/null +++ b/services/webhook/msteams_test.go @@ -0,0 +1,374 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package webhook + +import ( + "testing" + + "code.gitea.io/gitea/models" + api "code.gitea.io/gitea/modules/structs" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMSTeamsPayload(t *testing.T) { + t.Run("Create", func(t *testing.T) { + p := createTestPayload() + + d := new(MSTeamsPayload) + pl, err := d.Create(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &MSTeamsPayload{}, pl) + + assert.Equal(t, "[test/repo] branch test created", pl.(*MSTeamsPayload).Title) + assert.Equal(t, "[test/repo] branch test created", pl.(*MSTeamsPayload).Summary) + assert.Len(t, pl.(*MSTeamsPayload).Sections, 1) + assert.Equal(t, "user1", pl.(*MSTeamsPayload).Sections[0].ActivitySubtitle) + assert.Empty(t, pl.(*MSTeamsPayload).Sections[0].Text) + assert.Len(t, pl.(*MSTeamsPayload).Sections[0].Facts, 2) + for _, fact := range pl.(*MSTeamsPayload).Sections[0].Facts { + if fact.Name == "Repository:" { + assert.Equal(t, p.Repo.FullName, fact.Value) + } else if fact.Name == "branch:" { + assert.Equal(t, "test", fact.Value) + } else { + t.Fail() + } + } + assert.Len(t, pl.(*MSTeamsPayload).PotentialAction, 1) + assert.Len(t, pl.(*MSTeamsPayload).PotentialAction[0].Targets, 1) + assert.Equal(t, "http://localhost:3000/test/repo/src/test", pl.(*MSTeamsPayload).PotentialAction[0].Targets[0].URI) + }) + + t.Run("Delete", func(t *testing.T) { + p := deleteTestPayload() + + d := new(MSTeamsPayload) + pl, err := d.Delete(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &MSTeamsPayload{}, pl) + + assert.Equal(t, "[test/repo] branch test deleted", pl.(*MSTeamsPayload).Title) + assert.Equal(t, "[test/repo] branch test deleted", pl.(*MSTeamsPayload).Summary) + assert.Len(t, pl.(*MSTeamsPayload).Sections, 1) + assert.Equal(t, "user1", pl.(*MSTeamsPayload).Sections[0].ActivitySubtitle) + assert.Empty(t, pl.(*MSTeamsPayload).Sections[0].Text) + assert.Len(t, pl.(*MSTeamsPayload).Sections[0].Facts, 2) + for _, fact := range pl.(*MSTeamsPayload).Sections[0].Facts { + if fact.Name == "Repository:" { + assert.Equal(t, p.Repo.FullName, fact.Value) + } else if fact.Name == "branch:" { + assert.Equal(t, "test", fact.Value) + } else { + t.Fail() + } + } + assert.Len(t, pl.(*MSTeamsPayload).PotentialAction, 1) + assert.Len(t, pl.(*MSTeamsPayload).PotentialAction[0].Targets, 1) + assert.Equal(t, "http://localhost:3000/test/repo/src/test", pl.(*MSTeamsPayload).PotentialAction[0].Targets[0].URI) + }) + + t.Run("Fork", func(t *testing.T) { + p := forkTestPayload() + + d := new(MSTeamsPayload) + pl, err := d.Fork(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &MSTeamsPayload{}, pl) + + assert.Equal(t, "test/repo2 is forked to test/repo", pl.(*MSTeamsPayload).Title) + assert.Equal(t, "test/repo2 is forked to test/repo", pl.(*MSTeamsPayload).Summary) + assert.Len(t, pl.(*MSTeamsPayload).Sections, 1) + assert.Equal(t, "user1", pl.(*MSTeamsPayload).Sections[0].ActivitySubtitle) + assert.Empty(t, pl.(*MSTeamsPayload).Sections[0].Text) + assert.Len(t, pl.(*MSTeamsPayload).Sections[0].Facts, 2) + for _, fact := range pl.(*MSTeamsPayload).Sections[0].Facts { + if fact.Name == "Repository:" { + assert.Equal(t, p.Repo.FullName, fact.Value) + } else if fact.Name == "Forkee:" { + assert.Equal(t, p.Forkee.FullName, fact.Value) + } else { + t.Fail() + } + } + assert.Len(t, pl.(*MSTeamsPayload).PotentialAction, 1) + assert.Len(t, pl.(*MSTeamsPayload).PotentialAction[0].Targets, 1) + assert.Equal(t, "http://localhost:3000/test/repo", pl.(*MSTeamsPayload).PotentialAction[0].Targets[0].URI) + }) + + t.Run("Push", func(t *testing.T) { + p := pushTestPayload() + + d := new(MSTeamsPayload) + pl, err := d.Push(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &MSTeamsPayload{}, pl) + + assert.Equal(t, "[test/repo:test] 2 new commits", pl.(*MSTeamsPayload).Title) + assert.Equal(t, "[test/repo:test] 2 new commits", pl.(*MSTeamsPayload).Summary) + assert.Len(t, pl.(*MSTeamsPayload).Sections, 1) + assert.Equal(t, "user1", pl.(*MSTeamsPayload).Sections[0].ActivitySubtitle) + assert.Equal(t, "[2020558](http://localhost:3000/test/repo/commit/2020558fe2e34debb818a514715839cabd25e778) commit message - user1\n\n[2020558](http://localhost:3000/test/repo/commit/2020558fe2e34debb818a514715839cabd25e778) commit message - user1", pl.(*MSTeamsPayload).Sections[0].Text) + assert.Len(t, pl.(*MSTeamsPayload).Sections[0].Facts, 2) + for _, fact := range pl.(*MSTeamsPayload).Sections[0].Facts { + if fact.Name == "Repository:" { + assert.Equal(t, p.Repo.FullName, fact.Value) + } else if fact.Name == "Commit count:" { + assert.Equal(t, "2", fact.Value) + } else { + t.Fail() + } + } + assert.Len(t, pl.(*MSTeamsPayload).PotentialAction, 1) + assert.Len(t, pl.(*MSTeamsPayload).PotentialAction[0].Targets, 1) + assert.Equal(t, "http://localhost:3000/test/repo/src/test", pl.(*MSTeamsPayload).PotentialAction[0].Targets[0].URI) + }) + + t.Run("Issue", func(t *testing.T) { + p := issueTestPayload() + + d := new(MSTeamsPayload) + p.Action = api.HookIssueOpened + pl, err := d.Issue(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &MSTeamsPayload{}, pl) + + assert.Equal(t, "[test/repo] Issue opened: #2 crash", pl.(*MSTeamsPayload).Title) + assert.Equal(t, "[test/repo] Issue opened: #2 crash", pl.(*MSTeamsPayload).Summary) + assert.Len(t, pl.(*MSTeamsPayload).Sections, 1) + assert.Equal(t, "user1", pl.(*MSTeamsPayload).Sections[0].ActivitySubtitle) + assert.Equal(t, "issue body", pl.(*MSTeamsPayload).Sections[0].Text) + assert.Len(t, pl.(*MSTeamsPayload).Sections[0].Facts, 2) + for _, fact := range pl.(*MSTeamsPayload).Sections[0].Facts { + if fact.Name == "Repository:" { + assert.Equal(t, p.Repository.FullName, fact.Value) + } else if fact.Name == "Issue #:" { + assert.Equal(t, "2", fact.Value) + } else { + t.Fail() + } + } + assert.Len(t, pl.(*MSTeamsPayload).PotentialAction, 1) + assert.Len(t, pl.(*MSTeamsPayload).PotentialAction[0].Targets, 1) + assert.Equal(t, "http://localhost:3000/test/repo/issues/2", pl.(*MSTeamsPayload).PotentialAction[0].Targets[0].URI) + + p.Action = api.HookIssueClosed + pl, err = d.Issue(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &MSTeamsPayload{}, pl) + + assert.Equal(t, "[test/repo] Issue closed: #2 crash", pl.(*MSTeamsPayload).Title) + assert.Equal(t, "[test/repo] Issue closed: #2 crash", pl.(*MSTeamsPayload).Summary) + assert.Len(t, pl.(*MSTeamsPayload).Sections, 1) + assert.Equal(t, "user1", pl.(*MSTeamsPayload).Sections[0].ActivitySubtitle) + assert.Empty(t, pl.(*MSTeamsPayload).Sections[0].Text) + assert.Len(t, pl.(*MSTeamsPayload).Sections[0].Facts, 2) + for _, fact := range pl.(*MSTeamsPayload).Sections[0].Facts { + if fact.Name == "Repository:" { + assert.Equal(t, p.Repository.FullName, fact.Value) + } else if fact.Name == "Issue #:" { + assert.Equal(t, "2", fact.Value) + } else { + t.Fail() + } + } + assert.Len(t, pl.(*MSTeamsPayload).PotentialAction, 1) + assert.Len(t, pl.(*MSTeamsPayload).PotentialAction[0].Targets, 1) + assert.Equal(t, "http://localhost:3000/test/repo/issues/2", pl.(*MSTeamsPayload).PotentialAction[0].Targets[0].URI) + }) + + t.Run("IssueComment", func(t *testing.T) { + p := issueCommentTestPayload() + + d := new(MSTeamsPayload) + pl, err := d.IssueComment(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &MSTeamsPayload{}, pl) + + assert.Equal(t, "[test/repo] New comment on issue #2 crash", pl.(*MSTeamsPayload).Title) + assert.Equal(t, "[test/repo] New comment on issue #2 crash", pl.(*MSTeamsPayload).Summary) + assert.Len(t, pl.(*MSTeamsPayload).Sections, 1) + assert.Equal(t, "user1", pl.(*MSTeamsPayload).Sections[0].ActivitySubtitle) + assert.Equal(t, "more info needed", pl.(*MSTeamsPayload).Sections[0].Text) + assert.Len(t, pl.(*MSTeamsPayload).Sections[0].Facts, 2) + for _, fact := range pl.(*MSTeamsPayload).Sections[0].Facts { + if fact.Name == "Repository:" { + assert.Equal(t, p.Repository.FullName, fact.Value) + } else if fact.Name == "Issue #:" { + assert.Equal(t, "2", fact.Value) + } else { + t.Fail() + } + } + assert.Len(t, pl.(*MSTeamsPayload).PotentialAction, 1) + assert.Len(t, pl.(*MSTeamsPayload).PotentialAction[0].Targets, 1) + assert.Equal(t, "http://localhost:3000/test/repo/issues/2#issuecomment-4", pl.(*MSTeamsPayload).PotentialAction[0].Targets[0].URI) + }) + + t.Run("PullRequest", func(t *testing.T) { + p := pullRequestTestPayload() + + d := new(MSTeamsPayload) + pl, err := d.PullRequest(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &MSTeamsPayload{}, pl) + + assert.Equal(t, "[test/repo] Pull request opened: #12 Fix bug", pl.(*MSTeamsPayload).Title) + assert.Equal(t, "[test/repo] Pull request opened: #12 Fix bug", pl.(*MSTeamsPayload).Summary) + assert.Len(t, pl.(*MSTeamsPayload).Sections, 1) + assert.Equal(t, "user1", pl.(*MSTeamsPayload).Sections[0].ActivitySubtitle) + assert.Equal(t, "fixes bug #2", pl.(*MSTeamsPayload).Sections[0].Text) + assert.Len(t, pl.(*MSTeamsPayload).Sections[0].Facts, 2) + for _, fact := range pl.(*MSTeamsPayload).Sections[0].Facts { + if fact.Name == "Repository:" { + assert.Equal(t, p.Repository.FullName, fact.Value) + } else if fact.Name == "Pull request #:" { + assert.Equal(t, "12", fact.Value) + } else { + t.Fail() + } + } + assert.Len(t, pl.(*MSTeamsPayload).PotentialAction, 1) + assert.Len(t, pl.(*MSTeamsPayload).PotentialAction[0].Targets, 1) + assert.Equal(t, "http://localhost:3000/test/repo/pulls/12", pl.(*MSTeamsPayload).PotentialAction[0].Targets[0].URI) + }) + + t.Run("PullRequestComment", func(t *testing.T) { + p := pullRequestCommentTestPayload() + + d := new(MSTeamsPayload) + pl, err := d.IssueComment(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &MSTeamsPayload{}, pl) + + assert.Equal(t, "[test/repo] New comment on pull request #12 Fix bug", pl.(*MSTeamsPayload).Title) + assert.Equal(t, "[test/repo] New comment on pull request #12 Fix bug", pl.(*MSTeamsPayload).Summary) + assert.Len(t, pl.(*MSTeamsPayload).Sections, 1) + assert.Equal(t, "user1", pl.(*MSTeamsPayload).Sections[0].ActivitySubtitle) + assert.Equal(t, "changes requested", pl.(*MSTeamsPayload).Sections[0].Text) + assert.Len(t, pl.(*MSTeamsPayload).Sections[0].Facts, 2) + for _, fact := range pl.(*MSTeamsPayload).Sections[0].Facts { + if fact.Name == "Repository:" { + assert.Equal(t, p.Repository.FullName, fact.Value) + } else if fact.Name == "Issue #:" { + assert.Equal(t, "12", fact.Value) + } else { + t.Fail() + } + } + assert.Len(t, pl.(*MSTeamsPayload).PotentialAction, 1) + assert.Len(t, pl.(*MSTeamsPayload).PotentialAction[0].Targets, 1) + assert.Equal(t, "http://localhost:3000/test/repo/pulls/12#issuecomment-4", pl.(*MSTeamsPayload).PotentialAction[0].Targets[0].URI) + }) + + t.Run("Review", func(t *testing.T) { + p := pullRequestTestPayload() + p.Action = api.HookIssueReviewed + + d := new(MSTeamsPayload) + pl, err := d.Review(p, models.HookEventPullRequestReviewApproved) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &MSTeamsPayload{}, pl) + + assert.Equal(t, "[test/repo] Pull request review approved: #12 Fix bug", pl.(*MSTeamsPayload).Title) + assert.Equal(t, "[test/repo] Pull request review approved: #12 Fix bug", pl.(*MSTeamsPayload).Summary) + assert.Len(t, pl.(*MSTeamsPayload).Sections, 1) + assert.Equal(t, "user1", pl.(*MSTeamsPayload).Sections[0].ActivitySubtitle) + assert.Equal(t, "good job", pl.(*MSTeamsPayload).Sections[0].Text) + assert.Len(t, pl.(*MSTeamsPayload).Sections[0].Facts, 2) + for _, fact := range pl.(*MSTeamsPayload).Sections[0].Facts { + if fact.Name == "Repository:" { + assert.Equal(t, p.Repository.FullName, fact.Value) + } else if fact.Name == "Pull request #:" { + assert.Equal(t, "12", fact.Value) + } else { + t.Fail() + } + } + assert.Len(t, pl.(*MSTeamsPayload).PotentialAction, 1) + assert.Len(t, pl.(*MSTeamsPayload).PotentialAction[0].Targets, 1) + assert.Equal(t, "http://localhost:3000/test/repo/pulls/12", pl.(*MSTeamsPayload).PotentialAction[0].Targets[0].URI) + }) + + t.Run("Repository", func(t *testing.T) { + p := repositoryTestPayload() + + d := new(MSTeamsPayload) + pl, err := d.Repository(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &MSTeamsPayload{}, pl) + + assert.Equal(t, "[test/repo] Repository created", pl.(*MSTeamsPayload).Title) + assert.Equal(t, "[test/repo] Repository created", pl.(*MSTeamsPayload).Summary) + assert.Len(t, pl.(*MSTeamsPayload).Sections, 1) + assert.Equal(t, "user1", pl.(*MSTeamsPayload).Sections[0].ActivitySubtitle) + assert.Empty(t, pl.(*MSTeamsPayload).Sections[0].Text) + assert.Len(t, pl.(*MSTeamsPayload).Sections[0].Facts, 1) + for _, fact := range pl.(*MSTeamsPayload).Sections[0].Facts { + if fact.Name == "Repository:" { + assert.Equal(t, p.Repository.FullName, fact.Value) + } else { + t.Fail() + } + } + assert.Len(t, pl.(*MSTeamsPayload).PotentialAction, 1) + assert.Len(t, pl.(*MSTeamsPayload).PotentialAction[0].Targets, 1) + assert.Equal(t, "http://localhost:3000/test/repo", pl.(*MSTeamsPayload).PotentialAction[0].Targets[0].URI) + }) + + t.Run("Release", func(t *testing.T) { + p := pullReleaseTestPayload() + + d := new(MSTeamsPayload) + pl, err := d.Release(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &MSTeamsPayload{}, pl) + + assert.Equal(t, "[test/repo] Release created: v1.0", pl.(*MSTeamsPayload).Title) + assert.Equal(t, "[test/repo] Release created: v1.0", pl.(*MSTeamsPayload).Summary) + assert.Len(t, pl.(*MSTeamsPayload).Sections, 1) + assert.Equal(t, "user1", pl.(*MSTeamsPayload).Sections[0].ActivitySubtitle) + assert.Empty(t, pl.(*MSTeamsPayload).Sections[0].Text) + assert.Len(t, pl.(*MSTeamsPayload).Sections[0].Facts, 2) + for _, fact := range pl.(*MSTeamsPayload).Sections[0].Facts { + if fact.Name == "Repository:" { + assert.Equal(t, p.Repository.FullName, fact.Value) + } else if fact.Name == "Tag:" { + assert.Equal(t, "v1.0", fact.Value) + } else { + t.Fail() + } + } + assert.Len(t, pl.(*MSTeamsPayload).PotentialAction, 1) + assert.Len(t, pl.(*MSTeamsPayload).PotentialAction[0].Targets, 1) + assert.Equal(t, "http://localhost:3000/api/v1/repos/test/repo/releases/2", pl.(*MSTeamsPayload).PotentialAction[0].Targets[0].URI) + }) +} + +func TestMSTeamsJSONPayload(t *testing.T) { + p := pushTestPayload() + + pl, err := new(MSTeamsPayload).Push(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &MSTeamsPayload{}, pl) + + json, err := pl.JSONPayload() + require.NoError(t, err) + assert.NotEmpty(t, json) +} diff --git a/services/webhook/slack.go b/services/webhook/slack.go index f5c857f2a9614..1de9c9c37c70c 100644 --- a/services/webhook/slack.go +++ b/services/webhook/slack.go @@ -111,12 +111,7 @@ func (s *SlackPayload) Create(p *api.CreatePayload) (api.Payloader, error) { refLink := SlackLinkToRef(p.Repo.HTMLURL, p.Ref) text := fmt.Sprintf("[%s:%s] %s created by %s", repoLink, refLink, p.RefType, p.Sender.UserName) - return &SlackPayload{ - Channel: s.Channel, - Text: text, - Username: s.Username, - IconURL: s.IconURL, - }, nil + return s.createPayload(text, nil), nil } // Delete composes Slack payload for delete a branch or tag. @@ -124,12 +119,8 @@ func (s *SlackPayload) Delete(p *api.DeletePayload) (api.Payloader, error) { refName := git.RefEndName(p.Ref) repoLink := SlackLinkFormatter(p.Repo.HTMLURL, p.Repo.FullName) text := fmt.Sprintf("[%s:%s] %s deleted by %s", repoLink, refName, p.RefType, p.Sender.UserName) - return &SlackPayload{ - Channel: s.Channel, - Text: text, - Username: s.Username, - IconURL: s.IconURL, - }, nil + + return s.createPayload(text, nil), nil } // Fork composes Slack payload for forked by a repository. @@ -137,66 +128,46 @@ func (s *SlackPayload) Fork(p *api.ForkPayload) (api.Payloader, error) { baseLink := SlackLinkFormatter(p.Forkee.HTMLURL, p.Forkee.FullName) forkLink := SlackLinkFormatter(p.Repo.HTMLURL, p.Repo.FullName) text := fmt.Sprintf("%s is forked to %s", baseLink, forkLink) - return &SlackPayload{ - Channel: s.Channel, - Text: text, - Username: s.Username, - IconURL: s.IconURL, - }, nil + + return s.createPayload(text, nil), nil } // Issue implements PayloadConvertor Issue method func (s *SlackPayload) Issue(p *api.IssuePayload) (api.Payloader, error) { text, issueTitle, attachmentText, color := getIssuesPayloadInfo(p, SlackLinkFormatter, true) - pl := &SlackPayload{ - Channel: s.Channel, - Text: text, - Username: s.Username, - IconURL: s.IconURL, - } + var attachments []SlackAttachment if attachmentText != "" { attachmentText = SlackTextFormatter(attachmentText) issueTitle = SlackTextFormatter(issueTitle) - pl.Attachments = []SlackAttachment{{ + attachments = append(attachments, SlackAttachment{ Color: fmt.Sprintf("%x", color), Title: issueTitle, TitleLink: p.Issue.HTMLURL, Text: attachmentText, - }} + }) } - return pl, nil + return s.createPayload(text, attachments), nil } // IssueComment implements PayloadConvertor IssueComment method func (s *SlackPayload) IssueComment(p *api.IssueCommentPayload) (api.Payloader, error) { text, issueTitle, color := getIssueCommentPayloadInfo(p, SlackLinkFormatter, true) - return &SlackPayload{ - Channel: s.Channel, - Text: text, - Username: s.Username, - IconURL: s.IconURL, - Attachments: []SlackAttachment{{ - Color: fmt.Sprintf("%x", color), - Title: issueTitle, - TitleLink: p.Comment.HTMLURL, - Text: SlackTextFormatter(p.Comment.Body), - }}, - }, nil + return s.createPayload(text, []SlackAttachment{{ + Color: fmt.Sprintf("%x", color), + Title: issueTitle, + TitleLink: p.Comment.HTMLURL, + Text: SlackTextFormatter(p.Comment.Body), + }}), nil } // Release implements PayloadConvertor Release method func (s *SlackPayload) Release(p *api.ReleasePayload) (api.Payloader, error) { text, _ := getReleasePayloadInfo(p, SlackLinkFormatter, true) - return &SlackPayload{ - Channel: s.Channel, - Text: text, - Username: s.Username, - IconURL: s.IconURL, - }, nil + return s.createPayload(text, nil), nil } // Push implements PayloadConvertor Push method @@ -232,42 +203,31 @@ func (s *SlackPayload) Push(p *api.PushPayload) (api.Payloader, error) { } } - return &SlackPayload{ - Channel: s.Channel, - Text: text, - Username: s.Username, - IconURL: s.IconURL, - Attachments: []SlackAttachment{{ - Color: s.Color, - Title: p.Repo.HTMLURL, - TitleLink: p.Repo.HTMLURL, - Text: attachmentText, - }}, - }, nil + return s.createPayload(text, []SlackAttachment{{ + Color: s.Color, + Title: p.Repo.HTMLURL, + TitleLink: p.Repo.HTMLURL, + Text: attachmentText, + }}), nil } // PullRequest implements PayloadConvertor PullRequest method func (s *SlackPayload) PullRequest(p *api.PullRequestPayload) (api.Payloader, error) { text, issueTitle, attachmentText, color := getPullRequestPayloadInfo(p, SlackLinkFormatter, true) - pl := &SlackPayload{ - Channel: s.Channel, - Text: text, - Username: s.Username, - IconURL: s.IconURL, - } + var attachments []SlackAttachment if attachmentText != "" { attachmentText = SlackTextFormatter(p.PullRequest.Body) issueTitle = SlackTextFormatter(issueTitle) - pl.Attachments = []SlackAttachment{{ + attachments = append(attachments, SlackAttachment{ Color: fmt.Sprintf("%x", color), Title: issueTitle, TitleLink: p.PullRequest.URL, Text: attachmentText, - }} + }) } - return pl, nil + return s.createPayload(text, attachments), nil } // Review implements PayloadConvertor Review method @@ -288,12 +248,7 @@ func (s *SlackPayload) Review(p *api.PullRequestPayload, event models.HookEventT text = fmt.Sprintf("[%s] Pull request review %s: [%s](%s) by %s", repoLink, action, title, titleLink, senderLink) } - return &SlackPayload{ - Channel: s.Channel, - Text: text, - Username: s.Username, - IconURL: s.IconURL, - }, nil + return s.createPayload(text, nil), nil } // Repository implements PayloadConvertor Repository method @@ -309,12 +264,17 @@ func (s *SlackPayload) Repository(p *api.RepositoryPayload) (api.Payloader, erro text = fmt.Sprintf("[%s] Repository deleted by %s", repoLink, senderLink) } + return s.createPayload(text, nil), nil +} + +func (s *SlackPayload) createPayload(text string, attachments []SlackAttachment) *SlackPayload { return &SlackPayload{ - Channel: s.Channel, - Text: text, - Username: s.Username, - IconURL: s.IconURL, - }, nil + Channel: s.Channel, + Text: text, + Username: s.Username, + IconURL: s.IconURL, + Attachments: attachments, + } } // GetSlackPayload converts a slack webhook into a SlackPayload diff --git a/services/webhook/slack_test.go b/services/webhook/slack_test.go index 20de80bd656d8..3f279810c98af 100644 --- a/services/webhook/slack_test.go +++ b/services/webhook/slack_test.go @@ -7,74 +7,166 @@ package webhook import ( "testing" + "code.gitea.io/gitea/models" api "code.gitea.io/gitea/modules/structs" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestSlackIssuesPayloadOpened(t *testing.T) { - p := issueTestPayload() - p.Action = api.HookIssueOpened +func TestSlackPayload(t *testing.T) { + t.Run("Create", func(t *testing.T) { + p := createTestPayload() - s := new(SlackPayload) - s.Username = p.Sender.UserName + d := new(SlackPayload) + pl, err := d.Create(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &SlackPayload{}, pl) - pl, err := s.Issue(p) - require.NoError(t, err) - require.NotNil(t, pl) - assert.Equal(t, "[] Issue opened: by ", pl.(*SlackPayload).Text) + assert.Equal(t, "[:] branch created by user1", pl.(*SlackPayload).Text) + }) - p.Action = api.HookIssueClosed - pl, err = s.Issue(p) - require.NoError(t, err) - require.NotNil(t, pl) - assert.Equal(t, "[] Issue closed: by ", pl.(*SlackPayload).Text) -} + t.Run("Delete", func(t *testing.T) { + p := deleteTestPayload() -func TestSlackIssueCommentPayload(t *testing.T) { - p := issueCommentTestPayload() - s := new(SlackPayload) - s.Username = p.Sender.UserName + d := new(SlackPayload) + pl, err := d.Delete(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &SlackPayload{}, pl) - pl, err := s.IssueComment(p) - require.NoError(t, err) - require.NotNil(t, pl) + assert.Equal(t, "[:test] branch deleted by user1", pl.(*SlackPayload).Text) + }) - assert.Equal(t, "[] New comment on issue by ", pl.(*SlackPayload).Text) -} + t.Run("Fork", func(t *testing.T) { + p := forkTestPayload() -func TestSlackPullRequestCommentPayload(t *testing.T) { - p := pullRequestCommentTestPayload() - s := new(SlackPayload) - s.Username = p.Sender.UserName + d := new(SlackPayload) + pl, err := d.Fork(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &SlackPayload{}, pl) - pl, err := s.IssueComment(p) - require.NoError(t, err) - require.NotNil(t, pl) + assert.Equal(t, " is forked to ", pl.(*SlackPayload).Text) + }) - assert.Equal(t, "[] New comment on pull request by ", pl.(*SlackPayload).Text) -} + t.Run("Push", func(t *testing.T) { + p := pushTestPayload() -func TestSlackReleasePayload(t *testing.T) { - p := pullReleaseTestPayload() - s := new(SlackPayload) - s.Username = p.Sender.UserName + d := new(SlackPayload) + pl, err := d.Push(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &SlackPayload{}, pl) - pl, err := s.Release(p) - require.NoError(t, err) - require.NotNil(t, pl) + assert.Equal(t, "[:] 2 new commits pushed by user1", pl.(*SlackPayload).Text) + }) + + t.Run("Issue", func(t *testing.T) { + p := issueTestPayload() + + d := new(SlackPayload) + p.Action = api.HookIssueOpened + pl, err := d.Issue(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &SlackPayload{}, pl) + + assert.Equal(t, "[] Issue opened: by ", pl.(*SlackPayload).Text) - assert.Equal(t, "[] Release created: by ", pl.(*SlackPayload).Text) + p.Action = api.HookIssueClosed + pl, err = d.Issue(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &SlackPayload{}, pl) + + assert.Equal(t, "[] Issue closed: by ", pl.(*SlackPayload).Text) + }) + + t.Run("IssueComment", func(t *testing.T) { + p := issueCommentTestPayload() + + d := new(SlackPayload) + pl, err := d.IssueComment(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &SlackPayload{}, pl) + + assert.Equal(t, "[] New comment on issue by ", pl.(*SlackPayload).Text) + }) + + t.Run("PullRequest", func(t *testing.T) { + p := pullRequestTestPayload() + + d := new(SlackPayload) + pl, err := d.PullRequest(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &SlackPayload{}, pl) + + assert.Equal(t, "[] Pull request opened: by ", pl.(*SlackPayload).Text) + }) + + t.Run("PullRequestComment", func(t *testing.T) { + p := pullRequestCommentTestPayload() + + d := new(SlackPayload) + pl, err := d.IssueComment(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &SlackPayload{}, pl) + + assert.Equal(t, "[] New comment on pull request by ", pl.(*SlackPayload).Text) + }) + + t.Run("Review", func(t *testing.T) { + p := pullRequestTestPayload() + p.Action = api.HookIssueReviewed + + d := new(SlackPayload) + pl, err := d.Review(p, models.HookEventPullRequestReviewApproved) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &SlackPayload{}, pl) + + assert.Equal(t, "[] Pull request review approved: [#12 Fix bug](http://localhost:3000/test/repo/pulls/12) by ", pl.(*SlackPayload).Text) + }) + + t.Run("Repository", func(t *testing.T) { + p := repositoryTestPayload() + + d := new(SlackPayload) + pl, err := d.Repository(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &SlackPayload{}, pl) + + assert.Equal(t, "[] Repository created by ", pl.(*SlackPayload).Text) + }) + + t.Run("Release", func(t *testing.T) { + p := pullReleaseTestPayload() + + d := new(SlackPayload) + pl, err := d.Release(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &SlackPayload{}, pl) + + assert.Equal(t, "[] Release created: by ", pl.(*SlackPayload).Text) + }) } -func TestSlackPullRequestPayload(t *testing.T) { - p := pullRequestTestPayload() - s := new(SlackPayload) - s.Username = p.Sender.UserName +func TestSlackJSONPayload(t *testing.T) { + p := pushTestPayload() - pl, err := s.PullRequest(p) + pl, err := new(SlackPayload).Push(p) require.NoError(t, err) require.NotNil(t, pl) + require.IsType(t, &SlackPayload{}, pl) - assert.Equal(t, "[] Pull request opened: by ", pl.(*SlackPayload).Text) + json, err := pl.JSONPayload() + require.NoError(t, err) + assert.NotEmpty(t, json) } diff --git a/services/webhook/telegram.go b/services/webhook/telegram.go index 5b78b46f8ec02..f71352141dee2 100644 --- a/services/webhook/telegram.go +++ b/services/webhook/telegram.go @@ -68,9 +68,7 @@ func (t *TelegramPayload) Create(p *api.CreatePayload) (api.Payloader, error) { title := fmt.Sprintf(`[%s] %s %s created`, p.Repo.HTMLURL, p.Repo.FullName, p.RefType, p.Repo.HTMLURL+"/src/"+refName, refName) - return &TelegramPayload{ - Message: title, - }, nil + return createTelegramPayload(title), nil } // Delete implements PayloadConvertor Delete method @@ -80,18 +78,14 @@ func (t *TelegramPayload) Delete(p *api.DeletePayload) (api.Payloader, error) { title := fmt.Sprintf(`[%s] %s %s deleted`, p.Repo.HTMLURL, p.Repo.FullName, p.RefType, p.Repo.HTMLURL+"/src/"+refName, refName) - return &TelegramPayload{ - Message: title, - }, nil + return createTelegramPayload(title), nil } // Fork implements PayloadConvertor Fork method func (t *TelegramPayload) Fork(p *api.ForkPayload) (api.Payloader, error) { title := fmt.Sprintf(`%s is forked to %s`, p.Forkee.FullName, p.Repo.HTMLURL, p.Repo.FullName) - return &TelegramPayload{ - Message: title, - }, nil + return createTelegramPayload(title), nil } // Push implements PayloadConvertor Push method @@ -129,36 +123,28 @@ func (t *TelegramPayload) Push(p *api.PushPayload) (api.Payloader, error) { } } - return &TelegramPayload{ - Message: title + "\n" + text, - }, nil + return createTelegramPayload(title + "\n" + text), nil } // Issue implements PayloadConvertor Issue method func (t *TelegramPayload) Issue(p *api.IssuePayload) (api.Payloader, error) { text, _, attachmentText, _ := getIssuesPayloadInfo(p, htmlLinkFormatter, true) - return &TelegramPayload{ - Message: text + "\n\n" + attachmentText, - }, nil + return createTelegramPayload(text + "\n\n" + attachmentText), nil } // IssueComment implements PayloadConvertor IssueComment method func (t *TelegramPayload) IssueComment(p *api.IssueCommentPayload) (api.Payloader, error) { text, _, _ := getIssueCommentPayloadInfo(p, htmlLinkFormatter, true) - return &TelegramPayload{ - Message: text + "\n" + p.Comment.Body, - }, nil + return createTelegramPayload(text + "\n" + p.Comment.Body), nil } // PullRequest implements PayloadConvertor PullRequest method func (t *TelegramPayload) PullRequest(p *api.PullRequestPayload) (api.Payloader, error) { text, _, attachmentText, _ := getPullRequestPayloadInfo(p, htmlLinkFormatter, true) - return &TelegramPayload{ - Message: text + "\n" + attachmentText, - }, nil + return createTelegramPayload(text + "\n" + attachmentText), nil } // Review implements PayloadConvertor Review method @@ -173,12 +159,9 @@ func (t *TelegramPayload) Review(p *api.PullRequestPayload, event models.HookEve text = fmt.Sprintf("[%s] Pull request review %s: #%d %s", p.Repository.FullName, action, p.Index, p.PullRequest.Title) attachmentText = p.Review.Content - } - return &TelegramPayload{ - Message: text + "\n" + attachmentText, - }, nil + return createTelegramPayload(text + "\n" + attachmentText), nil } // Repository implements PayloadConvertor Repository method @@ -187,14 +170,10 @@ func (t *TelegramPayload) Repository(p *api.RepositoryPayload) (api.Payloader, e switch p.Action { case api.HookRepoCreated: title = fmt.Sprintf(`[%s] Repository created`, p.Repository.HTMLURL, p.Repository.FullName) - return &TelegramPayload{ - Message: title, - }, nil + return createTelegramPayload(title), nil case api.HookRepoDeleted: title = fmt.Sprintf("[%s] Repository deleted", p.Repository.FullName) - return &TelegramPayload{ - Message: title, - }, nil + return createTelegramPayload(title), nil } return nil, nil } @@ -203,12 +182,16 @@ func (t *TelegramPayload) Repository(p *api.RepositoryPayload) (api.Payloader, e func (t *TelegramPayload) Release(p *api.ReleasePayload) (api.Payloader, error) { text, _ := getReleasePayloadInfo(p, htmlLinkFormatter, true) - return &TelegramPayload{ - Message: text + "\n", - }, nil + return createTelegramPayload(text), nil } // GetTelegramPayload converts a telegram webhook into a TelegramPayload func GetTelegramPayload(p api.Payloader, event models.HookEventType, meta string) (api.Payloader, error) { return convertPayloader(new(TelegramPayload), p, event) } + +func createTelegramPayload(message string) *TelegramPayload { + return &TelegramPayload{ + Message: strings.TrimSpace(message), + } +} diff --git a/services/webhook/telegram_test.go b/services/webhook/telegram_test.go index 0e909343a86c6..037a2481d6df2 100644 --- a/services/webhook/telegram_test.go +++ b/services/webhook/telegram_test.go @@ -7,18 +7,166 @@ package webhook import ( "testing" + "code.gitea.io/gitea/models" api "code.gitea.io/gitea/modules/structs" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestGetTelegramIssuesPayload(t *testing.T) { - p := issueTestPayload() - p.Action = api.HookIssueClosed +func TestTelegramPayload(t *testing.T) { + t.Run("Create", func(t *testing.T) { + p := createTestPayload() + + d := new(TelegramPayload) + pl, err := d.Create(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &TelegramPayload{}, pl) + + assert.Equal(t, `[test/repo] branch test created`, pl.(*TelegramPayload).Message) + }) + + t.Run("Delete", func(t *testing.T) { + p := deleteTestPayload() + + d := new(TelegramPayload) + pl, err := d.Delete(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &TelegramPayload{}, pl) + + assert.Equal(t, `[test/repo] branch test deleted`, pl.(*TelegramPayload).Message) + }) + + t.Run("Fork", func(t *testing.T) { + p := forkTestPayload() + + d := new(TelegramPayload) + pl, err := d.Fork(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &TelegramPayload{}, pl) + + assert.Equal(t, `test/repo2 is forked to test/repo`, pl.(*TelegramPayload).Message) + }) + + t.Run("Push", func(t *testing.T) { + p := pushTestPayload() + + d := new(TelegramPayload) + pl, err := d.Push(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &TelegramPayload{}, pl) + + assert.Equal(t, "[test/repo:test] 2 new commits\n[2020558] commit message - user1\n[2020558] commit message - user1", pl.(*TelegramPayload).Message) + }) + + t.Run("Issue", func(t *testing.T) { + p := issueTestPayload() + + d := new(TelegramPayload) + p.Action = api.HookIssueOpened + pl, err := d.Issue(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &TelegramPayload{}, pl) + + assert.Equal(t, "[test/repo] Issue opened: #2 crash by user1\n\nissue body", pl.(*TelegramPayload).Message) + + p.Action = api.HookIssueClosed + pl, err = d.Issue(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &TelegramPayload{}, pl) + + assert.Equal(t, `[test/repo] Issue closed: #2 crash by user1`, pl.(*TelegramPayload).Message) + }) + + t.Run("IssueComment", func(t *testing.T) { + p := issueCommentTestPayload() + + d := new(TelegramPayload) + pl, err := d.IssueComment(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &TelegramPayload{}, pl) - pl, err := new(TelegramPayload).Issue(p) + assert.Equal(t, "[test/repo] New comment on issue #2 crash by user1\nmore info needed", pl.(*TelegramPayload).Message) + }) + + t.Run("PullRequest", func(t *testing.T) { + p := pullRequestTestPayload() + + d := new(TelegramPayload) + pl, err := d.PullRequest(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &TelegramPayload{}, pl) + + assert.Equal(t, "[test/repo] Pull request opened: #12 Fix bug by user1\nfixes bug #2", pl.(*TelegramPayload).Message) + }) + + t.Run("PullRequestComment", func(t *testing.T) { + p := pullRequestCommentTestPayload() + + d := new(TelegramPayload) + pl, err := d.IssueComment(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &TelegramPayload{}, pl) + + assert.Equal(t, "[test/repo] New comment on pull request #12 Fix bug by user1\nchanges requested", pl.(*TelegramPayload).Message) + }) + + t.Run("Review", func(t *testing.T) { + p := pullRequestTestPayload() + p.Action = api.HookIssueReviewed + + d := new(TelegramPayload) + pl, err := d.Review(p, models.HookEventPullRequestReviewApproved) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &TelegramPayload{}, pl) + + assert.Equal(t, "[test/repo] Pull request review approved: #12 Fix bug\ngood job", pl.(*TelegramPayload).Message) + }) + + t.Run("Repository", func(t *testing.T) { + p := repositoryTestPayload() + + d := new(TelegramPayload) + pl, err := d.Repository(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &TelegramPayload{}, pl) + + assert.Equal(t, `[test/repo] Repository created`, pl.(*TelegramPayload).Message) + }) + + t.Run("Release", func(t *testing.T) { + p := pullReleaseTestPayload() + + d := new(TelegramPayload) + pl, err := d.Release(p) + require.NoError(t, err) + require.NotNil(t, pl) + require.IsType(t, &TelegramPayload{}, pl) + + assert.Equal(t, `[test/repo] Release created: v1.0 by user1`, pl.(*TelegramPayload).Message) + }) +} + +func TestTelegramJSONPayload(t *testing.T) { + p := pushTestPayload() + + pl, err := new(TelegramPayload).Push(p) require.NoError(t, err) require.NotNil(t, pl) + require.IsType(t, &TelegramPayload{}, pl) - assert.Equal(t, "[test/repo] Issue closed: #2 crash by user1\n\n", pl.(*TelegramPayload).Message) + json, err := pl.JSONPayload() + require.NoError(t, err) + assert.NotEmpty(t, json) } From 681e81babdbbe777b3ea2af353b872aee2dd9b32 Mon Sep 17 00:00:00 2001 From: zeripath Date: Mon, 21 Jun 2021 14:01:44 +0100 Subject: [PATCH 06/10] reqOrgMembership calls need to be preceded by reqToken (#16198) ReqOrgMembership calls need to be preceded by reqToken Fix #16192 Signed-off-by: Andrew Thornton Co-authored-by: 6543 <6543@obermui.de> --- integrations/api_team_test.go | 4 ++++ routers/api/v1/api.go | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/integrations/api_team_test.go b/integrations/api_team_test.go index 8b202862a159c..0b77dc3be700b 100644 --- a/integrations/api_team_test.go +++ b/integrations/api_team_test.go @@ -144,7 +144,9 @@ func TestAPITeamSearch(t *testing.T) { var results TeamSearchResults session := loginUser(t, user.Name) + csrf := GetCSRF(t, session, "/"+org.Name) req := NewRequestf(t, "GET", "/api/v1/orgs/%s/teams/search?q=%s", org.Name, "_team") + req.Header.Add("X-Csrf-Token", csrf) resp := session.MakeRequest(t, req, http.StatusOK) DecodeJSON(t, resp, &results) assert.NotEmpty(t, results.Data) @@ -154,7 +156,9 @@ func TestAPITeamSearch(t *testing.T) { // no access if not organization member user5 := models.AssertExistsAndLoadBean(t, &models.User{ID: 5}).(*models.User) session = loginUser(t, user5.Name) + csrf = GetCSRF(t, session, "/"+org.Name) req = NewRequestf(t, "GET", "/api/v1/orgs/%s/teams/search?q=%s", org.Name, "team") + req.Header.Add("X-Csrf-Token", csrf) resp = session.MakeRequest(t, req, http.StatusForbidden) } diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 34cf80e0721c3..9efc2af2441fa 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -989,10 +989,10 @@ func Routes() *web.Route { Delete(reqToken(), reqOrgMembership(), org.ConcealMember) }) m.Group("/teams", func() { - m.Combo("", reqToken()).Get(org.ListTeams). - Post(reqOrgOwnership(), bind(api.CreateTeamOption{}), org.CreateTeam) + m.Get("", org.ListTeams) + m.Post("", reqOrgOwnership(), bind(api.CreateTeamOption{}), org.CreateTeam) m.Get("/search", org.SearchTeam) - }, reqOrgMembership()) + }, reqToken(), reqOrgMembership()) m.Group("/labels", func() { m.Get("", org.ListLabels) m.Post("", reqToken(), reqOrgOwnership(), bind(api.CreateLabelOption{}), org.CreateLabel) From 6a083a7234190d078ac51878bb8f39aa53ef1974 Mon Sep 17 00:00:00 2001 From: zeripath Date: Mon, 21 Jun 2021 19:34:37 +0100 Subject: [PATCH 07/10] Update documentation for Implicit TLS (#16220) As per RFC 8314, it is now recommended to prefer TLS over STARTTLS. Fix #16160 Signed-off-by: Andrew Thornton --- custom/conf/app.example.ini | 4 ++-- docs/content/doc/advanced/config-cheat-sheet.en-us.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 54320a58befb4..38759b8a4b8f7 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1387,8 +1387,8 @@ PATH = ;; Mail server ;; Gmail: smtp.gmail.com:587 ;; QQ: smtp.qq.com:465 -;; Using STARTTLS on port 587 is recommended per RFC 6409. -;; Note, if the port ends with "465", SMTPS will be used. +;; As per RFC 8314 using Implicit TLS/SMTPS on port 465 (if supported) is recommended, +;; otherwise STARTTLS on port 587 should be used. ;HOST = ;; ;; Disable HELO operation when hostnames are different. diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 4f84e2ac33269..35deeac02e22e 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -550,9 +550,9 @@ Define allowed algorithms and their minimum key length (use -1 to disable a type - `DISABLE_HELO`: **\**: Disable HELO operation. - `HELO_HOSTNAME`: **\**: Custom hostname for HELO operation. - `HOST`: **\**: SMTP mail host address and port (example: smtp.gitea.io:587). - - Using opportunistic TLS via STARTTLS on port 587 is recommended per RFC 6409. + - As per RFC 8314, if supported, Implicit TLS/SMTPS on port 465 is recommended, otherwise opportunistic TLS via STARTTLS on port 587 should be used. - `IS_TLS_ENABLED` : **false** : Forcibly use TLS to connect even if not on a default SMTPS port. - - Note, if the port ends with `465` SMTPS/SMTP over TLS will be used despite this setting. + - Note, if the port ends with `465` Implicit TLS/SMTPS/SMTP over TLS will be used despite this setting. - Otherwise if `IS_TLS_ENABLED=false` and the server supports `STARTTLS` this will be used. Thus if `STARTTLS` is preferred you should set `IS_TLS_ENABLED=false`. - `FROM`: **\**: Mail from address, RFC 5322. This can be just an email address, or the "Name" \ format. From 36c158bc9375d7ebb9aa749ccd6718d0d68e96d2 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 21 Jun 2021 20:34:58 +0200 Subject: [PATCH 08/10] Update milestone counters on new issue. (#16183) Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: zeripath --- models/consistency.go | 6 +++ models/issue.go | 8 ++-- models/issue_milestone.go | 75 ++++++++++++---------------------- models/issue_milestone_test.go | 6 +-- 4 files changed, 39 insertions(+), 56 deletions(-) diff --git a/models/consistency.go b/models/consistency.go index 9cfd02195effd..f037b0515704e 100644 --- a/models/consistency.go +++ b/models/consistency.go @@ -141,6 +141,12 @@ func (milestone *Milestone) checkForConsistency(t *testing.T) { actual := getCount(t, x.Where("is_closed=?", true), &Issue{MilestoneID: milestone.ID}) assert.EqualValues(t, milestone.NumClosedIssues, actual, "Unexpected number of closed issues for milestone %+v", milestone) + + completeness := 0 + if milestone.NumIssues > 0 { + completeness = milestone.NumClosedIssues * 100 / milestone.NumIssues + } + assert.Equal(t, completeness, milestone.Completeness) } func (label *Label) checkForConsistency(t *testing.T) { diff --git a/models/issue.go b/models/issue.go index ffbc110a6becc..b9643ae00ec42 100644 --- a/models/issue.go +++ b/models/issue.go @@ -647,8 +647,10 @@ func (issue *Issue) doChangeStatus(e *xorm.Session, doer *User, isMergePull bool } // Update issue count of milestone - if err := updateMilestoneClosedNum(e, issue.MilestoneID); err != nil { - return nil, err + if issue.MilestoneID > 0 { + if err := updateMilestoneCounters(e, issue.MilestoneID); err != nil { + return nil, err + } } if err := issue.updateClosedNum(e); err != nil { @@ -907,7 +909,7 @@ func newIssue(e *xorm.Session, doer *User, opts NewIssueOptions) (err error) { } if opts.Issue.MilestoneID > 0 { - if _, err = e.Exec("UPDATE `milestone` SET num_issues=num_issues+1 WHERE id=?", opts.Issue.MilestoneID); err != nil { + if err := updateMilestoneCounters(e, opts.Issue.MilestoneID); err != nil { return err } diff --git a/models/issue_milestone.go b/models/issue_milestone.go index 5aa83ea691d54..5e934cde0a02f 100644 --- a/models/issue_milestone.go +++ b/models/issue_milestone.go @@ -129,8 +129,12 @@ func GetMilestoneByRepoIDANDName(repoID int64, name string) (*Milestone, error) // GetMilestoneByID returns the milestone via id . func GetMilestoneByID(id int64) (*Milestone, error) { + return getMilestoneByID(x, id) +} + +func getMilestoneByID(e Engine, id int64) (*Milestone, error) { var m Milestone - has, err := x.ID(id).Get(&m) + has, err := e.ID(id).Get(&m) if err != nil { return nil, err } else if !has { @@ -155,10 +159,6 @@ func UpdateMilestone(m *Milestone, oldIsClosed bool) error { return err } - if err := updateMilestoneCompleteness(sess, m.ID); err != nil { - return err - } - // if IsClosed changed, update milestone numbers of repository if oldIsClosed != m.IsClosed { if err := updateRepoMilestoneNum(sess, m.RepoID); err != nil { @@ -171,23 +171,31 @@ func UpdateMilestone(m *Milestone, oldIsClosed bool) error { func updateMilestone(e Engine, m *Milestone) error { m.Name = strings.TrimSpace(m.Name) - _, err := e.ID(m.ID).AllCols(). + _, err := e.ID(m.ID).AllCols().Update(m) + if err != nil { + return err + } + return updateMilestoneCounters(e, m.ID) +} + +// updateMilestoneCounters calculates NumIssues, NumClosesIssues and Completeness +func updateMilestoneCounters(e Engine, id int64) error { + _, err := e.ID(id). SetExpr("num_issues", builder.Select("count(*)").From("issue").Where( - builder.Eq{"milestone_id": m.ID}, + builder.Eq{"milestone_id": id}, )). SetExpr("num_closed_issues", builder.Select("count(*)").From("issue").Where( builder.Eq{ - "milestone_id": m.ID, + "milestone_id": id, "is_closed": true, }, )). - Update(m) - return err -} - -func updateMilestoneCompleteness(e Engine, milestoneID int64) error { - _, err := e.Exec("UPDATE `milestone` SET completeness=100*num_closed_issues/(CASE WHEN num_issues > 0 THEN num_issues ELSE 1 END) WHERE id=?", - milestoneID, + Update(&Milestone{}) + if err != nil { + return err + } + _, err = e.Exec("UPDATE `milestone` SET completeness=100*num_closed_issues/(CASE WHEN num_issues > 0 THEN num_issues ELSE 1 END) WHERE id=?", + id, ) return err } @@ -256,25 +264,15 @@ func changeMilestoneAssign(e *xorm.Session, doer *User, issue *Issue, oldMilesto } if oldMilestoneID > 0 { - if err := updateMilestoneTotalNum(e, oldMilestoneID); err != nil { + if err := updateMilestoneCounters(e, oldMilestoneID); err != nil { return err } - if issue.IsClosed { - if err := updateMilestoneClosedNum(e, oldMilestoneID); err != nil { - return err - } - } } if issue.MilestoneID > 0 { - if err := updateMilestoneTotalNum(e, issue.MilestoneID); err != nil { + if err := updateMilestoneCounters(e, issue.MilestoneID); err != nil { return err } - if issue.IsClosed { - if err := updateMilestoneClosedNum(e, issue.MilestoneID); err != nil { - return err - } - } } if oldMilestoneID > 0 || issue.MilestoneID > 0 { @@ -622,29 +620,6 @@ func updateRepoMilestoneNum(e Engine, repoID int64) error { return err } -func updateMilestoneTotalNum(e Engine, milestoneID int64) (err error) { - if _, err = e.Exec("UPDATE `milestone` SET num_issues=(SELECT count(*) FROM issue WHERE milestone_id=?) WHERE id=?", - milestoneID, - milestoneID, - ); err != nil { - return - } - - return updateMilestoneCompleteness(e, milestoneID) -} - -func updateMilestoneClosedNum(e Engine, milestoneID int64) (err error) { - if _, err = e.Exec("UPDATE `milestone` SET num_closed_issues=(SELECT count(*) FROM issue WHERE milestone_id=? AND is_closed=?) WHERE id=?", - milestoneID, - true, - milestoneID, - ); err != nil { - return - } - - return updateMilestoneCompleteness(e, milestoneID) -} - // _____ _ _ _____ _ // |_ _| __ __ _ ___| | _____ __| |_ _(_)_ __ ___ ___ ___ // | || '__/ _` |/ __| |/ / _ \/ _` | | | | | '_ ` _ \ / _ \/ __| diff --git a/models/issue_milestone_test.go b/models/issue_milestone_test.go index af264aa274576..5406129884fb0 100644 --- a/models/issue_milestone_test.go +++ b/models/issue_milestone_test.go @@ -215,7 +215,7 @@ func TestChangeMilestoneStatus(t *testing.T) { CheckConsistencyFor(t, &Repository{ID: milestone.RepoID}, &Milestone{}) } -func TestUpdateMilestoneClosedNum(t *testing.T) { +func TestUpdateMilestoneCounters(t *testing.T) { assert.NoError(t, PrepareTestDatabase()) issue := AssertExistsAndLoadBean(t, &Issue{MilestoneID: 1}, "is_closed=0").(*Issue) @@ -224,14 +224,14 @@ func TestUpdateMilestoneClosedNum(t *testing.T) { issue.ClosedUnix = timeutil.TimeStampNow() _, err := x.ID(issue.ID).Cols("is_closed", "closed_unix").Update(issue) assert.NoError(t, err) - assert.NoError(t, updateMilestoneClosedNum(x, issue.MilestoneID)) + assert.NoError(t, updateMilestoneCounters(x, issue.MilestoneID)) CheckConsistencyFor(t, &Milestone{}) issue.IsClosed = false issue.ClosedUnix = 0 _, err = x.ID(issue.ID).Cols("is_closed", "closed_unix").Update(issue) assert.NoError(t, err) - assert.NoError(t, updateMilestoneClosedNum(x, issue.MilestoneID)) + assert.NoError(t, updateMilestoneCounters(x, issue.MilestoneID)) CheckConsistencyFor(t, &Milestone{}) } From d55b5eb0d3f1d129eb4d03cc7ddb56a59b4db8ff Mon Sep 17 00:00:00 2001 From: zeripath Date: Mon, 21 Jun 2021 23:12:22 +0100 Subject: [PATCH 09/10] Use html.Parse rather than html.ParseFragment (#16223) * Use html.Parse rather than html.ParseFragment There have been a few issues with html.ParseFragment - just use html.Parse instead. * Skip document node Signed-off-by: Andrew Thornton --- modules/markup/html.go | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/modules/markup/html.go b/modules/markup/html.go index edf860da4510d..0cc0e23b5c57d 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -304,27 +304,26 @@ func postProcess(ctx *RenderContext, procs []processor, input io.Reader, output _, _ = res.WriteString("") // parse the HTML - nodes, err := html.ParseFragment(res, nil) + node, err := html.Parse(res) if err != nil { return &postProcessError{"invalid HTML", err} } - for _, node := range nodes { - visitNode(ctx, procs, node, true) + if node.Type == html.DocumentNode { + node = node.FirstChild } - newNodes := make([]*html.Node, 0, len(nodes)) + visitNode(ctx, procs, node, true) - for _, node := range nodes { - if node.Data == "html" { - node = node.FirstChild - for node != nil && node.Data != "body" { - node = node.NextSibling - } - } - if node == nil { - continue + newNodes := make([]*html.Node, 0, 5) + + if node.Data == "html" { + node = node.FirstChild + for node != nil && node.Data != "body" { + node = node.NextSibling } + } + if node != nil { if node.Data == "body" { child := node.FirstChild for child != nil { From 66f8da538a8b1bd63ea1a0f97202ee0d46c15c4f Mon Sep 17 00:00:00 2001 From: sebastian-sauer Date: Tue, 22 Jun 2021 22:13:31 +0200 Subject: [PATCH 10/10] Use pulls url if issue is a pull request (#16230) if a pull request is displayed use the /pulls path if a pull requests diff is displayed use the /pulls/{id}/files url if an issue is displayed use the issues url Fixes #16102 Signed-off-by: Sebastian Sauer --- templates/repo/issue/view_content/context_menu.tmpl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/templates/repo/issue/view_content/context_menu.tmpl b/templates/repo/issue/view_content/context_menu.tmpl index e3001cddce5a8..fdefde979dc5a 100644 --- a/templates/repo/issue/view_content/context_menu.tmpl +++ b/templates/repo/issue/view_content/context_menu.tmpl @@ -6,7 +6,11 @@
{{ $referenceUrl := "" }} {{ if .issue }} - {{ $referenceUrl = Printf "%s%s/issues/%d#%s" AppUrl .ctx.Repository.FullName .ctx.Issue.Index .item.HashTag }} + {{ if .ctx.Issue.IsPull}} + {{ $referenceUrl = Printf "%s%s/pulls/%d#%s" AppUrl .ctx.Repository.FullName .ctx.Issue.Index .item.HashTag }} + {{ else }} + {{ $referenceUrl = Printf "%s%s/issues/%d#%s" AppUrl .ctx.Repository.FullName .ctx.Issue.Index .item.HashTag }} + {{ end }} {{ else }} {{ $referenceUrl = Printf "%s%s/pulls/%d/files#%s" AppUrl .ctx.Repository.FullName .ctx.Issue.Index .item.HashTag }} {{ end }}