Skip to content

Commit

Permalink
internal/relui,internal/task: move useful structs to task
Browse files Browse the repository at this point in the history 
For whatever reason I've decided to write the tagging workflow in the
task package, so I need the test helpers here. I also need the website
structs to find the latest Go release later.

For golang/go#48523.

Change-Id: I92a1430bd27f0aa90c9fccefa12fb057e7e7f864
Reviewed-on: https://go-review.googlesource.com/c/build/+/425090
Reviewed-by: Dmitri Shuralyov <dmitshur@golang.org>
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>
Run-TryBot: Heschi Kreinick <heschi@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Auto-Submit: Heschi Kreinick <heschi@google.com>
  • Loading branch information
@heschi @gopherbot
heschi authored and gopherbot committed Sep 15, 2022
1 parent 58cf979 commit 5c85681
Show file tree
Hide file tree
Showing 5 changed files with 117 additions and 119 deletions.
4 changes: 2 additions & 2 deletions cmd/relui/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ func main() {
ScratchURL: *scratchFilesBase,
ServingURL: *servingFilesBase,
DownloadURL: *edgeCacheURL,
PublishFile: func(f *relui.WebsiteFile) error {
PublishFile: func(f *task.WebsiteFile) error {
return publishFile(*websiteUploadURL, userPassAuth, f)
},
ApproveAction: relui.ApproveActionDep(dbPool),
Expand Down Expand Up @@ -228,7 +228,7 @@ func key(masterKey, principal string) string {
return fmt.Sprintf("%x", h.Sum(nil))
}

func publishFile(uploadURL string, auth buildlet.UserPass, f *relui.WebsiteFile) error {
func publishFile(uploadURL string, auth buildlet.UserPass, f *task.WebsiteFile) error {
req, err := json.Marshal(f)
if err != nil {
return err
Expand Down
30 changes: 15 additions & 15 deletions internal/relui/buildrelease_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ type releaseTestDeps struct {
versionTasks *task.VersionTasks
buildTasks *BuildReleaseTasks
milestoneTasks *task.MilestoneTasks
publishedFiles map[string]*WebsiteFile
publishedFiles map[string]*task.WebsiteFile
outputListener func(taskName string, output interface{})
}

Expand Down Expand Up @@ -115,8 +115,8 @@ func newReleaseTestDeps(t *testing.T, wantVersion string) *releaseTestDeps {

// Set up the fake website to publish to.
var filesMu sync.Mutex
files := map[string]*WebsiteFile{}
publishFile := func(f *WebsiteFile) error {
files := map[string]*task.WebsiteFile{}
publishFile := func(f *task.WebsiteFile) error {
filesMu.Lock()
defer filesMu.Unlock()
files[strings.TrimPrefix(f.Filename, wantVersion+".")] = f
Expand Down Expand Up @@ -197,20 +197,20 @@ func testRelease(t *testing.T, wantVersion string, kind task.ReleaseKind) {
}

dlURL, files := deps.buildTasks.DownloadURL, deps.publishedFiles
checkTGZ(t, dlURL, files, "src.tar.gz", &WebsiteFile{
checkTGZ(t, dlURL, files, "src.tar.gz", &task.WebsiteFile{
OS: "",
Arch: "",
Kind: "source",
}, map[string]string{
"go/VERSION": wantVersion,
"go/src/make.bash": makeScript,
})
checkContents(t, dlURL, files, "windows-amd64.msi", &WebsiteFile{
checkContents(t, dlURL, files, "windows-amd64.msi", &task.WebsiteFile{
OS: "windows",
Arch: "amd64",
Kind: "installer",
}, "I'm an MSI!\n")
checkTGZ(t, dlURL, files, "linux-amd64.tar.gz", &WebsiteFile{
checkTGZ(t, dlURL, files, "linux-amd64.tar.gz", &task.WebsiteFile{
OS: "linux",
Arch: "amd64",
Kind: "archive",
Expand All @@ -219,23 +219,23 @@ func testRelease(t *testing.T, wantVersion string, kind task.ReleaseKind) {
"go/tool/something_orother/compile": "",
"go/pkg/something_orother/race.a": "",
})
checkZip(t, dlURL, files, "windows-arm64.zip", &WebsiteFile{
checkZip(t, dlURL, files, "windows-arm64.zip", &task.WebsiteFile{
OS: "windows",
Arch: "arm64",
Kind: "archive",
}, map[string]string{
"go/VERSION": wantVersion,
"go/tool/something_orother/compile": "",
})
checkTGZ(t, dlURL, files, "linux-armv6l.tar.gz", &WebsiteFile{
checkTGZ(t, dlURL, files, "linux-armv6l.tar.gz", &task.WebsiteFile{
OS: "linux",
Arch: "armv6l",
Kind: "archive",
}, map[string]string{
"go/VERSION": wantVersion,
"go/tool/something_orother/compile": "",
})
checkContents(t, dlURL, files, "darwin-amd64.pkg", &WebsiteFile{
checkContents(t, dlURL, files, "darwin-amd64.pkg", &task.WebsiteFile{
OS: "darwin",
Arch: "amd64",
Kind: "installer",
Expand Down Expand Up @@ -312,7 +312,7 @@ func testSecurity(t *testing.T, mergeFixes bool) {
runToFailure(t, deps.ctx, w, "Check branch state matches source archive", &verboseListener{t, deps.outputListener})
return
}
checkTGZ(t, deps.buildTasks.DownloadURL, deps.publishedFiles, "src.tar.gz", &WebsiteFile{
checkTGZ(t, deps.buildTasks.DownloadURL, deps.publishedFiles, "src.tar.gz", &task.WebsiteFile{
OS: "",
Arch: "",
Kind: "source",
Expand Down Expand Up @@ -465,13 +465,13 @@ func serveTarball(pathMatch string, files map[string]string, w http.ResponseWrit
}
}

func checkFile(t *testing.T, dlURL string, files map[string]*WebsiteFile, filename string, meta *WebsiteFile, check func(*testing.T, []byte)) {
func checkFile(t *testing.T, dlURL string, files map[string]*task.WebsiteFile, filename string, meta *task.WebsiteFile, check func(*testing.T, []byte)) {
t.Run(filename, func(t *testing.T) {
f, ok := files[filename]
if !ok {
t.Fatalf("file %q not published", filename)
}
if diff := cmp.Diff(meta, f, cmpopts.IgnoreFields(WebsiteFile{}, "Filename", "Version", "ChecksumSHA256", "Size")); diff != "" {
if diff := cmp.Diff(meta, f, cmpopts.IgnoreFields(task.WebsiteFile{}, "Filename", "Version", "ChecksumSHA256", "Size")); diff != "" {
t.Errorf("file metadata mismatch (-want +got):\n%v", diff)
}
resp, err := http.Get(dlURL + "/" + f.Filename)
Expand All @@ -486,15 +486,15 @@ func checkFile(t *testing.T, dlURL string, files map[string]*WebsiteFile, filena
})
}

func checkContents(t *testing.T, dlURL string, files map[string]*WebsiteFile, filename string, meta *WebsiteFile, contents string) {
func checkContents(t *testing.T, dlURL string, files map[string]*task.WebsiteFile, filename string, meta *task.WebsiteFile, contents string) {
checkFile(t, dlURL, files, filename, meta, func(t *testing.T, b []byte) {
if got, want := string(b), contents; got != want {
t.Errorf("%v contains %q, want %q", filename, got, want)
}
})
}

func checkTGZ(t *testing.T, dlURL string, files map[string]*WebsiteFile, filename string, meta *WebsiteFile, contents map[string]string) {
func checkTGZ(t *testing.T, dlURL string, files map[string]*task.WebsiteFile, filename string, meta *task.WebsiteFile, contents map[string]string) {
checkFile(t, dlURL, files, filename, meta, func(t *testing.T, b []byte) {
gzr, err := gzip.NewReader(bytes.NewReader(b))
if err != nil {
Expand Down Expand Up @@ -528,7 +528,7 @@ func checkTGZ(t *testing.T, dlURL string, files map[string]*WebsiteFile, filenam
})
}

func checkZip(t *testing.T, dlURL string, files map[string]*WebsiteFile, filename string, meta *WebsiteFile, contents map[string]string) {
func checkZip(t *testing.T, dlURL string, files map[string]*task.WebsiteFile, filename string, meta *task.WebsiteFile, contents map[string]string) {
checkFile(t, dlURL, files, filename, meta, func(t *testing.T, b []byte) {
zr, err := zip.NewReader(bytes.NewReader(b), int64(len(b)))
if err != nil {
Expand Down
76 changes: 4 additions & 72 deletions internal/relui/workflows.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -550,7 +550,7 @@ type BuildReleaseTasks struct {
GCSClient *storage.Client
ScratchURL, ServingURL string
DownloadURL string
PublishFile func(*WebsiteFile) error
PublishFile func(*task.WebsiteFile) error
CreateBuildlet func(context.Context, string) (buildlet.RemoteClient, error)
ApproveAction func(*wf.TaskContext) error
}
Expand Down Expand Up @@ -705,8 +705,8 @@ func (b *BuildReleaseTasks) runBuildStep(
return artifact{}, err
}
defer client.Close()
w := &logWriter{logger: ctx}
go w.run(ctx)
w := &task.LogWriter{Logger: ctx}
go w.Run(ctx)
step = &task.BuildletStep{
Target: target,
Buildlet: client,
Expand Down Expand Up @@ -812,62 +812,6 @@ func (w *sizeWriter) Write(p []byte) (n int, err error) {
return len(p), nil
}

type logWriter struct {
flushTicker *time.Ticker

mu sync.Mutex
buf []byte
logger wf.Logger
}

func (w *logWriter) Write(b []byte) (int, error) {
w.mu.Lock()
defer w.mu.Unlock()

w.buf = append(w.buf, b...)
if len(w.buf) > 1<<20 {
w.flushLocked(false)
w.flushTicker.Reset(10 * time.Second)
}
return len(b), nil
}

func (w *logWriter) flush(force bool) {
w.mu.Lock()
defer w.mu.Unlock()
w.flushLocked(force)
}

func (w *logWriter) flushLocked(force bool) {
if len(w.buf) == 0 {
return
}
log, rest := w.buf, []byte(nil)
if !force {
nl := bytes.LastIndexByte(w.buf, '\n')
if nl == -1 {
return
}
log, rest = w.buf[:nl], w.buf[nl+1:]
}
w.logger.Printf("\n%s", string(log))
w.buf = append([]byte(nil), rest...) // don't leak
}

func (w *logWriter) run(ctx context.Context) {
w.flushTicker = time.NewTicker(10 * time.Second)
defer w.flushTicker.Stop()
for {
select {
case <-w.flushTicker.C:
w.flush(false)
case <-ctx.Done():
w.flush(true)
return
}
}
}

func (tasks *BuildReleaseTasks) startSigningCommand(ctx *wf.TaskContext, version string) (string, error) {
args := fmt.Sprintf("--relui_staging=%q", tasks.ScratchURL+"/"+signingStagingDir(ctx, version))
ctx.Printf("run signer with " + args)
Expand Down Expand Up @@ -1099,7 +1043,7 @@ func uploadArtifact(scratchFS, servingFS fs.FS, a artifact) error {
// The version string uses the same format as Go tags. For example, "go1.19rc1".
func (tasks *BuildReleaseTasks) publishArtifacts(ctx *wf.TaskContext, version string, artifacts []artifact) (publishedVersion string, _ error) {
for _, a := range artifacts {
f := &WebsiteFile{
f := &task.WebsiteFile{
Filename: a.Filename,
Version: version,
ChecksumSHA256: a.SHA256,
Expand Down Expand Up @@ -1127,15 +1071,3 @@ func (tasks *BuildReleaseTasks) publishArtifacts(ctx *wf.TaskContext, version st
ctx.Printf("Published %v artifacts for %v", len(artifacts), version)
return version, nil
}

// WebsiteFile represents a file on the go.dev downloads page.
// It should be kept in sync with the download code in x/website/internal/dl.
type WebsiteFile struct {
Filename string `json:"filename"`
OS string `json:"os"`
Arch string `json:"arch"`
Version string `json:"version"`
ChecksumSHA256 string `json:"sha256"`
Size int64 `json:"size"`
Kind string `json:"kind"` // "archive", "installer", "source"
}
93 changes: 91 additions & 2 deletions internal/task/task.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,12 @@
package task

import (
"bytes"
"context"
"sync"
"time"

"golang.org/x/build/internal/workflow"
wf "golang.org/x/build/internal/workflow"
)

// CommunicationTasks combines communication tasks together.
Expand All @@ -22,7 +25,7 @@ var AwaitDivisor int = 1
// AwaitCondition calls the condition function every period until it returns
// true to indicate success, or an error. If the condition succeeds,
// AwaitCondition returns its result.
func AwaitCondition[T any](ctx *workflow.TaskContext, period time.Duration, condition func() (T, bool, error)) (T, error) {
func AwaitCondition[T any](ctx *wf.TaskContext, period time.Duration, condition func() (T, bool, error)) (T, error) {
pollTimer := time.NewTicker(period / time.Duration(AwaitDivisor))
defer pollTimer.Stop()
for {
Expand All @@ -39,3 +42,89 @@ func AwaitCondition[T any](ctx *workflow.TaskContext, period time.Duration, cond
}
}
}

// LogWriter is an io.Writer that writes to a workflow task's log, flushing
// its buffer periodically to avoid too many writes.
type LogWriter struct {
Logger wf.Logger

flushTicker *time.Ticker

mu sync.Mutex
buf []byte
}

func (w *LogWriter) Write(b []byte) (int, error) {
w.mu.Lock()
defer w.mu.Unlock()

w.buf = append(w.buf, b...)
if len(w.buf) > 1<<20 {
w.flushLocked(false)
w.flushTicker.Reset(10 * time.Second)
}
return len(b), nil
}

func (w *LogWriter) flush(force bool) {
w.mu.Lock()
defer w.mu.Unlock()
w.flushLocked(force)
}

func (w *LogWriter) flushLocked(force bool) {
if len(w.buf) == 0 {
return
}
log, rest := w.buf, []byte(nil)
if !force {
nl := bytes.LastIndexByte(w.buf, '\n')
if nl == -1 {
return
}
log, rest = w.buf[:nl], w.buf[nl+1:]
}
w.Logger.Printf("\n%s", string(log))
w.buf = append([]byte(nil), rest...) // don't leak
}

func (w *LogWriter) Run(ctx context.Context) {
w.flushTicker = time.NewTicker(10 * time.Second)
defer w.flushTicker.Stop()
for {
select {
case <-w.flushTicker.C:
w.flush(false)
case <-ctx.Done():
w.flush(true)
return
}
}
}

// WebsiteFile represents a file on the go.dev downloads page.
// It should be kept in sync with the download code in x/website/internal/dl.
type WebsiteFile struct {
Filename string `json:"filename"`
OS string `json:"os"`
Arch string `json:"arch"`
Version string `json:"version"`
ChecksumSHA256 string `json:"sha256"`
Size int64 `json:"size"`
Kind string `json:"kind"` // "archive", "installer", "source"
}

func (f WebsiteFile) GOARCH() string {
if f.OS == "linux" && f.Arch == "armv6l" {
return "arm"
}
return f.Arch
}

type WebsiteRelease struct {
Version string `json:"version"`
Stable bool `json:"stable"`
Files []WebsiteFile `json:"files"`
Visible bool `json:"-"` // show files on page load
SplitPortTable bool `json:"-"` // whether files should be split by primary/other ports.
}
Loading

0 comments on commit 5c85681

Please sign in to comment.