diff --git a/cmd/relui/main.go b/cmd/relui/main.go index d263671aea..7335af32ea 100644 --- a/cmd/relui/main.go +++ b/cmd/relui/main.go @@ -175,7 +175,7 @@ func main() { ScratchURL: *scratchFilesBase, ServingURL: *servingFilesBase, DownloadURL: *edgeCacheURL, - PublishFile: func(f *relui.WebsiteFile) error { + PublishFile: func(f *task.WebsiteFile) error { return publishFile(*websiteUploadURL, userPassAuth, f) }, ApproveAction: relui.ApproveActionDep(dbPool), @@ -228,7 +228,7 @@ func key(masterKey, principal string) string { return fmt.Sprintf("%x", h.Sum(nil)) } -func publishFile(uploadURL string, auth buildlet.UserPass, f *relui.WebsiteFile) error { +func publishFile(uploadURL string, auth buildlet.UserPass, f *task.WebsiteFile) error { req, err := json.Marshal(f) if err != nil { return err diff --git a/internal/relui/buildrelease_test.go b/internal/relui/buildrelease_test.go index dddd85ff56..323b221a9f 100644 --- a/internal/relui/buildrelease_test.go +++ b/internal/relui/buildrelease_test.go @@ -66,7 +66,7 @@ type releaseTestDeps struct { versionTasks *task.VersionTasks buildTasks *BuildReleaseTasks milestoneTasks *task.MilestoneTasks - publishedFiles map[string]*WebsiteFile + publishedFiles map[string]*task.WebsiteFile outputListener func(taskName string, output interface{}) } @@ -115,8 +115,8 @@ func newReleaseTestDeps(t *testing.T, wantVersion string) *releaseTestDeps { // Set up the fake website to publish to. var filesMu sync.Mutex - files := map[string]*WebsiteFile{} - publishFile := func(f *WebsiteFile) error { + files := map[string]*task.WebsiteFile{} + publishFile := func(f *task.WebsiteFile) error { filesMu.Lock() defer filesMu.Unlock() files[strings.TrimPrefix(f.Filename, wantVersion+".")] = f @@ -197,7 +197,7 @@ func testRelease(t *testing.T, wantVersion string, kind task.ReleaseKind) { } dlURL, files := deps.buildTasks.DownloadURL, deps.publishedFiles - checkTGZ(t, dlURL, files, "src.tar.gz", &WebsiteFile{ + checkTGZ(t, dlURL, files, "src.tar.gz", &task.WebsiteFile{ OS: "", Arch: "", Kind: "source", @@ -205,12 +205,12 @@ func testRelease(t *testing.T, wantVersion string, kind task.ReleaseKind) { "go/VERSION": wantVersion, "go/src/make.bash": makeScript, }) - checkContents(t, dlURL, files, "windows-amd64.msi", &WebsiteFile{ + checkContents(t, dlURL, files, "windows-amd64.msi", &task.WebsiteFile{ OS: "windows", Arch: "amd64", Kind: "installer", }, "I'm an MSI!\n") - checkTGZ(t, dlURL, files, "linux-amd64.tar.gz", &WebsiteFile{ + checkTGZ(t, dlURL, files, "linux-amd64.tar.gz", &task.WebsiteFile{ OS: "linux", Arch: "amd64", Kind: "archive", @@ -219,7 +219,7 @@ func testRelease(t *testing.T, wantVersion string, kind task.ReleaseKind) { "go/tool/something_orother/compile": "", "go/pkg/something_orother/race.a": "", }) - checkZip(t, dlURL, files, "windows-arm64.zip", &WebsiteFile{ + checkZip(t, dlURL, files, "windows-arm64.zip", &task.WebsiteFile{ OS: "windows", Arch: "arm64", Kind: "archive", @@ -227,7 +227,7 @@ func testRelease(t *testing.T, wantVersion string, kind task.ReleaseKind) { "go/VERSION": wantVersion, "go/tool/something_orother/compile": "", }) - checkTGZ(t, dlURL, files, "linux-armv6l.tar.gz", &WebsiteFile{ + checkTGZ(t, dlURL, files, "linux-armv6l.tar.gz", &task.WebsiteFile{ OS: "linux", Arch: "armv6l", Kind: "archive", @@ -235,7 +235,7 @@ func testRelease(t *testing.T, wantVersion string, kind task.ReleaseKind) { "go/VERSION": wantVersion, "go/tool/something_orother/compile": "", }) - checkContents(t, dlURL, files, "darwin-amd64.pkg", &WebsiteFile{ + checkContents(t, dlURL, files, "darwin-amd64.pkg", &task.WebsiteFile{ OS: "darwin", Arch: "amd64", Kind: "installer", @@ -312,7 +312,7 @@ func testSecurity(t *testing.T, mergeFixes bool) { runToFailure(t, deps.ctx, w, "Check branch state matches source archive", &verboseListener{t, deps.outputListener}) return } - checkTGZ(t, deps.buildTasks.DownloadURL, deps.publishedFiles, "src.tar.gz", &WebsiteFile{ + checkTGZ(t, deps.buildTasks.DownloadURL, deps.publishedFiles, "src.tar.gz", &task.WebsiteFile{ OS: "", Arch: "", Kind: "source", @@ -465,13 +465,13 @@ func serveTarball(pathMatch string, files map[string]string, w http.ResponseWrit } } -func checkFile(t *testing.T, dlURL string, files map[string]*WebsiteFile, filename string, meta *WebsiteFile, check func(*testing.T, []byte)) { +func checkFile(t *testing.T, dlURL string, files map[string]*task.WebsiteFile, filename string, meta *task.WebsiteFile, check func(*testing.T, []byte)) { t.Run(filename, func(t *testing.T) { f, ok := files[filename] if !ok { t.Fatalf("file %q not published", filename) } - if diff := cmp.Diff(meta, f, cmpopts.IgnoreFields(WebsiteFile{}, "Filename", "Version", "ChecksumSHA256", "Size")); diff != "" { + if diff := cmp.Diff(meta, f, cmpopts.IgnoreFields(task.WebsiteFile{}, "Filename", "Version", "ChecksumSHA256", "Size")); diff != "" { t.Errorf("file metadata mismatch (-want +got):\n%v", diff) } resp, err := http.Get(dlURL + "/" + f.Filename) @@ -486,7 +486,7 @@ func checkFile(t *testing.T, dlURL string, files map[string]*WebsiteFile, filena }) } -func checkContents(t *testing.T, dlURL string, files map[string]*WebsiteFile, filename string, meta *WebsiteFile, contents string) { +func checkContents(t *testing.T, dlURL string, files map[string]*task.WebsiteFile, filename string, meta *task.WebsiteFile, contents string) { checkFile(t, dlURL, files, filename, meta, func(t *testing.T, b []byte) { if got, want := string(b), contents; got != want { t.Errorf("%v contains %q, want %q", filename, got, want) @@ -494,7 +494,7 @@ func checkContents(t *testing.T, dlURL string, files map[string]*WebsiteFile, fi }) } -func checkTGZ(t *testing.T, dlURL string, files map[string]*WebsiteFile, filename string, meta *WebsiteFile, contents map[string]string) { +func checkTGZ(t *testing.T, dlURL string, files map[string]*task.WebsiteFile, filename string, meta *task.WebsiteFile, contents map[string]string) { checkFile(t, dlURL, files, filename, meta, func(t *testing.T, b []byte) { gzr, err := gzip.NewReader(bytes.NewReader(b)) if err != nil { @@ -528,7 +528,7 @@ func checkTGZ(t *testing.T, dlURL string, files map[string]*WebsiteFile, filenam }) } -func checkZip(t *testing.T, dlURL string, files map[string]*WebsiteFile, filename string, meta *WebsiteFile, contents map[string]string) { +func checkZip(t *testing.T, dlURL string, files map[string]*task.WebsiteFile, filename string, meta *task.WebsiteFile, contents map[string]string) { checkFile(t, dlURL, files, filename, meta, func(t *testing.T, b []byte) { zr, err := zip.NewReader(bytes.NewReader(b), int64(len(b))) if err != nil { diff --git a/internal/relui/workflows.go b/internal/relui/workflows.go index cf4b80bf06..fe5659ca1f 100644 --- a/internal/relui/workflows.go +++ b/internal/relui/workflows.go @@ -550,7 +550,7 @@ type BuildReleaseTasks struct { GCSClient *storage.Client ScratchURL, ServingURL string DownloadURL string - PublishFile func(*WebsiteFile) error + PublishFile func(*task.WebsiteFile) error CreateBuildlet func(context.Context, string) (buildlet.RemoteClient, error) ApproveAction func(*wf.TaskContext) error } @@ -705,8 +705,8 @@ func (b *BuildReleaseTasks) runBuildStep( return artifact{}, err } defer client.Close() - w := &logWriter{logger: ctx} - go w.run(ctx) + w := &task.LogWriter{Logger: ctx} + go w.Run(ctx) step = &task.BuildletStep{ Target: target, Buildlet: client, @@ -812,62 +812,6 @@ func (w *sizeWriter) Write(p []byte) (n int, err error) { return len(p), nil } -type logWriter struct { - flushTicker *time.Ticker - - mu sync.Mutex - buf []byte - logger wf.Logger -} - -func (w *logWriter) Write(b []byte) (int, error) { - w.mu.Lock() - defer w.mu.Unlock() - - w.buf = append(w.buf, b...) - if len(w.buf) > 1<<20 { - w.flushLocked(false) - w.flushTicker.Reset(10 * time.Second) - } - return len(b), nil -} - -func (w *logWriter) flush(force bool) { - w.mu.Lock() - defer w.mu.Unlock() - w.flushLocked(force) -} - -func (w *logWriter) flushLocked(force bool) { - if len(w.buf) == 0 { - return - } - log, rest := w.buf, []byte(nil) - if !force { - nl := bytes.LastIndexByte(w.buf, '\n') - if nl == -1 { - return - } - log, rest = w.buf[:nl], w.buf[nl+1:] - } - w.logger.Printf("\n%s", string(log)) - w.buf = append([]byte(nil), rest...) // don't leak -} - -func (w *logWriter) run(ctx context.Context) { - w.flushTicker = time.NewTicker(10 * time.Second) - defer w.flushTicker.Stop() - for { - select { - case <-w.flushTicker.C: - w.flush(false) - case <-ctx.Done(): - w.flush(true) - return - } - } -} - func (tasks *BuildReleaseTasks) startSigningCommand(ctx *wf.TaskContext, version string) (string, error) { args := fmt.Sprintf("--relui_staging=%q", tasks.ScratchURL+"/"+signingStagingDir(ctx, version)) ctx.Printf("run signer with " + args) @@ -1099,7 +1043,7 @@ func uploadArtifact(scratchFS, servingFS fs.FS, a artifact) error { // The version string uses the same format as Go tags. For example, "go1.19rc1". func (tasks *BuildReleaseTasks) publishArtifacts(ctx *wf.TaskContext, version string, artifacts []artifact) (publishedVersion string, _ error) { for _, a := range artifacts { - f := &WebsiteFile{ + f := &task.WebsiteFile{ Filename: a.Filename, Version: version, ChecksumSHA256: a.SHA256, @@ -1127,15 +1071,3 @@ func (tasks *BuildReleaseTasks) publishArtifacts(ctx *wf.TaskContext, version st ctx.Printf("Published %v artifacts for %v", len(artifacts), version) return version, nil } - -// WebsiteFile represents a file on the go.dev downloads page. -// It should be kept in sync with the download code in x/website/internal/dl. -type WebsiteFile struct { - Filename string `json:"filename"` - OS string `json:"os"` - Arch string `json:"arch"` - Version string `json:"version"` - ChecksumSHA256 string `json:"sha256"` - Size int64 `json:"size"` - Kind string `json:"kind"` // "archive", "installer", "source" -} diff --git a/internal/task/task.go b/internal/task/task.go index 4c3fbd8ac2..ef0355e2cc 100644 --- a/internal/task/task.go +++ b/internal/task/task.go @@ -6,9 +6,12 @@ package task import ( + "bytes" + "context" + "sync" "time" - "golang.org/x/build/internal/workflow" + wf "golang.org/x/build/internal/workflow" ) // CommunicationTasks combines communication tasks together. @@ -22,7 +25,7 @@ var AwaitDivisor int = 1 // AwaitCondition calls the condition function every period until it returns // true to indicate success, or an error. If the condition succeeds, // AwaitCondition returns its result. -func AwaitCondition[T any](ctx *workflow.TaskContext, period time.Duration, condition func() (T, bool, error)) (T, error) { +func AwaitCondition[T any](ctx *wf.TaskContext, period time.Duration, condition func() (T, bool, error)) (T, error) { pollTimer := time.NewTicker(period / time.Duration(AwaitDivisor)) defer pollTimer.Stop() for { @@ -39,3 +42,89 @@ func AwaitCondition[T any](ctx *workflow.TaskContext, period time.Duration, cond } } } + +// LogWriter is an io.Writer that writes to a workflow task's log, flushing +// its buffer periodically to avoid too many writes. +type LogWriter struct { + Logger wf.Logger + + flushTicker *time.Ticker + + mu sync.Mutex + buf []byte +} + +func (w *LogWriter) Write(b []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + + w.buf = append(w.buf, b...) + if len(w.buf) > 1<<20 { + w.flushLocked(false) + w.flushTicker.Reset(10 * time.Second) + } + return len(b), nil +} + +func (w *LogWriter) flush(force bool) { + w.mu.Lock() + defer w.mu.Unlock() + w.flushLocked(force) +} + +func (w *LogWriter) flushLocked(force bool) { + if len(w.buf) == 0 { + return + } + log, rest := w.buf, []byte(nil) + if !force { + nl := bytes.LastIndexByte(w.buf, '\n') + if nl == -1 { + return + } + log, rest = w.buf[:nl], w.buf[nl+1:] + } + w.Logger.Printf("\n%s", string(log)) + w.buf = append([]byte(nil), rest...) // don't leak +} + +func (w *LogWriter) Run(ctx context.Context) { + w.flushTicker = time.NewTicker(10 * time.Second) + defer w.flushTicker.Stop() + for { + select { + case <-w.flushTicker.C: + w.flush(false) + case <-ctx.Done(): + w.flush(true) + return + } + } +} + +// WebsiteFile represents a file on the go.dev downloads page. +// It should be kept in sync with the download code in x/website/internal/dl. +type WebsiteFile struct { + Filename string `json:"filename"` + OS string `json:"os"` + Arch string `json:"arch"` + Version string `json:"version"` + ChecksumSHA256 string `json:"sha256"` + Size int64 `json:"size"` + Kind string `json:"kind"` // "archive", "installer", "source" +} + +func (f WebsiteFile) GOARCH() string { + if f.OS == "linux" && f.Arch == "armv6l" { + return "arm" + } + return f.Arch +} + +type WebsiteRelease struct { + Version string `json:"version"` + Stable bool `json:"stable"` + Files []WebsiteFile `json:"files"` + Visible bool `json:"-"` // show files on page load + SplitPortTable bool `json:"-"` // whether files should be split by primary/other ports. +} diff --git a/internal/task/tweet.go b/internal/task/tweet.go index 2d78950922..04ce69b2d9 100644 --- a/internal/task/tweet.go +++ b/internal/task/tweet.go @@ -378,15 +378,15 @@ func digits(i int64) int { // fetchRandomArchive downloads all release archives for Go version goVer, // and selects a random archive to showcase in the image that displays // sample output from the 'go install golang.org/dl/...@latest' command. -func fetchRandomArchive(goVer string, rnd *rand.Rand) (archive golangorgDLFile, _ error) { +func fetchRandomArchive(goVer string, rnd *rand.Rand) (archive WebsiteFile, _ error) { archives, err := fetchReleaseArchives(goVer) if err != nil { - return golangorgDLFile{}, err + return WebsiteFile{}, err } return archives[rnd.Intn(len(archives))], nil } -func fetchReleaseArchives(goVer string) (archives []golangorgDLFile, _ error) { +func fetchReleaseArchives(goVer string) (archives []WebsiteFile, _ error) { url := "https://go.dev/dl/?mode=json" if strings.Contains(goVer, "beta") || strings.Contains(goVer, "rc") || goVer == "go1.17" || goVer == "go1.17.1" || goVer == "go1.11.1" /* For TestTweetRelease. */ { @@ -403,7 +403,7 @@ func fetchReleaseArchives(goVer string) (archives []golangorgDLFile, _ error) { } else if ct := resp.Header.Get("Content-Type"); ct != "application/json" { return nil, fmt.Errorf("got Content-Type %q, want %q", ct, "application/json") } - var releases []golangorgDLRelease + var releases []WebsiteRelease err = json.NewDecoder(resp.Body).Decode(&releases) if err != nil { return nil, err @@ -412,7 +412,7 @@ func fetchReleaseArchives(goVer string) (archives []golangorgDLFile, _ error) { if r.Version != goVer { continue } - var archives []golangorgDLFile + var archives []WebsiteFile for _, f := range r.Files { if f.Kind != "archive" { continue @@ -428,29 +428,6 @@ func fetchReleaseArchives(goVer string) (archives []golangorgDLFile, _ error) { return nil, fmt.Errorf("release version %q not found", goVer) } -// golangorgDLRelease represents a release on the go.dev downloads page. -type golangorgDLRelease struct { - Version string - Files []golangorgDLFile -} - -// golangorgDLFile represents a file on the go.dev downloads page. -// It should be kept in sync with code in x/build/cmd/release and x/website/internal/dl. -type golangorgDLFile struct { - Filename string - OS string - Arch string - Size int64 - Kind string // One of "archive", "installer", "source". -} - -func (f golangorgDLFile) GOARCH() string { - if f.OS == "linux" && f.Arch == "armv6l" { - return "arm" - } - return f.Arch -} - // drawTerminal draws an image of a terminal window // with the given text displayed. func drawTerminal(text string) (image.Image, error) {