Merge cockroachdb#139026 cockroachdb#139192 cockroachdb#139200 cockro…

…achdb#139246 cockroachdb#139327

139026: licences: update THIRD-PARTY-NOTICES.txt r=celiala a=rail

Fixes: REL-1744
Release note: None

139192: roachtest: disable 4K block size intent resolution test r=pav-kv a=andrewbaptist

With a 4K block size, the intent resolution test will cause unacceptable slowdowns. This commit changes the perturbation/*/intents to only test up to 1024 block sizes.

This also reduces the max perturbation duration to 10 minutes.

Informs: cockroachdb#139187
Informs: cockroachdb#139188
Informs: cockroachdb#137590 
Informs: cockroachdb#135934
Fixes: cockroachdb#137590

Epic: none

Release note: None

139200: roachprod: Create/Destroy should avoid listing _all_ providers r=golgeek,RaduBerinde a=srosenberg

Previously, both `Create` and `Destroy` would attempt to list VMs across _all_ active cloud providers. Not only is it inefficient, but `Create` may also fail if
the user isn't re-authenticated to an unrelated
provider. E.g., `create --clouds gce` may fail
if AWS SSO token expired.

This change derives the set of required providers
from either the user-specified `--clouds` CLI option or the local cluster cache. The set is then used with `ListCloud` to skip listing unrelated providers.
The use of the local cache is sound at this time;
cluster's providers are immutable.

Epic: none

Release note: None

139246: ui: show job messages on the job detail page r=dt a=dt

Release note (ui change): Jobs can now choose to emit messages that are shown on the job detail page in 25.1+.

Epic: none.

<img width="1513" alt="Screenshot 2025-01-16 at 14 46 12" src="https://github.com/user-attachments/assets/1718ccdd-e8a1-4ec2-a032-a646f410f918" />


139327: roachtest/cdc: skip cdc/bank r=rharding6373 a=wenyihu6

This patch skips the cdc/bank test, as it has been broken since cockroachdb#137947. This
patch disables the test while we work on the fix.

Informs: cockroachdb#139109
Release note: none
Epic: none

Co-authored-by: Rail Aliiev <rail@iqchoice.com>
Co-authored-by: Andrew Baptist <baptist@cockroachlabs.com>
Co-authored-by: Stan Rosenberg <stan.rosenberg@gmail.com>
Co-authored-by: David Taylor <tinystatemachine@gmail.com>
Co-authored-by: Wenyi Hu <wenyi@cockroachlabs.com>

pkg/cmd/roachtest/tests/cdc.go

@@ -2294,6 +2294,7 @@ func registerCDC(r registry.Registry) {
 	r.Add(registry.TestSpec{
 		Name:             "cdc/bank",
 		Owner:            `cdc`,
+		Skip:             "#139109",
 		Cluster:          r.MakeClusterSpec(4, spec.WorkloadNode()),
 		Leases:           registry.MetamorphicLeases,
 		CompatibleClouds: registry.AllExceptAWS,

pkg/cmd/roachtest/tests/perturbation/intents.go

@@ -33,7 +33,19 @@ func (i intents) setup() variations {
 }
 
 func (i intents) setupMetamorphic(rng *rand.Rand) variations {
-	return i.setup().randomize(rng)
+	v := i.setup()
+	v = v.randomize(rng)
+
+	// TODO(#139187): Large block sizes result in a big slowdown when the intents
+	// are written.
+	// TODO(#139188): Large block sizes result in a big slowdown when the intents
+	// are resolved.
+	v.blockSize = min(v.blockSize, 1024)
+
+	// TODO(#135934): A large buildup of intents still causes a large slowdown
+	// when the intents are resolved.
+	v.perturbationDuration = max(v.perturbationDuration, 10*time.Minute)
+	return v
 }
 
 func (intents) startTargetNode(ctx context.Context, t test.Test, v variations) {

pkg/roachprod/cloud/cluster_cloud.go

@@ -10,6 +10,7 @@ import (
 	"context"
 	"fmt"
 	"regexp"
+	"slices"
 	"sort"
 	"text/tabwriter"
 	"time"
@@ -323,6 +324,17 @@ type ClusterCreateOpts struct {
 	ProviderOptsContainer vm.ProviderOptionsContainer
 }
 
+// Extracts o.CreateOpts.VMProviders from the provided opts.
+func Providers(opts ...*ClusterCreateOpts) []string {
+	providers := []string{}
+	for _, o := range opts {
+		providers = append(providers, o.CreateOpts.VMProviders...)
+	}
+	// Remove dupes, if any.
+	slices.Sort(providers)
+	return slices.Compact(providers)
+}
+
 // CreateCluster TODO(peter): document
 // opts is a slice of all node VM specs to be provisioned for the cluster. Generally,
 // non uniform VM specs are not supported for a CRDB cluster, but we often want to provision

pkg/roachprod/roachprod.go

@@ -1404,6 +1404,25 @@ func Pprof(ctx context.Context, l *logger.Logger, clusterName string, opts Pprof
 	return nil
 }
 
+// Returns a set of cloud providers for the given clusters, found in the local cache.
+// Typically used in conjunction with ListCloud() to avoid listing across all providers.
+func cachedProvidersForClusters(clusterNames ...string) []string {
+	providers := []string{}
+	for _, clusterName := range clusterNames {
+		c, err := getClusterFromCache(nil, clusterName)
+		if err != nil {
+			continue
+		}
+		// N.B. We can't use c.Clouds() because it may insert a project name.
+		for _, m := range c.VMs {
+			providers = append(providers, m.Provider)
+		}
+	}
+	// Remove dupes, if any.
+	slices.Sort(providers)
+	return slices.Compact(providers)
+}
+
 // Destroy TODO
 func Destroy(
 	l *logger.Logger,
@@ -1463,7 +1482,7 @@ func Destroy(
 				// ListCloud may fail due to a transient provider error, but we may have still
 				// found the cluster(s) we care about. Destroy the cluster(s) we know about
 				// and let the caller retry.
-				cld, _ = cloud.ListCloud(l, vm.ListOptions{IncludeEmptyClusters: true})
+				cld, _ = cloud.ListCloud(l, vm.ListOptions{IncludeEmptyClusters: true, IncludeProviders: cachedProvidersForClusters(name)})
 			}
 			err := destroyCluster(ctx, cld, l, name)
 			if err != nil {
@@ -1617,13 +1636,14 @@ func Create(
 	if err := LoadClusters(); err != nil {
 		return errors.Wrap(err, "problem loading clusters")
 	}
+	includeProviders := cloud.Providers(opts...)
 
 	if !isLocal {
 		// ListCloud may fail due to a transient provider error, but
 		// we may not even be creating a cluster with that provider.
 		// If the cluster does exist, and we didn't find it, it will
 		// fail on the provider's end.
-		cld, _ := cloud.ListCloud(l, vm.ListOptions{})
+		cld, _ := cloud.ListCloud(l, vm.ListOptions{IncludeProviders: includeProviders})
 		if _, ok := cld.Clusters[clusterName]; ok {
 			return &ClusterAlreadyExistsError{name: clusterName}
 		}
@@ -3035,7 +3055,7 @@ func getClusterFromCloud(l *logger.Logger, clusterName string) (*cloud.Cluster,
 	// ListCloud may fail due to a transient provider error, but
 	// we may have still found the cluster we care about. It will
 	// fail below if it can't find the cluster.
-	cld, err := cloud.ListCloud(l, vm.ListOptions{})
+	cld, err := cloud.ListCloud(l, vm.ListOptions{IncludeProviders: cachedProvidersForClusters(clusterName)})
 	c, ok := cld.Clusters[clusterName]
 	if !ok {
 		if err != nil {

pkg/roachprod/vm/aws/support.go

@@ -314,7 +314,7 @@ func (p *Provider) runCommand(l *logger.Logger, args []string) ([]byte, error) {
 	output, err := cmd.Output()
 	if err != nil {
 		if exitErr := (*exec.ExitError)(nil); errors.As(err, &exitErr) {
-			l.Printf("%s", string(exitErr.Stderr))
+			l.Printf("%s", exitErr)
 		}
 		return nil, errors.Wrapf(err, "failed to run: aws %s: stderr: %v",
 			strings.Join(args, " "), stderrBuf.String())

pkg/roachprod/vm/vm.go

@@ -621,7 +621,9 @@ func FindActiveAccounts(l *logger.Logger) (map[string]string, error) {
 		err := ProvidersSequential(AllProviderNames(), func(p Provider) error {
 			account, err := p.FindActiveAccount(l)
 			if err != nil {
-				return err
+				l.Printf("WARN: provider=%q has no active account", p.Name())
+				//nolint:returnerrcheck
+				return nil
 			}
 			if len(account) > 0 {
 				source[p.Name()] = account

pkg/ui/workspaces/cluster-ui/src/api/jobsApi.ts

@@ -16,6 +16,7 @@ export type JobsResponse = cockroach.server.serverpb.JobsResponse;
 
 export type JobRequest = cockroach.server.serverpb.JobRequest;
 export type JobResponse = cockroach.server.serverpb.JobResponse;
+
 export type JobResponseWithKey = {
   jobResponse: JobResponse;
   key: string;

pkg/ui/workspaces/cluster-ui/src/jobs/jobDetailsPage/jobDetails.tsx

@@ -16,19 +16,23 @@ import { JobRequest, JobResponse } from "src/api/jobsApi";
 import { Button } from "src/button";
 import { commonStyles } from "src/common";
 import { CockroachCloudContext } from "src/contexts";
+import { EmptyTable } from "src/empty";
 import jobStyles from "src/jobs/jobs.module.scss";
 import { HighwaterTimestamp } from "src/jobs/util/highwaterTimestamp";
 import { JobStatusCell } from "src/jobs/util/jobStatusCell";
 import { Loading } from "src/loading";
+import { SortedTable } from "src/sortedtable";
 import { SqlBox, SqlBoxSize } from "src/sql";
 import { UIConfigState } from "src/store";
 import { SummaryCard, SummaryCardItem } from "src/summaryCard";
 import summaryCardStyles from "src/summaryCard/summaryCard.module.scss";
+import { Text, TextTypes } from "src/text";
 import {
+  DATE_WITH_SECONDS_AND_MILLISECONDS_FORMAT_24_TZ,
+  DATE_WITH_SECONDS_FORMAT,
   TimestampToMoment,
-  idAttr,
   getMatchParamByName,
-  DATE_WITH_SECONDS_AND_MILLISECONDS_FORMAT_24_TZ,
+  idAttr,
 } from "src/util";
 
 import {
@@ -43,6 +47,8 @@ import { isTerminalState } from "../util/jobOptions";
 
 import { JobProfilerView } from "./jobProfilerView";
 
+type JobMessage = JobResponse["messages"][number];
+
 const { TabPane } = Tabs;
 
 const cardCx = classNames.bind(summaryCardStyles);
@@ -151,9 +157,43 @@ export class JobDetails extends React.Component<
       return null;
     }
 
+    const messageColumns = [
+      {
+        name: "timestamp",
+        title: "When",
+        hideTitleUnderline: true,
+        cell: (x: JobMessage) => (
+          <Timestamp
+            time={TimestampToMoment(x.timestamp, null)}
+            format={DATE_WITH_SECONDS_FORMAT}
+          />
+        ),
+      },
+      {
+        name: "kind",
+        title: "Kind",
+        hideTitleUnderline: true,
+        cell: (x: JobMessage) => x.kind,
+      },
+      {
+        name: "message",
+        title: "Message",
+        hideTitleUnderline: true,
+        cell: (x: JobMessage) => (
+          <p className={jobCx("message")}>{x.message}</p>
+        ),
+      },
+    ];
+
     return (
       <Row gutter={24}>
-        <Col className="gutter-row" span={24}>
+        <Col className="gutter-row" span={8}>
+          <Text
+            textType={TextTypes.Heading5}
+            className={jobCx("details-header")}
+          >
+            Details
+          </Text>
           <SummaryCard className={cardCx("summary-card")}>
             <SummaryCardItem
               label="Status"
@@ -206,6 +246,22 @@ export class JobDetails extends React.Component<
             )}
           </SummaryCard>
         </Col>
+        <Col className="gutter-row" span={16}>
+          <Text
+            textType={TextTypes.Heading5}
+            className={jobCx("details-header")}
+          >
+            Events
+          </Text>
+          <SummaryCard className={jobCx("messages-card")}>
+            <SortedTable
+              data={job.messages}
+              columns={messageColumns}
+              tableWrapperClassName={jobCx("job-messages", "sorted-table")}
+              renderNoResult={<EmptyTable title="No messages recorded." />}
+            />
+          </SummaryCard>
+        </Col>
       </Row>
     );
   };

pkg/ui/workspaces/cluster-ui/src/jobs/jobs.module.scss

@@ -190,6 +190,21 @@
     line-height: 22px;
     color: $colors--neutral-7;
   }
+
+  .job-messages {
+    overflow: scroll;
+    .message {
+      white-space: pre;
+    }
+  }
+
+  .details-header {
+    margin-top: 12px;
+    margin-bottom: 12px;
+  }
+  .messages-card {
+    padding: 0;
+  }
 }
 
 .inline-message {