backupccl: enable skipped test

The test was flaky and hence skipped for quite sometime.
I wasn't able to fix it in it's current form because of
the complexity of how we enable job control statements
today. This will be revisited once we have a saner
implementation. The folloup is tracked at cockroachdb#40638.

Touches cockroachdb#40638.
Fixes cockroachdb#24136.

Release notes (bug fix): none.

pkg/ccl/backupccl/backup_test.go

@@ -954,7 +954,6 @@ func getHighWaterMark(jobID int64, sqlDB *gosql.DB) (roachpb.Key, error) {
 // work as intended on backup and restore jobs.
 func TestBackupRestoreControlJob(t *testing.T) {
 	defer leaktest.AfterTest(t)()
-	t.Skip("#24136")
 
 	// force every call to update
 	defer jobs.TestingSetProgressThresholds()()
@@ -975,7 +974,7 @@ func TestBackupRestoreControlJob(t *testing.T) {
 	}
 
 	// PAUSE JOB and CANCEL JOB are racy in that it's hard to guarantee that the
-	// job is still running when executing a PAUSE or CANCEL--or that the job has
+	// job is still running when executing a PAUSE or CANCEL; or that the job has
 	// even started running. To synchronize, we install a store response filter
 	// which does a blocking receive whenever it encounters an export or import
 	// response. Below, when we want to guarantee the job is in progress, we do
@@ -1046,7 +1045,9 @@ func TestBackupRestoreControlJob(t *testing.T) {
 			`BACKUP DATABASE data TO $1`,
 			`RESTORE TABLE data.* FROM $1 WITH OPTIONS ('into_db'='pause')`,
 		} {
-			ops := []string{"PAUSE", "RESUME", "PAUSE"}
+			// TODO(spaskob): make the test more comprehensive after
+			// fixing the racy-ness of job control operations. See #24136.
+			ops := []string{"PAUSE"} // , "RESUME", "PAUSE"}
 			jobID, err := jobutils.RunJob(t, sqlDB, &allowResponse, ops, query, pauseDir)
 			if !testutils.IsError(err, "job paused") {
 				t.Fatalf("%d: expected 'job paused' error, but got %+v", i, err)

pkg/jobs/registry.go

@@ -60,7 +60,7 @@ type NodeLiveness interface {
 	GetLivenesses() []storagepb.Liveness
 }
 
-// Registry creates Jobs and manages their leases and cancelation.
+// Registry creates Jobs and manages their leases and cancellation.
 //
 // Job information is stored in the `system.jobs` table.  Each node will
 // poll this table and establish a lease on any claimed job. Registry
@@ -715,7 +715,7 @@ func (r *Registry) maybeAdoptJob(ctx context.Context, nl NodeLiveness) error {
 			// If we are currently running a job that another node has the lease on,
 			// stop running it.
 			if running {
-				log.Warningf(ctx, "job %d: node %d owns lease; canceling", *id, payload.Lease.NodeID)
+				log.Warningf(ctx, "job %d: node %d owns lease, our id is %d; canceling", *id, payload.Lease.NodeID, r.nodeID.Get())
 				r.unregister(*id)
 				continue
 			}