chore(storage): fix ReadStallTimeout test (#10992)

* chore(storage): fix ReadStallTimeout test

Fix this test to work in the CI. Also remove a spurious log
statement.

* improve resp body logging

* re-add debug logging

* reduce stall times

* fix timings

storage/client_test.go

@@ -1456,53 +1456,54 @@ func TestRetryDeadlineExceedeEmulated(t *testing.T) {
 
 // Test validates the retry for stalled read-request, when client is created with
 // WithReadStallTimeout.
-func TestRetryReadReqStallEmulated(t *testing.T) {
+func TestRetryReadStallEmulated(t *testing.T) {
  checkEmulatorEnvironment(t)
- multiTransportTest(skipJSONReads(skipGRPC("not supported"), "not supported"), t, func(t *testing.T, ctx context.Context, project, _ string, client *Client) {
- // Setup bucket and upload object.
- bucket := fmt.Sprintf("http-bucket-%d", time.Now().Nanosecond())
- if _, err := client.tc.CreateBucket(context.Background(), project, bucket, &BucketAttrs{Name: bucket}, nil); err != nil {
- t.Fatalf("client.CreateBucket: %v", err)
- }
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
 
- name, _, _, err := createObjectWithContent(ctx, bucket, randomBytes3MiB)
- if err != nil {
- t.Fatalf("createObject: %v", err)
- }
+ // Initialize new storage.Client with ReadStallTimeout option. This cannot be initialized
+ // at the transportClient level so we must use NewClient for this test.
+ client, err := NewClient(ctx, experimental.WithReadStallTimeout(
+ &experimental.ReadStallTimeoutConfig{
+ TargetPercentile: 0.99,
+ Min: 10 * time.Millisecond,
+ }))
+ if err != nil {
+ t.Fatalf("storage.NewClient: %v", err)
+ }
 
- // Plant stall at start for 2s.
- instructions := map[string][]string{"storage.objects.get": {"stall-for-2s-after-0K"}}
- testID := createRetryTest(t, client.tc, instructions)
- ctx = callctx.SetHeaders(ctx, "x-retry-test-id", testID)
+ // Setup bucket and upload object.
+ project := "fake-project"
+ bucket := fmt.Sprintf("http-bucket-%d", time.Now().Nanosecond())
+ if err := client.Bucket(bucket).Create(ctx, project, nil); err != nil {
+ t.Fatalf("client.Bucket.Create: %v", err)
+ }
 
- ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
- defer cancel()
+ name, _, _, err := createObjectWithContent(ctx, bucket, randomBytes3MiB)
+ if err != nil {
+ t.Fatalf("createObject: %v", err)
+ }
 
- r, err := client.tc.NewRangeReader(ctx, &newRangeReaderParams{
- bucket: bucket,
- object: name,
- gen: defaultGen,
- offset: 0,
- length: -1,
- }, idempotent(true))
- if err != nil {
- t.Fatalf("NewRangeReader: %v", err)
- }
- defer r.Close()
+ // Plant stall at start for 10s.
+ // The ReadStallTimeout should cause the stalled request to be stopped and
+ // retried before hitting the 5s context deadline.
+ instructions := map[string][]string{"storage.objects.get": {"stall-for-10s-after-0K"}}
+ testID := createRetryTest(t, client.tc, instructions)
 
- buf := &bytes.Buffer{}
- if _, err := io.Copy(buf, r); err != nil {
- t.Fatalf("io.Copy: %v", err)
- }
- if !bytes.Equal(buf.Bytes(), randomBytes3MiB) {
- t.Errorf("content does not match, got len %v, want len %v", buf.Len(), len(randomBytes3MiB))
- }
+ ctx = callctx.SetHeaders(ctx, "x-retry-test-id", testID)
+ r, err := client.Bucket(bucket).Object(name).NewReader(ctx)
+ if err != nil {
+ t.Fatalf("NewReader: %v", err)
+ }
+ defer r.Close()
 
- }, experimental.WithReadStallTimeout(
- &experimental.ReadStallTimeoutConfig{
- TargetPercentile: 0.99,
- Min: time.Second,
- }))
+ buf := &bytes.Buffer{}
+ if _, err := io.Copy(buf, r); err != nil {
+ t.Fatalf("io.Copy: %v", err)
+ }
+ if !bytes.Equal(buf.Bytes(), randomBytes3MiB) {
+ t.Errorf("content does not match, got len %v, want len %v", buf.Len(), len(randomBytes3MiB))
+ }
 }
 
 // createRetryTest creates a bucket in the emulator and sets up a test using the

storage/retry_conformance_test.go

@@ -777,7 +777,11 @@ func (et *emulatorTest) create(instructions map[string][]string, transport strin
  et.T.Skip("This retry test case is not yet supported in the testbench.")
  }
  if err != nil || resp.StatusCode != 200 {
- et.Fatalf("creating retry test: err: %v, resp: %+v", err, resp)
+ var respBody string
+ if body, err := io.ReadAll(resp.Body); err != nil {
+ respBody = string(body)
+ }
+ et.Fatalf("creating retry test: err: %v, resp: %+v, resp body: %v", err, resp, respBody)
  }
  defer func() {
  closeErr := resp.Body.Close()