Implement gateway retry logic for requests to GCS

packages/apollo-gateway/src/__tests__/integration/networkRequests.test.ts

@@ -1,22 +1,28 @@
 import nock from 'nock';
-import { ApolloGateway } from '../..';
+import { fetch } from 'apollo-server-env';
+import { ApolloGateway, GCS_RETRY_COUNT, getDefaultGcsFetcher } from '../..';
 import {
   mockLocalhostSDLQuery,
   mockStorageSecretSuccess,
+  mockStorageSecret,
   mockCompositionConfigLinkSuccess,
+  mockCompositionConfigLink,
   mockCompositionConfigsSuccess,
+  mockCompositionConfigs,
   mockImplementingServicesSuccess,
+  mockImplementingServices,
   mockRawPartialSchemaSuccess,
+  mockRawPartialSchema,
   apiKeyHash,
   graphId,
-  mockImplementingServices,
-  mockRawPartialSchema,
 } from './nockMocks';
 
 // This is a nice DX hack for GraphQL code highlighting and formatting within the file.
 // Anything wrapped within the gql tag within this file is just a string, not an AST.
 const gql = String.raw;
 
+let fetcher: typeof fetch;
+
 const service = {
   implementingServicePath: 'service-definition.json',
   partialSchemaPath: 'accounts-partial-schema.json',
@@ -33,7 +39,7 @@ const service = {
       name: String
       username: String
     }
-  `
+  `,
 };
 
 const updatedService = {
@@ -52,11 +58,19 @@ const updatedService = {
       name: String
       username: String
     }
-  `
-}
+  `,
+};
 
 beforeEach(() => {
   if (!nock.isActive()) nock.activate();
+
+  fetcher = getDefaultGcsFetcher().defaults({
+    retry: {
+      retries: GCS_RETRY_COUNT,
+      minTimeout: 10,
+      maxTimeout: 100,
+    }
+  });
 });
 
 afterEach(() => {
@@ -143,3 +157,58 @@ it('Rollsback to a previous schema when triggered', async () => {
 
   expect(onChange.mock.calls.length).toBe(2);
 });
+
+function failNTimes(n: number, fn: () => nock.Interceptor) {
+  for (let i = 0; i < n; i++) {
+    fn().reply(500);
+  }
+}
+
+it(`Retries GCS (up to ${GCS_RETRY_COUNT} times) on failure for each request and succeeds`, async () => {
+  failNTimes(GCS_RETRY_COUNT, mockStorageSecret);
+  mockStorageSecretSuccess();
+
+  failNTimes(GCS_RETRY_COUNT, mockCompositionConfigLink);
+  mockCompositionConfigLinkSuccess();
+
+  failNTimes(GCS_RETRY_COUNT, mockCompositionConfigs);
+  mockCompositionConfigsSuccess([service.implementingServicePath]);
+
+  failNTimes(GCS_RETRY_COUNT, () => mockImplementingServices(service));
+  mockImplementingServicesSuccess(service);
+
+  failNTimes(GCS_RETRY_COUNT, () => mockRawPartialSchema(service));
+  mockRawPartialSchemaSuccess(service);
+
+  const gateway = new ApolloGateway({ fetcher });
+
+  await gateway.load({ engine: { apiKeyHash, graphId } });
+  expect(gateway.schema!.getType('User')!.description).toBe('This is my User');
+});
+
+it(`Fails after the ${GCS_RETRY_COUNT + 1}th attempt to reach GCS`, async () => {
+  failNTimes(GCS_RETRY_COUNT + 1, mockStorageSecret);
+
+  const gateway = new ApolloGateway({ fetcher });
+  await expect(
+    gateway.load({ engine: { apiKeyHash, graphId } }),
+  ).rejects.toThrowErrorMatchingInlineSnapshot(
+    `"Could not communicate with Apollo Graph Manager storage: "`,
+  );
+});
+
+it(`Errors when the secret isn't hosted on GCS`, async () => {
+  mockStorageSecret().reply(
+    403,
+    `<Error><Code>AccessDenied</Code>
+    Anonymous caller does not have storage.objects.get`,
+    { 'content-type': 'application/xml' },
+  );
+
+  const gateway = new ApolloGateway({ fetcher });
+  await expect(
+    gateway.load({ engine: { apiKeyHash, graphId } }),
+  ).rejects.toThrowErrorMatchingInlineSnapshot(
+    `"Unable to authenticate with Apollo Graph Manager storage while fetching https://storage.googleapis.com/engine-partial-schema-prod/federated-service/storage-secret/dd55a79d467976346d229a7b12b673ce.json"`,
+  );
+});

packages/apollo-gateway/src/index.ts

@@ -152,6 +152,28 @@ type RequestContext<TContext> = WithRequired<
   'document' | 'queryHash'
 >;
 
+export const GCS_RETRY_COUNT = 5;
+
+export function getDefaultGcsFetcher() {
+  return fetcher.defaults({
+    cacheManager: new HttpRequestCache(),
+    // All headers should be lower-cased here, as `make-fetch-happen`
+    // treats differently cased headers as unique (unlike the `Headers` object).
+    // @see: https://git.io/JvRUa
+    headers: {
+      'user-agent': `apollo-gateway/${require('../package.json').version}`,
+    },
+    retry: {
+      retries: GCS_RETRY_COUNT,
+      // 1 second
+      minTimeout: 1000,
+      // 60 seconds - but this shouldn't be reachable based on current settings
+      maxTimeout: 60 * 1000,
+      randomize: true,
+    },
+  });
+}
+
 export class ApolloGateway implements GraphQLService {
   public schema?: GraphQLSchema;
   protected serviceMap: DataSourceCache = Object.create(null);
@@ -165,15 +187,7 @@ export class ApolloGateway implements GraphQLService {
   private compositionMetadata?: CompositionMetadata;
   private serviceSdlCache = new Map<string, string>();
 
-  private fetcher: typeof fetch = fetcher.defaults({
-    cacheManager: new HttpRequestCache(),
-    // All headers should be lower-cased here, as `make-fetch-happen`
-    // treats differently cased headers as unique (unlike the `Headers` object).
-    // @see: https://git.io/JvRUa
-    headers: {
-      'user-agent': `apollo-gateway/${require('../package.json').version}`
-    }
-  });
+  private fetcher: typeof fetch = getDefaultGcsFetcher();
 
   // Observe query plan, service info, and operation info prior to execution.
   // The information made available here will give insight into the resulting