feat: add multistep upload support

src/commands/split.ts

@@ -2,7 +2,7 @@ import util from 'util';
 
 import axios from 'axios';
 
-import {client} from '../lib/axios';
+import {client, getRequestId} from '../lib/axios';
 import {multiGlob} from '../lib/file';
 import {Context} from '../lib/types';
 
@@ -83,7 +83,7 @@ export async function split(
         throw err;
       }
 
-      logger.error(`Request ID: ${err.response.headers['x-request-id']}`);
+      logger.error(`Request ID: ${getRequestId(err.response)}`);
       logger.error(util.inspect(err.response.data, {depth: 2}));
     }
 

src/commands/submit.spec.ts

@@ -15,7 +15,44 @@ describe('submit()', () => {
   beforeEach(() => nock.disableNetConnect());
   afterEach(() => nock.enableNetConnect());
 
-  it('submits reports to check run reporter', async () => {
+  it('submits reports to check run reporter using multistep upload', async () => {
+    nock('https://api.check-run-reporter.com')
+      .post('/api/v1/submissions/upload')
+      .reply(201, {
+        keys: [
+          'ianwremmel/check-run-reporter/c6fb3d5423762aa2b3a8f63717ef6b320e5a1b5a/SOMEUUID-reports/junit/jest.xml',
+        ],
+        signature:
+          '19c3cb9748107142317f4eb212b61d2be19a8c4b2aff9dc6117a7936b28313e5',
+        urls: {
+          'reports/junit/jest.xml': 'https://example.com/1',
+        },
+      });
+
+    nock('https://example.com').put('/1').reply(200);
+
+    nock('https://api.check-run-reporter.com')
+      .patch('/api/v1/submissions/upload')
+      .reply(202);
+
+    await submit(
+      {
+        label: 'foo',
+        report: ['reports/junit/**/*.xml'],
+        root: '/',
+        sha: '40923a72ddf9eefef938355fa96246607c706f6c',
+        token: 'FAKE TOKEN',
+        url: 'https://api.check-run-reporter.com/api/v1/submissions',
+      },
+      {logger}
+    );
+  });
+
+  it('submits reports to check run reporter, falling back to single step upload', async () => {
+    nock('https://api.check-run-reporter.com')
+      .post('/api/v1/submissions/upload')
+      .reply(404);
+
     nock('https://api.check-run-reporter.com')
       .post('/api/v1/submissions')
       .reply(201);

src/commands/submit.ts

@@ -4,7 +4,8 @@ import axios from 'axios';
 
 import {Context, Optional} from '../lib/types';
 // eslint-disable-next-line import/no-deprecated
-import {singleStepUpload} from '../lib/upload';
+import {multiStepUpload, singleStepUpload} from '../lib/upload';
+import {getRequestId} from '../lib/axios';
 
 interface SubmitArgs {
   readonly label: Optional<string>;
@@ -22,20 +23,8 @@ export async function submit(input: SubmitArgs, context: Context) {
   const {logger} = context;
 
   try {
-    const {label, report, root, sha, token, url} = input;
     logger.group('Uploading report to Check Run Reporter');
-    logger.info(`Label: ${label}`);
-    logger.info(`Root: ${root}`);
-    logger.info(`SHA: ${sha}`);
-    logger.debug(`URL: ${url}`);
-
-    // eslint-disable-next-line import/no-deprecated
-    const response = await singleStepUpload(input, context);
-
-    logger.info(`Request ID: ${response.headers['x-request-id']}`);
-    logger.info(`Status: ${response.status}`);
-    logger.info(`StatusText: ${response.statusText}`);
-    logger.info(JSON.stringify(response.data, null, 2));
+    await tryMultiStepUploadOrFallbackToSingle(input, context);
   } catch (err) {
     if (axios.isAxiosError(err)) {
       if (!err.response) {
@@ -45,12 +34,41 @@ export async function submit(input: SubmitArgs, context: Context) {
         throw err;
       }
 
-      logger.error(`Request ID: ${err.response.headers['x-request-id']}`);
-      logger.error(util.inspect(err.response.data, {depth: 2}));
+      logger.error(`Request ID: ${getRequestId(err.response)}`);
+      logger.error(
+        `Response Headers: ${util.inspect(err.response.headers, {depth: 2})}`
+      );
+      logger.error(
+        `Response Body: ${util.inspect(err.response.data, {depth: 2})}`
+      );
+      logger.error(`Request URL: ${err.response.config.url}`);
     }
 
     throw err;
   } finally {
     logger.groupEnd();
   }
 }
+/**
+ * Attempts to use multistep upload, but falls back to the legacy system if it
+ * gets a 404. This _should_ make things future proof so it'll get more
+ * efficient once the new version is released.
+ */
+async function tryMultiStepUploadOrFallbackToSingle(
+  input: SubmitArgs,
+  context: Context
+) {
+  try {
+    return await multiStepUpload(input, context);
+  } catch (err) {
+    if (axios.isAxiosError(err)) {
+      // CI doesn't like safe-access here.
+      if (err.response && err.response.status === 404) {
+        // eslint-disable-next-line import/no-deprecated
+        return await singleStepUpload(input, context);
+      }
+    }
+
+    throw err;
+  }
+}

src/lib/axios.ts

@@ -1,16 +1,29 @@
 import axiosRetry from 'axios-retry';
-import axios from 'axios';
+import axios, {AxiosResponse} from 'axios';
 import ci from 'ci-info';
 
 import pkg from '../../package.json';
 
 const {version} = pkg;
 
-const prInfo = typeof ci.isPR === 'boolean' ? `(PR: ${ci.isPR})` : '';
-
 export const client = axios.create({
   headers: {
-    'user-agent': `crr/${version} ${ci.name} ${prInfo}`,
+    'user-agent': [
+      `crr/${version}`,
+      ci.name,
+      typeof ci.isPR === 'boolean' ? `(PR: ${ci.isPR})` : null,
+    ]
+      .filter(Boolean)
+      .join(' '),
   },
 });
 axiosRetry(client, {retries: 3});
+
+/**
+ * extract the request id from the response object
+ */
+export function getRequestId(response: AxiosResponse) {
+  return (
+    response.headers['x-request-id'] || response.headers['x-amz-request-id']
+  );
+}

src/lib/upload.ts

@@ -2,7 +2,7 @@ import fs from 'fs';
 
 import FormData from 'form-data';
 
-import {client} from './axios';
+import {client, getRequestId} from './axios';
 import {multiGlob} from './file';
 import {Context, Optional} from './types';
 
@@ -15,6 +15,7 @@ interface UploadArgs {
   readonly url: string;
 }
 
+type URLs = Record<string, string>;
 /**
  * Uploads directly to Check Run Reporter. This is a legacy solution that no
  * longer works for large submissions thanks to new backend architecture. It
@@ -26,6 +27,13 @@ export async function singleStepUpload(
   {label, report, root, sha, token, url}: UploadArgs,
   context: Context
 ) {
+  const {logger} = context;
+
+  logger.info(`Label: ${label}`);
+  logger.info(`Root: ${root}`);
+  logger.info(`SHA: ${sha}`);
+  logger.debug(`URL: ${url}`);
+
   const filenames = await multiGlob(report, context);
 
   const formData = new FormData();
@@ -46,5 +54,104 @@ export async function singleStepUpload(
     },
     maxContentLength: Infinity,
   });
+
+  logger.info(`Request ID: ${getRequestId(response)}`);
+  logger.info(`Status: ${response.status}`);
+  logger.info(`StatusText: ${response.statusText}`);
+  logger.info(JSON.stringify(response.data, null, 2));
+
+  return response;
+}
+
+/**
+ * Orchestrates the multi-step upload process.
+ * @param args
+ * @param context
+ */
+export async function multiStepUpload(args: UploadArgs, context: Context) {
+  const {logger} = context;
+
+  const {label, report, root, sha, url} = args;
+
+  logger.info(`Label: ${label}`);
+  logger.info(`Root: ${root}`);
+  logger.info(`SHA: ${sha}`);
+  logger.debug(`URL: ${url}/upload`);
+
+  const filenames = await multiGlob(report, context);
+  logger.group('Requesing signed urls');
+  const {keys, urls, signature} = await getSignedUploadUrls(args, filenames);
+  logger.groupEnd();
+
+  logger.group('Uploading reports');
+  await uploadToSignedUrls(filenames, urls);
+  logger.groupEnd();
+
+  logger.group('Finalizing upload');
+  await finishMultistepUpload(args, keys, signature);
+  logger.groupEnd();
+}
+
+/** Fetches signed URLs */
+export async function getSignedUploadUrls(
+  args: UploadArgs,
+  filenames: readonly string[]
+): Promise<{keys: string[]; signature: string; urls: Record<string, string>}> {
+  const {label, root, sha, token, url} = args;
+
+  const response = await client.post(
+    `${url}/upload`,
+    {filenames, label, root, sha},
+    {
+      auth: {password: token, username: 'token'},
+
+      maxContentLength: Infinity,
+    }
+  );
+
+  return response.data;
+}
+
+/** Uploads directly to S3. */
+export async function uploadToSignedUrls(
+  filenames: readonly string[],
+  urls: URLs
+) {
+  for (const filename of filenames) {
+    const stream = fs.createReadStream(filename);
+    await client.put(urls[filename], stream, {
+      headers: {
+        'Content-Length': String((await fs.promises.stat(filename)).size),
+      },
+    });
+  }
+}
+
+/**
+ * Informs Check Run Reporter that all files have been uploaded and that
+ * processing may begin.
+ */
+export async function finishMultistepUpload(
+  args: UploadArgs,
+  keys: readonly string[],
+  signature: string
+) {
+  const {label, root, sha, token, url} = args;
+
+  const response = await client.patch(
+    `${url}/upload`,
+    {
+      keys,
+      label,
+      root,
+      sha,
+      signature,
+    },
+    {
+      auth: {password: token, username: 'token'},
+      maxContentLength: Infinity,
+    }
+  );
+
   return response;
 }