feat(github-releases): getDigest()

jest.config.ts

@@ -10,7 +10,7 @@ const config: InitialOptionsTsJest = {
   collectCoverageFrom: [
     'lib/**/*.{js,ts}',
     '!lib/**/*.{d,spec}.ts',
-    '!lib/**/{__fixtures__,__mocks__,__testutil__}/**/*.{js,ts}',
+    '!lib/**/{__fixtures__,__mocks__,__testutil__,test}/**/*.{js,ts}',
     '!lib/**/types.ts',
   ],
   coverageReporters: ci

lib/datasource/github-releases/common.spec.ts

@@ -0,0 +1,42 @@
+import { getName } from '../../../test/util';
+import { getApiBaseUrl, getGithubRelease, getSourceUrlBase } from './common';
+import { GitHubReleaseMocker } from './test';
+
+describe(getName(), () => {
+  describe('getSourceUrlBase', () => {
+    it('ensures trailing slash', () => {
+      const sourceUrl = getSourceUrlBase('https://gh.my-company.com');
+      expect(sourceUrl).toBe('https://gh.my-company.com/');
+    });
+    it('defaults to github.com', () => {
+      const sourceUrl = getSourceUrlBase(null);
+      expect(sourceUrl).toBe('https://github.com/');
+    });
+  });
+
+  describe('getApiBaseUrl', () => {
+    it('maps to api.github.com', () => {
+      const apiUrl = getApiBaseUrl('https://github.com/');
+      expect(apiUrl).toBe('https://api.github.com/');
+    });
+
+    it('supports local github installations', () => {
+      const apiUrl = getApiBaseUrl('https://gh.my-company.com/');
+      expect(apiUrl).toBe('https://gh.my-company.com/api/v3/');
+    });
+  });
+
+  describe('getGithubRelease', () => {
+    const apiUrl = 'https://github.com/';
+    const lookupName = 'someDep';
+    const releaseMock = new GitHubReleaseMocker(apiUrl, lookupName);
+
+    it('returns release', async () => {
+      const version = 'v1.0.0';
+      releaseMock.release(version);
+
+      const release = await getGithubRelease(apiUrl, lookupName, version);
+      expect(release.tag_name).toBe(version);
+    });
+  });
+});

lib/datasource/github-releases/common.ts

@@ -0,0 +1,29 @@
+import { GithubHttp } from '../../util/http/github';
+import { ensureTrailingSlash } from '../../util/url';
+import type { GithubRelease } from './types';
+
+const defaultSourceUrlBase = 'https://github.com/';
+
+export const cacheNamespace = 'datasource-github-releases';
+export const http = new GithubHttp();
+
+export function getSourceUrlBase(registryUrl: string): string {
+  // default to GitHub.com if no GHE host is specified.
+  return ensureTrailingSlash(registryUrl ?? defaultSourceUrlBase);
+}
+
+export function getApiBaseUrl(sourceUrlBase: string): string {
+  return sourceUrlBase === defaultSourceUrlBase
+    ? `https://api.github.com/`
+    : `${sourceUrlBase}api/v3/`;
+}
+
+export async function getGithubRelease(
+  apiBaseUrl: string,
+  repo: string,
+  version: string
+): Promise<GithubRelease> {
+  const url = `${apiBaseUrl}repos/${repo}/releases/tags/${version}`;
+  const res = await http.getJson<GithubRelease>(url);
+  return res.body;
+}

lib/datasource/github-releases/digest.spec.ts

@@ -0,0 +1,143 @@
+import hasha from 'hasha';
+import * as httpMock from '../../../test/http-mock';
+import { getName } from '../../../test/util';
+import { findDigestAsset, mapDigestAssetToRelease } from './digest';
+import { GitHubReleaseMocker } from './test';
+import { DigestAsset } from './types';
+
+describe(getName(), () => {
+  const lookupName = 'some/dep';
+  const releaseMock = new GitHubReleaseMocker(
+    'https://api.github.com',
+    lookupName
+  );
+
+  describe('findDigestAsset', () => {
+    it('finds SHASUMS.txt file containing digest', async () => {
+      const release = releaseMock.withDigestFileAsset(
+        'v1.0.0',
+        'test-digest    linux-amd64.tar.gz',
+        'another-digest linux-arm64.tar.gz'
+      );
+
+      const digestAsset = await findDigestAsset(release, 'test-digest');
+      expect(digestAsset.assetName).toBe('SHASUMS.txt');
+      expect(digestAsset.digestedFileName).toBe('linux-amd64.tar.gz');
+    });
+
+    it('returns null when not found in digest file asset', async () => {
+      const release = releaseMock.withDigestFileAsset(
+        'v1.0.0',
+        'another-digest linux-arm64.tar.gz'
+      );
+      // Small assets like this digest file may be downloaded twice
+      httpMock
+        .scope('https://api.github.com')
+        .get(`/repos/${lookupName}/releases/download/v1.0.0/SHASUMS.txt`)
+        .reply(200, '');
+
+      const digestAsset = await findDigestAsset(release, 'test-digest');
+      expect(digestAsset).toBeNull();
+    });
+
+    it('finds asset by digest', async () => {
+      const content = '1'.repeat(10 * 1024);
+      const release = releaseMock.withAssets('v1.0.0', {
+        'smaller.zip': '1'.repeat(9 * 1024),
+        'same-size.zip': '2'.repeat(10 * 1024),
+        'asset.zip': content,
+        'smallest.zip': '1'.repeat(8 * 1024),
+      });
+      const contentDigest = await hasha.async(content, { algorithm: 'sha256' });
+
+      const digestAsset = await findDigestAsset(release, contentDigest);
+      expect(digestAsset.assetName).toBe('asset.zip');
+      expect(digestAsset.digestedFileName).toBeUndefined();
+    });
+
+    it('returns null when no assets available', async () => {
+      const release = releaseMock.release('v1.0.0');
+      const digestAsset = await findDigestAsset(release, 'test-digest');
+      expect(digestAsset).toBeNull();
+    });
+  });
+
+  describe('mapDigestAssetToRelease', () => {
+    describe('with digest file', () => {
+      const digestAsset: DigestAsset = {
+        assetName: 'SHASUMS.txt',
+        currentVersion: 'v1.0.0',
+        currentDigest: 'old-digest',
+        digestedFileName: 'asset.zip',
+      };
+
+      it('downloads updated digest file', async () => {
+        const release = releaseMock.withDigestFileAsset(
+          'v1.0.1',
+          'updated-digest  asset.zip'
+        );
+        const digest = await mapDigestAssetToRelease(digestAsset, release);
+        expect(digest).toBe('updated-digest');
+      });
+
+      it('maps digested file name to new version', async () => {
+        const digestAssetWithVersion = {
+          ...digestAsset,
+          digestedFileName: 'asset-1.0.0.zip',
+        };
+
+        const release = releaseMock.withDigestFileAsset(
+          'v1.0.1',
+          'updated-digest  asset-1.0.1.zip'
+        );
+        const digest = await mapDigestAssetToRelease(
+          digestAssetWithVersion,
+          release
+        );
+        expect(digest).toBe('updated-digest');
+      });
+
+      it('returns null when not found in digest file', async () => {
+        const release = releaseMock.withDigestFileAsset(
+          'v1.0.1',
+          'moot-digest asset.tar.gz'
+        );
+        const digest = await mapDigestAssetToRelease(digestAsset, release);
+        expect(digest).toBeNull();
+      });
+
+      it('returns null when digest file not found', async () => {
+        const release = releaseMock.release('v1.0.1');
+        const digest = await mapDigestAssetToRelease(digestAsset, release);
+        expect(digest).toBeNull();
+      });
+    });
+
+    describe('with digested file', () => {
+      const digestAsset: DigestAsset = {
+        assetName: 'asset.zip',
+        currentVersion: 'v1.0.0',
+        currentDigest: '0'.repeat(64),
+      };
+
+      it('digests updated file', async () => {
+        const updatedContent = 'new content';
+        const release = releaseMock.withAssets('v1.0.1', {
+          'asset.zip': updatedContent,
+        });
+        const contentDigest = await hasha.async(updatedContent, {
+          algorithm: 'sha256',
+        });
+
+        const digest = await mapDigestAssetToRelease(digestAsset, release);
+        expect(digest).toEqual(contentDigest);
+      });
+
+      it('returns null when not found', async () => {
+        const release = releaseMock.release('v1.0.1');
+        const digest = await mapDigestAssetToRelease(digestAsset, release);
+        expect(digest).toBeNull();
+      });
+    });
+  });
+});

lib/datasource/github-releases/digest.ts

@@ -0,0 +1,141 @@
+import hasha from 'hasha';
+import * as packageCache from '../../util/cache/package';
+import { cacheNamespace, http } from './common';
+import type { DigestAsset, GithubRelease, GithubReleaseAsset } from './types';
+
+async function findDigestFile(
+  release: GithubRelease,
+  digest: string
+): Promise<DigestAsset | null> {
+  const smallAssets = release.assets.filter(
+    (a: GithubReleaseAsset) => a.size < 5 * 1024
+  );
+  for (const asset of smallAssets) {
+    const res = await http.get(asset.browser_download_url);
+    for (const line of res.body.split('\n')) {
+      const [lineDigest, lineFn] = line.split(/\s+/, 2);
+      if (lineDigest === digest) {
+        return {
+          assetName: asset.name,
+          digestedFileName: lineFn,
+          currentVersion: release.tag_name,
+          currentDigest: lineDigest,
+        };
+      }
+    }
+  }
+  return null;
+}
+
+function inferHashAlg(digest: string): string {
+  switch (digest.length) {
+    case 64:
+      return 'sha256';
+    default:
+    case 96:
+      return 'sha512';
+  }
+}
+
+function getAssetDigestCacheKey(
+  downloadUrl: string,
+  algorithm: string
+): string {
+  const type = 'assetDigest';
+  return `${downloadUrl}:${algorithm}:${type}`;
+}
+
+async function downloadAndDigest(
+  asset: GithubReleaseAsset,
+  algorithm: string
+): Promise<string> {
+  const downloadUrl = asset.browser_download_url;
+  const cacheKey = getAssetDigestCacheKey(downloadUrl, algorithm);
+  const cachedResult = await packageCache.get<string>(cacheNamespace, cacheKey);
+  // istanbul ignore if
+  if (cachedResult) {
+    return cachedResult;
+  }
+
+  const res = http.stream(downloadUrl);
+  const digest = await hasha.fromStream(res, { algorithm });
+
+  const cacheMinutes = 1440;
+  await packageCache.set(cacheNamespace, cacheKey, digest, cacheMinutes);
+  return digest;
+}
+
+async function findAssetWithDigest(
+  release: GithubRelease,
+  digest: string
+): Promise<DigestAsset | null> {
+  const algorithm = inferHashAlg(digest);
+  const assetsBySize = release.assets.sort(
+    (a: GithubReleaseAsset, b: GithubReleaseAsset) => {
+      if (a.size < b.size) {
+        return -1;
+      }
+      if (a.size > b.size) {
+        return 1;
+      }
+      return 0;
+    }
+  );
+
+  for (const asset of assetsBySize) {
+    const assetDigest = await downloadAndDigest(asset, algorithm);
+    if (assetDigest === digest) {
+      return {
+        assetName: asset.name,
+        currentVersion: release.tag_name,
+        currentDigest: assetDigest,
+      };
+    }
+  }
+  return null;
+}
+
+/** Identify the asset associated with a known digest. */
+export async function findDigestAsset(
+  release: GithubRelease,
+  digest: string
+): Promise<DigestAsset> {
+  const digestFile = await findDigestFile(release, digest);
+  if (digestFile) {
+    return digestFile;
+  }
+
+  const asset = await findAssetWithDigest(release, digest);
+  return asset;
+}
+
+/** Given a digest asset, find the equivalent digest in a different release. */
+export async function mapDigestAssetToRelease(
+  digestAsset: DigestAsset,
+  release: GithubRelease
+): Promise<string | null> {
+  const current = digestAsset.currentVersion.replace(/^v/, '');
+  const next = release.tag_name.replace(/^v/, '');
+  const releaseChecksumAssetName = digestAsset.assetName.replace(current, next);
+  const releaseAsset = release.assets.find(
+    (a: GithubReleaseAsset) => a.name === releaseChecksumAssetName
+  );
+  if (!releaseAsset) {
+    return null;
+  }
+  if (digestAsset.digestedFileName) {
+    const releaseFilename = digestAsset.digestedFileName.replace(current, next);
+    const res = await http.get(releaseAsset.browser_download_url);
+    for (const line of res.body.split('\n')) {
+      const [lineDigest, lineFn] = line.split(/\s+/, 2);
+      if (lineFn === releaseFilename) {
+        return lineDigest;
+      }
+    }
+  } else {
+    const algorithm = inferHashAlg(digestAsset.currentDigest);
+    const newDigest = await downloadAndDigest(releaseAsset, algorithm);
+    return newDigest;
+  }
+  return null;
+}