From 9a7a8a761e2eb17006ee4fecd34e49cdc63a71d4 Mon Sep 17 00:00:00 2001 From: Peter Bengtsson Date: Tue, 7 Dec 2021 08:21:23 -0500 Subject: [PATCH] make lib/rest/index.js callable (#23214) * Experiment with making the tarball smaller Part of #1248 * try this * stop debugging * delete translations too * delete heavy search indexes too * push and popd * try this hack * delete but leave directory * debug more * faster delete of translations * less loud * async await * async await * no tree * simplify * experimenting more * unfinished * only the large files * change order * brotli with level 6 * cope better with decorated rest json files * tidying * keep images * cleaning * cleaning up * refactored function * try this * better comment * remove console logging * more important changes * make lib/rest/index.js callable Part of #1177 * memoize * remove console logging --- lib/rest/index.js | 104 +++++++++++++++-------------- middleware/contextualizers/rest.js | 13 +++- tests/rendering/rest.js | 6 +- tests/unit/openapi-schema.js | 11 ++- 4 files changed, 76 insertions(+), 58 deletions(-) diff --git a/lib/rest/index.js b/lib/rest/index.js index bcf61aa0b9c0..90c3f3f7408d 100644 --- a/lib/rest/index.js +++ b/lib/rest/index.js @@ -8,66 +8,68 @@ import { readCompressedJsonFileFallback } from '../read-json-file.js' const __dirname = path.dirname(fileURLToPath(import.meta.url)) const schemasPath = path.join(__dirname, 'static/decorated') -export const operations = {} -fs.readdirSync(schemasPath).forEach((filename) => { - // In staging deploys, the `.json` files might have been converted to - // to `.json.br`. The `readCompressedJsonFileFallback()` function - // can handle both but you need to call it with the `.json` filename. - const key = path.parse(filename).name - const value = readCompressedJsonFileFallback(path.join(schemasPath, filename)) - operations[key] = value -}) - -// Get initial set of keys that will be deleted once new keys are set -const openApiOperationKeys = Object.keys(operations) +export default async function getRest() { + const operations = {} + fs.readdirSync(schemasPath).forEach((filename) => { + // In staging deploys, the `.json` files might have been converted to + // to `.json.br`. The `readCompressedJsonFileFallback()` function + // can handle both but you need to call it with the `.json` filename. + const key = path.parse(filename).name + const value = readCompressedJsonFileFallback(path.join(schemasPath, filename)) + operations[key] = value + }) -let allCategories = [] -allVersionKeys.forEach((currentVersion) => { - // Translate the versions from the openapi to versions used in the docs - const openApiVersion = allVersions[currentVersion].openApiVersionName + // Get initial set of keys that will be deleted once new keys are set + const openApiOperationKeys = Object.keys(operations) - // Check that the openApiVersion is configured in OpenAPI - if (!operations[openApiVersion]) return + let allCategories = [] + allVersionKeys.forEach((currentVersion) => { + // Translate the versions from the openapi to versions used in the docs + const openApiVersion = allVersions[currentVersion].openApiVersionName - operations[currentVersion] = operations[openApiVersion] + // Check that the openApiVersion is configured in OpenAPI + if (!operations[openApiVersion]) return - // This list is generated for use in the tests, - // so we can verify that the names of the markdown files - // in content/rest/reference/*.md are congruous with the - // set of REST resource names like activity, gists, repos, etc. - allCategories = allCategories.concat( - chain(operations[currentVersion]).map('category').sort().uniq().value() - ) + operations[currentVersion] = operations[openApiVersion] - // Attach convenience properties to each operation that can't easily be created in Liquid - operations[currentVersion].forEach((operation) => { - operation.hasRequiredPreviews = get(operation, 'x-github.previews', []).some( - (preview) => preview.required + // This list is generated for use in the tests, + // so we can verify that the names of the markdown files + // in content/rest/reference/*.md are congruous with the + // set of REST resource names like activity, gists, repos, etc. + allCategories = allCategories.concat( + chain(operations[currentVersion]).map('category').sort().uniq().value() ) + + // Attach convenience properties to each operation that can't easily be created in Liquid + operations[currentVersion].forEach((operation) => { + operation.hasRequiredPreviews = get(operation, 'x-github.previews', []).some( + (preview) => preview.required + ) + }) }) -}) -// Get the unique set of categories -const categories = [...new Set(allCategories)] + // Get the unique set of categories + const categories = [...new Set(allCategories)] -// Remove openapi base name keys that have been replaced with version key -openApiOperationKeys.forEach((openApiVersionName) => { - delete operations[openApiVersionName] -}) + // Remove openapi base name keys that have been replaced with version key + openApiOperationKeys.forEach((openApiVersionName) => { + delete operations[openApiVersionName] + }) -// This is a collection of operations that have `enabledForGitHubApps = true` -// It's grouped by resource title to make rendering easier -const operationsEnabledForGitHubApps = allVersionKeys.reduce((acc, currentVersion) => { - acc[currentVersion] = chain(operations[currentVersion] || []) - .filter((operation) => operation['x-github'].enabledForGitHubApps) - .orderBy('category') - .value() - acc[currentVersion] = groupBy(acc[currentVersion], 'category') - return acc -}, {}) + // This is a collection of operations that have `enabledForGitHubApps = true` + // It's grouped by resource title to make rendering easier + const operationsEnabledForGitHubApps = allVersionKeys.reduce((acc, currentVersion) => { + acc[currentVersion] = chain(operations[currentVersion] || []) + .filter((operation) => operation['x-github'].enabledForGitHubApps) + .orderBy('category') + .value() + acc[currentVersion] = groupBy(acc[currentVersion], 'category') + return acc + }, {}) -export default { - categories, - operations, - operationsEnabledForGitHubApps, + return { + categories, + operations, + operationsEnabledForGitHubApps, + } } diff --git a/middleware/contextualizers/rest.js b/middleware/contextualizers/rest.js index 3a695eaba81e..9ed8f2a06e7b 100644 --- a/middleware/contextualizers/rest.js +++ b/middleware/contextualizers/rest.js @@ -1,8 +1,17 @@ import path from 'path' -import rest from '../../lib/rest/index.js' +import getRest from '../../lib/rest/index.js' import removeFPTFromPath from '../../lib/remove-fpt-from-path.js' -export default function restContext(req, res, next) { +// Global cache to avoid calling getRest() more than once +let rest = null + +export default async function restContext(req, res, next) { + // Bail early because these are pointless to contextualize + if (req.path.startsWith('/_next/static')) return next() + + if (!rest) { + rest = await getRest() + } req.context.rest = rest // link to include in `Works with GitHub Apps` notes diff --git a/tests/rendering/rest.js b/tests/rendering/rest.js index 53f616fba814..8f6609759939 100644 --- a/tests/rendering/rest.js +++ b/tests/rendering/rest.js @@ -4,7 +4,7 @@ import fs from 'fs/promises' import { difference, isPlainObject } from 'lodash-es' import { getJSON } from '../helpers/supertest.js' import enterpriseServerReleases from '../../lib/enterprise-server-releases.js' -import rest from '../../lib/rest/index.js' +import getRest from '../../lib/rest/index.js' import { jest } from '@jest/globals' const __dirname = path.dirname(fileURLToPath(import.meta.url)) @@ -20,7 +20,7 @@ describe('REST references docs', () => { jest.setTimeout(3 * 60 * 1000) test('markdown file exists for every operationId prefix in the api.github.com schema', async () => { - const { categories } = rest + const { categories } = await getRest() const referenceDir = path.join(__dirname, '../../content/rest/reference') const filenames = (await fs.readdir(referenceDir)) .filter( @@ -61,7 +61,7 @@ describe('REST references docs', () => { }) test('no wrongly detected AppleScript syntax highlighting in schema data', async () => { - const { operations } = rest + const { operations } = await getRest() expect(JSON.stringify(operations).includes('hljs language-applescript')).toBe(false) }) }) diff --git a/tests/unit/openapi-schema.js b/tests/unit/openapi-schema.js index 7bf10786b3f5..1e49f50dbbc5 100644 --- a/tests/unit/openapi-schema.js +++ b/tests/unit/openapi-schema.js @@ -4,11 +4,18 @@ import walk from 'walk-sync' import { get, isPlainObject } from 'lodash-es' import { allVersions } from '../../lib/all-versions.js' import nonEnterpriseDefaultVersion from '../../lib/non-enterprise-default-version.js' -import { operations } from '../../lib/rest/index.js' +import getRest from '../../lib/rest/index.js' import dedent from 'dedent' +import { beforeAll } from '@jest/globals' const __dirname = path.dirname(fileURLToPath(import.meta.url)) const schemasPath = path.join(__dirname, '../../lib/rest/static/decorated') -const nonEnterpriseDefaultVersionSchema = operations[nonEnterpriseDefaultVersion] + +let operations = null +let nonEnterpriseDefaultVersionSchema = null +beforeAll(async () => { + operations = (await getRest()).operations + nonEnterpriseDefaultVersionSchema = operations[nonEnterpriseDefaultVersion] +}) describe('OpenAPI schema validation', () => { test('makes an object', () => {