Skip to content

Commit

Permalink
refactor rest code (#25879)
Browse files Browse the repository at this point in the history
  • Loading branch information
@rachmari
rachmari authored Mar 11, 2022
1 parent 142ffd0 commit 67360f5
Show file tree
Hide file tree
Showing 4 changed files with 275 additions and 271 deletions.
124 changes: 71 additions & 53 deletions lib/rest/index.js
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,15 @@
import { fileURLToPath } from 'url'
import path from 'path'
import fs from 'fs'
import { readCompressedJsonFileFallback } from '../read-json-file.js'
import renderContent from '../render-content/index.js'
import getMiniTocItems from '../get-mini-toc-items.js'
import { allVersions } from '../all-versions.js'
import { get } from 'lodash-es'
import languages from '../languages.js'

const __dirname = path.dirname(fileURLToPath(import.meta.url))
const schemasPath = path.join(__dirname, 'static/decorated')
const ENABLED_APPS_FILENAME = path.join(__dirname, 'static/apps/enabled-for-apps.json')
/*
/*
Loads the schemas from the static/decorated folder into a single
object organized by version.
Expand All @@ -23,23 +22,33 @@ const ENABLED_APPS_FILENAME = path.join(__dirname, 'static/apps/enabled-for-apps
}
}
*/
export default async function getRest() {
const operations = {}
fs.readdirSync(schemasPath).forEach((filename) => {
// In staging deploys, the `.json` files might have been converted to
// to `.json.br`. In that case, we need to also remove the `.json`
// extension from the key
const key = path.parse(filename).name.replace('.json', '')
const restOperationData = new Map()
Object.keys(languages).forEach((language) => {
restOperationData.set(language, new Map())
Object.keys(allVersions).forEach((version) => {
// setting to undefined will allow us to perform checks
// more easily later on
restOperationData.get(language).set(version, new Map())
})
})

const restOperations = new Map()
export default async function getRest(version, category) {
const openApiVersion = getOpenApiVersion(version)
if (!restOperations.has(openApiVersion)) {
const filename = `${openApiVersion}.json`

// The `readCompressedJsonFileFallback()` function
// will check for both a .br and .json extension.
const value = readCompressedJsonFileFallback(path.join(schemasPath, filename))
const docsVersions = getDocsVersions(key)
docsVersions.forEach((docsVersion) => {
operations[docsVersion] = value
})
})
restOperations.set(
openApiVersion,
readCompressedJsonFileFallback(path.join(schemasPath, filename))
)
}

return operations
return category
? restOperations.get(openApiVersion)[category]
: restOperations.get(openApiVersion)
}

// Right now there is not a 1:1 mapping of openapi to docs versions,
Expand All @@ -52,48 +61,57 @@ function getDocsVersions(openApiVersion) {
return versions
}

// Used for units tests
export function getFlatListOfOperations(operations) {
const flatList = []
Object.keys(operations).forEach((version) => {
Object.keys(operations[version]).forEach((category) => {
Object.keys(operations[version][category]).forEach((subcategory) => {
flatList.push(...operations[version][category][subcategory])
})
})
})
return flatList
function getOpenApiVersion(version) {
if (!(version in allVersions)) {
throw new Error(`Unrecognized version '${version}'. Not found in ${Object.keys(allVersions)}`)
}
return allVersions[version].openApiVersionName
}

// Generates the rendered Markdown from the data files in the
// data/reusables/rest-reference directory for a given category
// and generates the miniToc for a rest reference page.
export async function getRestOperationData(category, categoryOperations, context) {
const descriptions = {}
let toc = ''
const reusablePath = context.site.data.reusables
const subcategories = Object.keys(categoryOperations)
let introContent = null
for (const subcategory of subcategories) {
const markdown = get(reusablePath['rest-reference'][category], subcategory)
const renderedMarkdown = await renderContent(markdown, context)
descriptions[subcategory] = renderedMarkdown
// only a string with the raw HTML of each heading level 2 and 3
// is needed to generate the toc
const titles = categoryOperations[subcategory]
.map((operation) => `### ${operation.summary}\n`)
.join('')
toc += renderedMarkdown + (await renderContent(titles, context))
}
// This is Markdown content at the path
// data/reusables/rest-reference/<category>/<subcategory>
// that doesn't map directory to a group of operations.
if (get(reusablePath['rest-reference'][category], category) && !categoryOperations[category]) {
const markdown = get(reusablePath['rest-reference'][category], category)
introContent = await renderContent(markdown, context)
export async function getRestOperationData(category, language, version, context) {
if (!restOperationData.get(language).get(version).has(category)) {
const languageTree = restOperationData.get(language)
const descriptions = {}
let toc = ''
const reusablePath = context.site.data.reusables
const categoryOperations = await getRest(version, category)
const subcategories = Object.keys(categoryOperations)
let introContent = null
for (const subcategory of subcategories) {
const markdown = reusablePath['rest-reference']?.[category]?.[subcategory]
const renderedMarkdown = await renderContent(markdown, context)
descriptions[subcategory] = renderedMarkdown
// only a string with the raw HTML of each heading level 2 and 3
// is needed to generate the toc
const titles = categoryOperations[subcategory]
.map((operation) => `### ${operation.summary}\n`)
.join('')
toc += renderedMarkdown + (await renderContent(titles, context))
}
// Usually a Markdown file in
// data/resuables/rest-reference/<category>/<subcategory>
// will always map to a set of operations. But sometimes we have
// introductory text that doesn't map to any operations.
// When that is the case, the category and subcategory are the same.
// Example data/resuables/rest-reference/actions/actions
// The content in this file is called introContent and is displayed
// at the top of the rest reference page.
if (reusablePath['rest-reference']?.[category]?.[category] && !categoryOperations[category]) {
const markdown = reusablePath['rest-reference']?.[category]?.[category]
introContent = await renderContent(markdown, context)
}
const miniTocItems = getMiniTocItems(toc, 3)
languageTree.get(version).set(category, {
descriptions,
miniTocItems,
introContent,
})
restOperationData.set(restOperationData, languageTree)
}
const miniTocItems = getMiniTocItems(toc, 3)
return { descriptions, miniTocItems, introContent }
return restOperationData.get(language).get(version).get(category)
}

export async function getEnabledForApps() {
Expand Down
60 changes: 10 additions & 50 deletions pages/[versionId]/rest/reference/[category].tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,6 @@ import { RestCategoryOperationsT } from 'components/rest/types'
import { MiniTocItem } from 'components/context/ArticleContext'
import { RestReferencePage } from 'components/rest/RestReferencePage'

type RestOperationsT = {
[version: string]: {
[category: string]: RestCategoryOperationsT
}
}

type CategoryDataT = {
descriptions: {
[subcategory: string]: string
Expand All @@ -19,14 +13,6 @@ type CategoryDataT = {
introContent: string
}

type RestDataT = {
[language: string]: {
[version: string]: {
[category: string]: CategoryDataT
}
}
}

type Props = {
mainContext: MainContextT
restOperations: RestCategoryOperationsT
Expand All @@ -35,9 +21,6 @@ type Props = {
introContent: string
}

let rest: RestOperationsT | null = null
let restOperationData: RestDataT | null = null

export default function Category({
mainContext,
restOperations,
Expand Down Expand Up @@ -65,48 +48,25 @@ export const getServerSideProps: GetServerSideProps<Props> = async (context) =>
const currentVersion = context.params!.versionId as string
const currentLanguage = req.context.currentLanguage as string

// Use a local cache to store all of the REST operations, so
// we only read the directory of static/decorated files once
if (!rest) {
rest = (await getRest()) as RestOperationsT
}

/* This sets up a skeleton object in the format:
{
'en': { free-pro-team@latest: {}, enterprise-cloud@latest: {}},
'ja': { free-pro-team@latest: {}, enterprise-cloud@latest: {}}
}
*/
if (!restOperationData) {
restOperationData = {}
Object.keys(req.context.languages).forEach((language) => {
restOperationData![language] = {}
Object.keys(req.context.allVersions).forEach(
(version) => (restOperationData![language][version] = {})
)
})
}

const restOperations = rest[currentVersion][category]
const restOperations = await getRest(currentVersion, category)

// The context passed will have the Markdown content for the language
// of the page being requested and the Markdown will be rendered
// using the `currentVersion`
if (!(category in restOperationData[currentLanguage][currentVersion])) {
restOperationData[currentLanguage][currentVersion][category] = (await getRestOperationData(
category,
restOperations,
req.context
)) as CategoryDataT
}
const { descriptions, miniTocItems, introContent } = (await getRestOperationData(
category,
currentLanguage,
currentVersion,
req.context
)) as CategoryDataT

return {
props: {
restOperations,
mainContext: getMainContext(req, res),
descriptions: restOperationData[currentLanguage][currentVersion][category].descriptions,
miniTocItems: restOperationData[currentLanguage][currentVersion][category].miniTocItems,
introContent: restOperationData[currentLanguage][currentVersion][category].introContent,
descriptions,
miniTocItems,
introContent,
},
}
}
52 changes: 2 additions & 50 deletions tests/rendering/rest.js
View file
Original file line number Diff line number Diff line change
@@ -1,62 +1,19 @@
import { fileURLToPath } from 'url'
import path from 'path'
import fs from 'fs/promises'
import { difference } from 'lodash-es'
import { getDOM } from '../helpers/supertest.js'
import getRest, { getEnabledForApps } from '../../lib/rest/index.js'
import { jest } from '@jest/globals'
import { allVersions } from '../../lib/all-versions.js'

const __dirname = path.dirname(fileURLToPath(import.meta.url))
// list of REST markdown files that do not correspond to REST API resources
// TODO could we get this list dynamically, say via page frontmatter?
const excludeFromResourceNameCheck = [
'endpoints-available-for-github-apps.md',
'permissions-required-for-github-apps.md',
'index.md',
]

describe('REST references docs', () => {
jest.setTimeout(3 * 60 * 1000)

let operations

beforeAll(async () => {
operations = await getRest()
})

test('markdown file exists for every operationId prefix in the api.github.com schema', async () => {
const categories = [
...new Set(
Object.values(operations)
.map((version) => Object.keys(version))
.flat()
),
]
const referenceDir = path.join(__dirname, '../../content/rest/reference')
const filenames = (await fs.readdir(referenceDir))
.filter(
(filename) =>
!excludeFromResourceNameCheck.find((excludedFile) => filename.endsWith(excludedFile))
)
.map((filename) => filename.replace('.md', ''))

const missingResource =
'Found a markdown file in content/rest/reference that is not represented by an OpenAPI REST operation category.'
expect(difference(filenames, categories), missingResource).toEqual([])

const missingFile =
'Found an OpenAPI REST operation category that is not represented by a markdown file in content/rest/reference.'
expect(difference(categories, filenames), missingFile).toEqual([])
})

test('loads schema data for all versions', async () => {
for (const version in allVersions) {
const checksRestOperations = getRest(version, 'checks')
const $ = await getDOM(`/en/${version}/rest/reference/checks`)
const domH3Ids = $('h3')
.map((i, h3) => $(h3).attr('id'))
.get()
const schemaSlugs = Object.values(operations[version].checks)
const schemaSlugs = Object.values(checksRestOperations)
.flat()
.map((operation) => operation.slug)
expect(schemaSlugs.every((slug) => domH3Ids.includes(slug))).toBe(true)
Expand All @@ -80,9 +37,4 @@ describe('REST references docs', () => {
expect(schemaSlugs.every((slug) => domH3Ids.includes(slug))).toBe(true)
}
})

test('no wrongly detected AppleScript syntax highlighting in schema data', async () => {
const operations = await getRest()
expect(JSON.stringify(operations).includes('hljs language-applescript')).toBe(false)
})
})
Loading

0 comments on commit 67360f5

Please sign in to comment.