feat: preparation for v11 & gh api rate limiting (npm#1402)

Hawthorne001 · Dec 17, 2024 · 4a49edb · 4a49edb
1 parent 110d75c
commit 4a49edb
Show file tree

Hide file tree

Showing 101 changed files with 1,202 additions and 190 deletions.
diff --git a/.gitignore b/.gitignore
@@ -40,5 +40,6 @@
 !/test/
 !/tsconfig.json
 !/webpack.config.js
+!cli-cache.json
 tap-testdir*/
 !/cli/
diff --git a/.prettierignore b/.prettierignore
@@ -5,3 +5,4 @@
 /.reuse/
 .nyc_output/
 coverage/
+cli-cache.json
diff --git a/cli-cache.json b/cli-cache.json
@@ -0,0 +1,6 @@
+{
+  "v11": "cf52b8be2645ee098ee83ea9981fc32a11932fad",
+  "v9": "64763a341e7aa5b456e696f956759bf9b3440dc1",
+  "v10": "a3041941586b6fb8ed7403fe3c24d81138a96005",
+  "v8": "aa8fff11cdab94fff1a2160ee5241f5f4632e96b"
+}
diff --git a/cli/bin/build.js b/cli/bin/build.js
@@ -2,6 +2,7 @@ const {resolve, relative, join} = require('path')
 const {spawnSync} = require('child_process')
 const build = require('../lib/build.js')
 const {nwo} = require('../lib/gh')
+const {CacheVersionSha} = require('../lib/cache.js')
 
 // check only build with the current versions instead of checking the registry
 // and also fails if any changes are detected. this is used in CI to make sure
@@ -24,21 +25,23 @@ const checkContent = () => {
   }
 }
 
-build({
-  releases: require('../releases.json'),
-  loglevel: process.argv.includes('--debug') || process.env.CI ? 'verbose' : 'info',
-  prerelease: false,
-  useCurrent: checkOnly,
-  contentPath,
-  navPath,
-})
-  .then(() => {
+;(async () => {
+  try {
+    await build({
+      cache: await CacheVersionSha.load(join(ROOT, 'cli-cache.json')),
+      releases: require('../releases.json'),
+      loglevel: process.argv.includes('--debug') || process.env.CI ? 'verbose' : 'info',
+      prerelease: false,
+      useCurrent: checkOnly,
+      contentPath,
+      navPath,
+    })
     if (checkOnly) {
       checkContent()
     }
     return console.log('DONE')
-  })
-  .catch(e => {
+  } catch (e) {
     console.error(e)
     process.exit(1)
-  })
+  }
+})()
diff --git a/cli/lib/build.js b/cli/lib/build.js
@@ -55,7 +55,7 @@ const getCurrentVersions = nav => {
   }
 }
 
-const main = async ({loglevel, releases: rawReleases, useCurrent, navPath, contentPath, prerelease}) => {
+const main = async ({loglevel, releases: rawReleases, useCurrent, navPath, contentPath, prerelease, cache}) => {
   /* istanbul ignore next */
   if (loglevel) {
     log.on(loglevel)
@@ -114,9 +114,11 @@ const main = async ({loglevel, releases: rawReleases, useCurrent, navPath, conte
   })
 
   const updates = await Promise.all(
-    releases.map(r => extractRelease(r, {contentPath, baseNav: navData, prerelease})),
+    releases.map(r => extractRelease(r, {cache, contentPath, baseNav: navData, prerelease})),
   ).then(r => r.filter(Boolean))
 
+  await cache?.save()
+
   await updateNav(updates, {nav: navDoc, path: navPath})
 }
 

diff --git a/cli/lib/cache.js b/cli/lib/cache.js
@@ -0,0 +1,31 @@
+const fs = require('fs/promises')
+
+/** cache npm cli version shas to NOT pull down changes we already have */
+class CacheVersionSha {
+  constructor(cache, path) {
+    this.cache = cache
+    this.path = path
+  }
+
+  static async load(path) {
+    return new CacheVersionSha(JSON.parse(await fs.readFile(path, 'utf-8')), path)
+  }
+
+  async save() {
+    await fs.writeFile(this.path, JSON.stringify(this.cache, null, 2))
+    return this
+  }
+
+  set(id, sha) {
+    this.cache[id] = sha
+    return this
+  }
+
+  same(id, value) {
+    return this.cache[id] === value
+  }
+}
+
+module.exports = {
+  CacheVersionSha,
+}
diff --git a/cli/lib/extract.js b/cli/lib/extract.js
@@ -138,7 +138,16 @@ const writeChangelog = async ({release, nav, cwd, srcPath, contentPath}) => {
   })
 }
 
-const unpackRelease = async (release, {contentPath, baseNav, prerelease = false}) => {
+const unpackRelease = async (release, {cache, contentPath, baseNav, prerelease = false}) => {
+  if (cache) {
+    const sha = await gh.getCurrentSha(release.branch)
+    if (cache.same(release.id, sha)) {
+      log.info(`Skipping ${release.id} due to cache`)
+      return
+    }
+    cache.set(release.id, sha)
+  }
+
   if (release.prerelease && !prerelease) {
     log.info(`Skipping ${release.id} due to prerelease ${release.version}`)
     return

diff --git a/cli/lib/gh.js b/cli/lib/gh.js
@@ -1,14 +1,32 @@
 const {posix, sep} = require('node:path')
+const {execSync} = require('node:child_process')
 
 if (!process.env.GITHUB_TOKEN) {
-  throw new Error('GITHUB_TOKEN env var is required to build CLI docs')
+  try {
+    // this allows people to run this locally
+    process.env.GITHUB_TOKEN = execSync('gh auth token', {encoding: 'utf8'}).trim()
+  } catch (err) {
+    throw new Error('GITHUB_TOKEN env var is required to build CLI docs')
+  }
 }
 
 let octokit
 const owner = 'npm'
 const repo = 'cli'
 const opts = {owner, repo}
 
+const getCurrentSha = async branch => {
+  if (!octokit) {
+    const {Octokit} = await import('@octokit/rest')
+    octokit = new Octokit({auth: process.env.GITHUB_TOKEN})
+  }
+  const {data} = await octokit.repos.getBranch({
+    ...opts,
+    branch,
+  })
+  return data.commit.sha
+}
+
 const getFile = async ({sha, ref, path}) => {
   if (!octokit) {
     const {Octokit} = await import('@octokit/rest')
@@ -51,5 +69,6 @@ const pathExists = async (ref, path) => {
 module.exports = {
   getFile,
   pathExists,
+  getCurrentSha,
   nwo: `${owner}/${repo}`,
 }
diff --git a/cli/package.json b/cli/package.json
@@ -40,7 +40,7 @@
   },
   "templateOSS": {
     "//@npmcli/template-oss": "This file is partially managed by @npmcli/template-oss. Edits may be overwritten.",
-    "version": "4.23.6",
+    "version": "4.23.5",
     "content": "./scripts/template-oss"
   },
   "files": [

diff --git a/cli/releases.json b/cli/releases.json
@@ -1,12 +1,4 @@
 [
-  {
-    "id": "v6",
-    "branch": "release/v6"
-  },
-  {
-    "id": "v7",
-    "branch": "release/v7"
-  },
   {
     "id": "v8",
     "branch": "release/v8"
@@ -17,6 +9,10 @@
   },
   {
     "id": "v10",
+    "branch": "release/v10"
+  },
+  {
+    "id": "v11",
     "branch": "latest"
   }
 ]
diff --git a/cli/test/index.js b/cli/test/index.js
@@ -66,6 +66,7 @@ const mockBuild = async (t, {releases = getReleases(), packument = {}, testdir:
     return yaml.stringify(children).replace(new RegExp(`/cli/${id}/`, 'g'), '/')
   }
 
+  let shaCounter = 0
   const build = t.mockRequire('../lib/build', {
     pacote: {
       ...pacote,
@@ -83,6 +84,10 @@ const mockBuild = async (t, {releases = getReleases(), packument = {}, testdir:
     },
     '@prettier/sync': {format: s => s},
     '../lib/gh.js': {
+      getCurrentSha: async () => {
+        shaCounter = shaCounter + 1
+        return 'abc' + shaCounter
+      },
       getFile: async ({ref}) => navSection(ref),
       pathExists: async (ref, p) => {
         if (ref.includes('v6') && p.includes('docs/lib/content')) {

diff --git a/content/cli/v10/commands/index.mdx b/content/cli/v10/commands/index.mdx
@@ -2,7 +2,7 @@
 title: CLI Commands
 shortName: Commands
 github_repo: npm/cli
-github_branch: latest
+github_branch: release/v10
 github_path: docs/lib/content/nav.yml
 redirect_from:
   - /cli-commands

diff --git a/content/cli/v10/commands/npm-access.mdx b/content/cli/v10/commands/npm-access.mdx
@@ -3,7 +3,7 @@ title: npm-access
 section: 1
 description: Set access level on published packages
 github_repo: npm/cli
-github_branch: latest
+github_branch: release/v10
 github_path: docs/lib/content/commands/npm-access.md
 redirect_from:
   - /cli-commands/access

diff --git a/content/cli/v10/commands/npm-adduser.mdx b/content/cli/v10/commands/npm-adduser.mdx
@@ -3,7 +3,7 @@ title: npm-adduser
 section: 1
 description: Add a registry user account
 github_repo: npm/cli
-github_branch: latest
+github_branch: release/v10
 github_path: docs/lib/content/commands/npm-adduser.md
 redirect_from:
   - /cli-commands/adduser

diff --git a/content/cli/v10/commands/npm-audit.mdx b/content/cli/v10/commands/npm-audit.mdx
@@ -3,7 +3,7 @@ title: npm-audit
 section: 1
 description: Run a security audit
 github_repo: npm/cli
-github_branch: latest
+github_branch: release/v10
 github_path: docs/lib/content/commands/npm-audit.md
 redirect_from:
   - /cli-commands/audit

diff --git a/content/cli/v10/commands/npm-bugs.mdx b/content/cli/v10/commands/npm-bugs.mdx
@@ -3,7 +3,7 @@ title: npm-bugs
 section: 1
 description: Report bugs for a package in a web browser
 github_repo: npm/cli
-github_branch: latest
+github_branch: release/v10
 github_path: docs/lib/content/commands/npm-bugs.md
 redirect_from:
   - /cli-commands/bugs

diff --git a/content/cli/v10/commands/npm-cache.mdx b/content/cli/v10/commands/npm-cache.mdx
@@ -3,7 +3,7 @@ title: npm-cache
 section: 1
 description: Manipulates packages cache
 github_repo: npm/cli
-github_branch: latest
+github_branch: release/v10
 github_path: docs/lib/content/commands/npm-cache.md
 redirect_from:
   - /cli-commands/cache

diff --git a/content/cli/v10/commands/npm-ci.mdx b/content/cli/v10/commands/npm-ci.mdx
@@ -3,7 +3,7 @@ title: npm-ci
 section: 1
 description: Clean install a project
 github_repo: npm/cli
-github_branch: latest
+github_branch: release/v10
 github_path: docs/lib/content/commands/npm-ci.md
 redirect_from:
   - /cli-commands/ci

diff --git a/content/cli/v10/commands/npm-completion.mdx b/content/cli/v10/commands/npm-completion.mdx
@@ -3,7 +3,7 @@ title: npm-completion
 section: 1
 description: Tab Completion for npm
 github_repo: npm/cli
-github_branch: latest
+github_branch: release/v10
 github_path: docs/lib/content/commands/npm-completion.md
 redirect_from:
   - /cli-commands/completion

diff --git a/content/cli/v10/commands/npm-config.mdx b/content/cli/v10/commands/npm-config.mdx
@@ -3,7 +3,7 @@ title: npm-config
 section: 1
 description: Manage the npm configuration files
 github_repo: npm/cli
-github_branch: latest
+github_branch: release/v10
 github_path: docs/lib/content/commands/npm-config.md
 redirect_from:
   - /cli-commands/config

diff --git a/content/cli/v10/commands/npm-dedupe.mdx b/content/cli/v10/commands/npm-dedupe.mdx
@@ -3,7 +3,7 @@ title: npm-dedupe
 section: 1
 description: Reduce duplication in the package tree
 github_repo: npm/cli
-github_branch: latest
+github_branch: release/v10
 github_path: docs/lib/content/commands/npm-dedupe.md
 redirect_from:
   - /cli-commands/dedupe

diff --git a/content/cli/v10/commands/npm-deprecate.mdx b/content/cli/v10/commands/npm-deprecate.mdx
@@ -3,7 +3,7 @@ title: npm-deprecate
 section: 1
 description: Deprecate a version of a package
 github_repo: npm/cli
-github_branch: latest
+github_branch: release/v10
 github_path: docs/lib/content/commands/npm-deprecate.md
 redirect_from:
   - /cli-commands/deprecate

diff --git a/content/cli/v10/commands/npm-diff.mdx b/content/cli/v10/commands/npm-diff.mdx
@@ -3,7 +3,7 @@ title: npm-diff
 section: 1
 description: The registry diff command
 github_repo: npm/cli
-github_branch: latest
+github_branch: release/v10
 github_path: docs/lib/content/commands/npm-diff.md
 redirect_from:
   - /cli-commands/diff

diff --git a/content/cli/v10/commands/npm-dist-tag.mdx b/content/cli/v10/commands/npm-dist-tag.mdx
@@ -3,7 +3,7 @@ title: npm-dist-tag
 section: 1
 description: Modify package distribution tags
 github_repo: npm/cli
-github_branch: latest
+github_branch: release/v10
 github_path: docs/lib/content/commands/npm-dist-tag.md
 redirect_from:
   - /cli-commands/dist-tag

diff --git a/content/cli/v10/commands/npm-docs.mdx b/content/cli/v10/commands/npm-docs.mdx
@@ -3,7 +3,7 @@ title: npm-docs
 section: 1
 description: Open documentation for a package in a web browser
 github_repo: npm/cli
-github_branch: latest
+github_branch: release/v10
 github_path: docs/lib/content/commands/npm-docs.md
 redirect_from:
   - /cli-commands/docs

diff --git a/content/cli/v10/commands/npm-doctor.mdx b/content/cli/v10/commands/npm-doctor.mdx
@@ -3,7 +3,7 @@ title: npm-doctor
 section: 1
 description: Check the health of your npm environment
 github_repo: npm/cli
-github_branch: latest
+github_branch: release/v10
 github_path: docs/lib/content/commands/npm-doctor.md
 redirect_from:
   - /cli-commands/doctor

diff --git a/content/cli/v10/commands/npm-edit.mdx b/content/cli/v10/commands/npm-edit.mdx
@@ -3,7 +3,7 @@ title: npm-edit
 section: 1
 description: Edit an installed package
 github_repo: npm/cli
-github_branch: latest
+github_branch: release/v10
 github_path: docs/lib/content/commands/npm-edit.md
 redirect_from:
   - /cli-commands/edit

diff --git a/content/cli/v10/commands/npm-exec.mdx b/content/cli/v10/commands/npm-exec.mdx
@@ -3,7 +3,7 @@ title: npm-exec
 section: 1
 description: Run a command from a local or remote npm package
 github_repo: npm/cli
-github_branch: latest
+github_branch: release/v10
 github_path: docs/lib/content/commands/npm-exec.md
 redirect_from:
   - /cli-commands/exec

diff --git a/content/cli/v10/commands/npm-explain.mdx b/content/cli/v10/commands/npm-explain.mdx
@@ -3,7 +3,7 @@ title: npm-explain
 section: 1
 description: Explain installed packages
 github_repo: npm/cli
-github_branch: latest
+github_branch: release/v10
 github_path: docs/lib/content/commands/npm-explain.md
 redirect_from:
   - /cli-commands/explain

diff --git a/content/cli/v10/commands/npm-explore.mdx b/content/cli/v10/commands/npm-explore.mdx
@@ -3,7 +3,7 @@ title: npm-explore
 section: 1
 description: Browse an installed package
 github_repo: npm/cli
-github_branch: latest
+github_branch: release/v10
 github_path: docs/lib/content/commands/npm-explore.md
 redirect_from:
   - /cli-commands/explore