Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Avoid downloading avtars everytime on docs:dev #4603

Merged
merged 5 commits into from
Jul 6, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions cSpell.json
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,8 @@
"mult",
"neurodiverse",
"nextra",
"nikolay",
"nirname",
"orlandoni",
"pathe",
"pbrolin",
Expand All @@ -102,9 +104,11 @@
"ranksep",
"rect",
"rects",
"reda",
"redmine",
"rehype",
"roledescription",
"rozhkov",
"sandboxed",
"sankey",
"setupgraphviewbox",
Expand All @@ -121,6 +125,7 @@
"stylis",
"subhash-halder",
"substate",
"sulais",
"sveidqvist",
"swimm",
"techn",
Expand All @@ -144,6 +149,7 @@
"vueuse",
"xlink",
"yash",
"yokozuna",
"zenuml"
],
"patterns": [
Expand Down
2 changes: 1 addition & 1 deletion packages/mermaid/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
"docs:code": "typedoc src/defaultConfig.ts src/config.ts src/mermaidAPI.ts && prettier --write ./src/docs/config/setup",
"docs:build": "rimraf ../../docs && pnpm docs:spellcheck && pnpm docs:code && ts-node-esm src/docs.mts",
"docs:verify": "pnpm docs:spellcheck && pnpm docs:code && ts-node-esm src/docs.mts --verify",
"docs:pre:vitepress": "rimraf src/vitepress && pnpm docs:code && ts-node-esm src/docs.mts --vitepress && pnpm --filter ./src/vitepress install --no-frozen-lockfile --ignore-scripts ",
"docs:pre:vitepress": "pnpm --filter ./src/docs prefetch && rimraf src/vitepress && pnpm docs:code && ts-node-esm src/docs.mts --vitepress && pnpm --filter ./src/vitepress install --no-frozen-lockfile --ignore-scripts",
"docs:build:vitepress": "pnpm docs:pre:vitepress && (cd src/vitepress && pnpm run build) && cpy --flat src/docs/landing/ ./src/vitepress/.vitepress/dist/landing",
"docs:dev": "pnpm docs:pre:vitepress && concurrently \"pnpm --filter ./src/vitepress dev\" \"ts-node-esm src/docs.mts --watch --vitepress\"",
"docs:dev:docker": "pnpm docs:pre:vitepress && concurrently \"pnpm --filter ./src/vitepress dev:docker\" \"ts-node-esm src/docs.mts --watch --vitepress\"",
Expand Down
16 changes: 8 additions & 8 deletions packages/mermaid/src/docs.mts
Original file line number Diff line number Diff line change
Expand Up @@ -481,15 +481,15 @@ const transformHtml = (filename: string) => {
};

const getGlobs = (globs: string[]): string[] => {
globs.push(
'!**/dist/**',
'!**/redirect.spec.ts',
'!**/landing/**',
'!**/node_modules/**',
'!**/user-avatars/**'
);
globs.push('!**/dist/**', '!**/redirect.spec.ts', '!**/landing/**', '!**/node_modules/**');
if (!vitepress) {
globs.push('!**/.vitepress/**', '!**/vite.config.ts', '!src/docs/index.md', '!**/package.json');
globs.push(
'!**/.vitepress/**',
'!**/vite.config.ts',
'!src/docs/index.md',
'!**/package.json',
'!**/user-avatars/**'
);
}
return globs;
};
Expand Down
113 changes: 2 additions & 111 deletions packages/mermaid/src/docs/.vitepress/contributors.ts
Original file line number Diff line number Diff line change
@@ -1,30 +1,5 @@
import contributorUsernamesJson from './contributor-names.json';

export interface Contributor {
name: string;
avatar: string;
}

export interface SocialEntry {
icon: string | { svg: string };
link: string;
}

export interface CoreTeam {
name: string;
// required to download avatars from GitHub
github: string;
avatar?: string;
twitter?: string;
mastodon?: string;
sponsor?: string;
website?: string;
linkedIn?: string;
title?: string;
org?: string;
desc?: string;
links?: SocialEntry[];
}
import { CoreTeam, knut, plainTeamMembers } from './teamMembers.js';

const contributorUsernames: string[] = contributorUsernamesJson;

Expand All @@ -38,6 +13,7 @@ const websiteSVG = {

const createLinks = (tm: CoreTeam): CoreTeam => {
tm.avatar = `/user-avatars/${tm.github}.png`;
tm.title = tm.title ?? 'Developer';
tm.links = [{ icon: 'github', link: `https://github.com/${tm.github}` }];
if (tm.mastodon) {
tm.links.push({ icon: 'mastodon', link: tm.mastodon });
Expand All @@ -54,91 +30,6 @@ const createLinks = (tm: CoreTeam): CoreTeam => {
return tm;
};

const knut: CoreTeam = {
github: 'knsv',
name: 'Knut Sveidqvist',
title: 'Creator',
twitter: 'knutsveidqvist',
sponsor: 'https://github.com/sponsors/knsv',
};

const plainTeamMembers: CoreTeam[] = [
{
github: 'NeilCuzon',
name: 'Neil Cuzon',
title: 'Developer',
},
{
github: 'tylerlong',
name: 'Tyler Liu',
title: 'Developer',
},
{
github: 'sidharthv96',
name: 'Sidharth Vinod',
title: 'Developer',
twitter: 'sidv42',
mastodon: 'https://techhub.social/@sidv',
sponsor: 'https://github.com/sponsors/sidharthv96',
linkedIn: 'sidharth-vinod',
website: 'https://sidharth.dev',
},
{
github: 'ashishjain0512',
name: 'Ashish Jain',
title: 'Developer',
},
{
github: 'mmorel-35',
name: 'Matthieu Morel',
title: 'Developer',
linkedIn: 'matthieumorel35',
},
{
github: 'aloisklink',
name: 'Alois Klink',
title: 'Developer',
linkedIn: 'aloisklink',
website: 'https://aloisklink.com',
},
{
github: 'pbrolin47',
name: 'Per Brolin',
title: 'Developer',
},
{
github: 'Yash-Singh1',
name: 'Yash Singh',
title: 'Developer',
},
{
github: 'GDFaber',
name: 'Marc Faber',
title: 'Developer',
linkedIn: 'marc-faber',
},
{
github: 'MindaugasLaganeckas',
name: 'Mindaugas Laganeckas',
title: 'Developer',
},
{
github: 'jgreywolf',
name: 'Justin Greywolf',
title: 'Developer',
},
{
github: 'IOrlandoni',
name: 'Nacho Orlandoni',
title: 'Developer',
},
{
github: 'huynhicode',
name: 'Steph Huynh',
title: 'Developer',
},
];

const teamMembers = plainTeamMembers.map((tm) => createLinks(tm));
teamMembers.sort(
(a, b) => contributorUsernames.indexOf(a.github) - contributorUsernames.indexOf(b.github)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@ async function download(url: string, fileName: URL) {
await writeFile(fileName, Buffer.from(await image.arrayBuffer()));
} catch (error) {
console.error('failed to load', url, error);
// Exit the build process if we are in CI
if (process.env.CI) {
throw error;
}
}
}

Expand All @@ -32,7 +36,7 @@ async function fetchAvatars() {
download(`https://github.com/${name}.png?size=100`, getAvatarPath(name));
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should probably modify the download() function so that it's doesn't run fully in parallel. Every time I try to download the avatars, GitHub seems to block my computer from accessing GitHub for a few minutes 😆

We should also set the appropriate HTTP headers for GitHub's API.

We could also install a JavaScript client to download the avatar, which should handle rate-limiting, headers, auth, etc. all for us. The only problem is that there doesn't seem to be a REST API JavaScript client that can download the avatar. Only the GraphQL API has an avatarUrl field: https://docs.github.com/en/graphql/reference/objects#user

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It was, I was tired of waiting and replaced it with parallel version. We need either to satisfy GH rate limit, or may be refuse from downloading avatars in local dev at all, or limit them like to 10 only, that would be enough for debugging and visual purposes.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

At least in CI, there's a GITHUB_TOKEN environment variable that can be used with the API to heavily reduce rate limits. :)

I'd be 👍 to just downloading the first 10 pictures, then replacing all the other avatars with a dummy picture, like
image

});

await Promise.all(avatars);
await Promise.allSettled(avatars);
}

fetchAvatars();
sidharthv96 marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
// Adapted from https://github.dev/vitest-dev/vitest/blob/991ff33ab717caee85ef6cbe1c16dc514186b4cc/scripts/update-contributors.ts#L6

import { writeFile } from 'node:fs/promises';
import { knut, plainTeamMembers } from '../teamMembers.js';
import { existsSync } from 'node:fs';

const pathContributors = new URL('../contributor-names.json', import.meta.url);

Expand Down Expand Up @@ -35,7 +37,15 @@ async function fetchContributors() {
}

async function generate() {
const collaborators = await fetchContributors();
if (existsSync(pathContributors)) {
// Only fetch contributors once, when running locally.
// In CI, the file won't exist, so it'll fetch every time as expected.
return;
}
// Will fetch all contributors only in CI to speed up local development.
const collaborators = process.env.CI
? await fetchContributors()
: [knut, ...plainTeamMembers].map((m) => m.github);
await writeFile(pathContributors, JSON.stringify(collaborators, null, 2) + '\n', 'utf8');
}

Expand Down
105 changes: 105 additions & 0 deletions packages/mermaid/src/docs/.vitepress/teamMembers.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
export interface Contributor {
name: string;
avatar: string;
}

export interface SocialEntry {
icon: string | { svg: string };
link: string;
}

export interface CoreTeam {
name: string;
// required to download avatars from GitHub
github: string;
avatar?: string;
twitter?: string;
mastodon?: string;
sponsor?: string;
website?: string;
linkedIn?: string;
title?: string;
org?: string;
desc?: string;
links?: SocialEntry[];
}

export const knut: CoreTeam = {
github: 'knsv',
name: 'Knut Sveidqvist',
title: 'Creator',
twitter: 'knutsveidqvist',
sponsor: 'https://github.com/sponsors/knsv',
};

export const plainTeamMembers: CoreTeam[] = [
{
github: 'NeilCuzon',
name: 'Neil Cuzon',
},
{
github: 'tylerlong',
name: 'Tyler Liu',
},
{
github: 'sidharthv96',
name: 'Sidharth Vinod',
twitter: 'sidv42',
mastodon: 'https://techhub.social/@sidv',
sponsor: 'https://github.com/sponsors/sidharthv96',
linkedIn: 'sidharth-vinod',
website: 'https://sidharth.dev',
},
{
github: 'ashishjain0512',
name: 'Ashish Jain',
},
{
github: 'mmorel-35',
name: 'Matthieu Morel',
linkedIn: 'matthieumorel35',
},
{
github: 'aloisklink',
name: 'Alois Klink',
linkedIn: 'aloisklink',
website: 'https://aloisklink.com',
},
{
github: 'pbrolin47',
name: 'Per Brolin',
},
{
github: 'Yash-Singh1',
name: 'Yash Singh',
},
{
github: 'GDFaber',
name: 'Marc Faber',
linkedIn: 'marc-faber',
},
{
github: 'MindaugasLaganeckas',
name: 'Mindaugas Laganeckas',
},
{
github: 'jgreywolf',
name: 'Justin Greywolf',
},
{
github: 'IOrlandoni',
name: 'Nacho Orlandoni',
},
{
github: 'huynhicode',
name: 'Steph Huynh',
},
{
github: 'Yokozuna59',
name: 'Reda Al Sulais',
},
{
github: 'nirname',
name: 'Nikolay Rozhkov',
},
];