[docs] Use custom webpack loader for markdown

.eslintrc.js

@@ -229,6 +229,9 @@ module.exports = {
  {
  files: ['docs/pages/**/*.js'],
  rules: {
+ // The code is already coupled to webpack.
+ 'import/no-webpack-loader-syntax': 'off',
+
  'react/prop-types': 'off',
  },
  },

docs/next.config.js

@@ -191,7 +191,8 @@ module.exports = {
 
  // We want to speed-up the build of pull requests.
  // For crowdin PRs we want to build all locales for testing.
- if (process.env.PULL_REQUEST === 'true' && !l10nPRInNetlify && !vercelDeploy) {
+ // FIXME: Revert before merging
+ if (process.env.PULL_REQUEST === 'false' && !l10nPRInNetlify && !vercelDeploy) {
  // eslint-disable-next-line no-console
  console.log('Considering only English for SSR');
  traverse(pages, 'en');

docs/package.json

@@ -85,13 +85,11 @@
  "lodash": "^4.17.15",
  "lz-string": "^1.4.4",
  "markdown-to-jsx": "^7.0.0",
- "marked": "^2.0.0",
  "material-ui-popup-state": "^1.4.1",
  "next": "^10.0.0",
  "notistack": "^1.0.0",
  "nprogress": "^0.2.0",
  "postcss": "^8.0.6",
- "prismjs": "^1.17.1",
  "prop-types": "^15.7.2",
  "raw-loader": "^1.0.0",
  "react": "^17.0.1",

docs/packages/markdown/loader.js

@@ -0,0 +1,127 @@
+const { promises: fs, readFileSync } = require('fs');
+const path = require('path');
+const { notEnglishMarkdownRegExp, prepareMarkdown } = require('./parseMarkdown');
+
+/**
+ * @param {string} string
+ */
+function upperCaseFirst(string) {
+ return `${string[0].toUpperCase()}${string.slice(1)}`;
+}
+
+/**
+ * @param {string} key
+ * @example keyToJSIdentifier('index.md') === 'IndexMd'
+ * @example keyToJSIdentifier('index-ja.md') === 'IndexJaMd'
+ */
+function keyToJSIdentifier(key) {
+ const delimiter = /(\.|-)/;
+ return key
+ .split(delimiter)
+ .filter((part) => !delimiter.test(part))
+ .map(upperCaseFirst)
+ .join('');
+}
+
+/**
+ * @example findTranslatedVersions('/a/b/index.md') === ['index-en.md', 'index-ja.md', 'index.md']
+ * @param {string} englishFilepath An absolute path to the english version written in markdown (.md)
+ */
+async function findTranslatedVersions(englishFilepath) {
+ const filename = path.basename(englishFilepath, '.md');
+ const files = await fs.readdir(path.dirname(englishFilepath));
+
+ // Given: index.md
+ // Match: index-en.md, index-ja.md
+ // Don't Match: otherindex-en.md, index-eng.md, index-e.md, index.tsx
+ const translatedVersionRegExp = new RegExp(`^${filename}${notEnglishMarkdownRegExp.source}`);
+
+ return files
+ .filter((filepath) => {
+ return translatedVersionRegExp.test(filepath);
+ })
+ .concat(path.basename(englishFilepath));
+}
+
+/**
+ * @type {import('webpack').loader.Loader}
+ */
+module.exports = async function demoLoader() {
+ const rawKeys = await findTranslatedVersions(this.resourcePath);
+
+ // TODO: Remove requireRaw mock (needs work in prepareMarkdown)
+ const requireRaw = (key) => {
+ const filepath = path.join(path.dirname(this.resourcePath), key);
+ this.addDependency(filepath);
+ return readFileSync(filepath, { encoding: 'utf-8' });
+ };
+ requireRaw.keys = () => rawKeys;
+ const pageFilename = this.context.replace(this.rootContext, '').replace(/^\/src\/pages\//, '');
+ const { docs } = prepareMarkdown({ pageFilename, requireRaw });
+
+ const demos = {};
+ const demoKeys = [];
+ new Set(
+ docs.en.rendered
+ .filter((markdownOrComponentConfig) => {
+ return typeof markdownOrComponentConfig !== 'string' && markdownOrComponentConfig.demo;
+ })
+ .map((demoConfig) => {
+ return path.basename(demoConfig.demo);
+ }),
+ ).forEach((filename) => {
+ const demoName = `pages/${pageFilename}/${filename
+ .replace(/\.\//g, '')
+ .replace(/\.tsx/g, '.js')}`;
+
+ demos[demoName] = {
+ module: filename,
+ raw: requireRaw(filename),
+ };
+ demoKeys.push(filename);
+
+ try {
+ const moduleTS = filename.replace(/\.js$/, '.tsx');
+ const rawTS = requireRaw(moduleTS);
+
+ demos[demoName].moduleTS = moduleTS;
+ demos[demoName].rawTS = rawTS;
+ demoKeys.push(moduleTS);
+ } catch (error) {
+ // TS version of the demo doesn't exist. This is fine.
+ }
+ });
+
+ /**
+ * @param {string} key
+ */
+ function getRequireDemoIdentifier(key) {
+ return keyToJSIdentifier(key);
+ }
+
+ const transformed = `
+ ${demoKeys
+ .map((key) => {
+ return `import ${getRequireDemoIdentifier(key)} from './${key}';`;
+ })
+ .join('\n')}
+
+ export const docs = ${JSON.stringify(docs, null, 2)};
+ export const demos = ${JSON.stringify(demos, null, 2)};
+ export function requireDemo(module) {
+ return {
+ ${demoKeys
+ .map((key) => {
+ // TODO: Remove ES module interop once all demos are loaded via loader
+ // i.e. replace `{ default: ... }` with `...`
+ return `'${key}': { default: ${getRequireDemoIdentifier(key)} }`;
+ })
+ .join(',\n')}
+ }[module];
+ }
+ requireDemo.keys = () => {
+ return ${JSON.stringify(demoKeys, null, 2)}
+ }`;
+
+ return transformed;
+};

docs/packages/markdown/package.json

@@ -0,0 +1,17 @@
+{
+ "name": "@material-ui/markdown",
+ "version": "0.1.0",
+ "private": true,
+ "type": "commonjs",
+ "main": "./parseMarkdown.js",
+ "exports": {
+ ".": "./parseMarkdown.js",
+ "./loader": "./loader.js",
+ "./prism": "./prism.js"
+ },
+ "dependencies": {
+ "lodash": "^4.17.15",
+ "marked": "^2.0.0",
+ "prismjs": "^1.17.1"
+ }
+}

docs/src/modules/utils/parseMarkdown.js → docs/packages/markdown/parseMarkdown.js

@@ -1,8 +1,10 @@
-import marked from 'marked';
-import { LANGUAGES_IN_PROGRESS } from 'docs/src/modules/constants';
-import kebabCase from 'lodash/kebabCase';
-import textToHash from 'docs/src/modules/utils/textToHash';
-import prism from 'docs/src/modules/utils/prism';
+const marked = require('marked');
+const kebabCase = require('lodash/kebabCase');
+const textToHash = require('./textToHash');
+const prism = require('./prism');
+
+// TODO: pass as argument
+const LANGUAGES_IN_PROGRESS = ['en', 'zh', 'ru', 'pt', 'es', 'fr', 'de', 'ja'];
 
 const headerRegExp = /---[\r\n]([\s\S]*)[\r\n]---/;
 const titleRegExp = /# (.*)[\r\n]/;
@@ -25,7 +27,7 @@ const notEnglishMarkdownRegExp = /-([a-z]{2})\.md$/;
  * should output:
  * { title: 'Backdrop React Component', components: ['Backdrop'] }
  */
-export function getHeaders(markdown) {
+function getHeaders(markdown) {
  let header = markdown.match(headerRegExp);
 
  if (!header) {
@@ -64,14 +66,14 @@ export function getHeaders(markdown) {
  return headers;
 }
 
-export function getContents(markdown) {
+function getContents(markdown) {
  return markdown
  .replace(headerRegExp, '') // Remove header information
  .split(/^{{("(?:demo|component)":[^}]*)}}$/gm) // Split markdown into an array, separating demos
  .filter((content) => !emptyRegExp.test(content)); // Remove empty lines
 }
 
-export function getTitle(markdown) {
+function getTitle(markdown) {
  const matches = markdown.match(titleRegExp);
 
  if (!matches || !matches[1]) {
@@ -81,16 +83,19 @@ export function getTitle(markdown) {
  return matches[1];
 }
 
-export function getDescription(markdown) {
+function getDescription(markdown) {
  const matches = markdown.match(descriptionRegExp);
+ if (matches === null) {
+ return undefined;
+ }
 
- return matches?.[1].trim();
+ return matches[1].trim();
 }
 
 /**
  * @param {string} markdown
  */
-export function renderInline(markdown) {
+function renderInline(markdown) {
  return marked.parseInline(markdown);
 }
 
@@ -120,7 +125,7 @@ const externs = [
  * @param {TableOfContentsEntry[]} context.toc - WILL BE MUTATED
  * @param {string} context.userLanguage
  */
-export function createRender(context) {
+function createRender(context) {
  const { headingHashes, toc, userLanguage } = context;
  const headingHashesFallbackTranslated = {};
  let headingIndex = -1;
@@ -238,10 +243,13 @@ export function createRender(context) {
  * @param {() => string} config.requireRaw - returnvalue of require.context
  * @param {string} config.pageFilename - filename relative to nextjs pages directory
  */
-export function prepareMarkdown(config) {
+function prepareMarkdown(config) {
  const { pageFilename, requireRaw } = config;
 
  const demos = {};
+ /**
+ * @type {Record<string, { rendered: Array<string | { component: string } | { demo:string }> }>}
+ */
  const docs = {};
  const headingHashes = {};
 
@@ -336,3 +344,14 @@ ${headers.components
 
  return { demos, docs };
 }
+
+module.exports = {
+ createRender,
+ notEnglishMarkdownRegExp,
+ getContents,
+ getDescription,
+ getHeaders,
+ getTitle,
+ prepareMarkdown,
+ renderInline,
+};

docs/src/modules/utils/prism.js → docs/packages/markdown/prism.js

@@ -1,13 +1,13 @@
-import prism from 'prismjs';
-import 'prismjs/components/prism-css';
-import 'prismjs/components/prism-diff';
-import 'prismjs/components/prism-javascript';
-import 'prismjs/components/prism-json';
-import 'prismjs/components/prism-jsx';
-import 'prismjs/components/prism-markup';
-import 'prismjs/components/prism-tsx';
+const prism = require('prismjs');
+require('prismjs/components/prism-css');
+require('prismjs/components/prism-diff');
+require('prismjs/components/prism-javascript');
+require('prismjs/components/prism-json');
+require('prismjs/components/prism-jsx');
+require('prismjs/components/prism-markup');
+require('prismjs/components/prism-tsx');
 
-export default function highlight(code, language) {
+function highlight(code, language) {
  let prismLanguage;
  switch (language) {
  case 'ts':
@@ -42,3 +42,5 @@ export default function highlight(code, language) {
 
  return prism.highlight(code, prismLanguage);
 }
+
+module.exports = highlight;

docs/src/modules/utils/textToHash.js → docs/packages/markdown/textToHash.js

@@ -14,7 +14,7 @@ function makeUnique(hash, unique, i = 1) {
  * @param {Record<string, boolean>} [unique] - Ensures that each output is unique in `unique`
  * @returns {string} that is safe to use in fragment links
  */
-export default function textToHash(text, unique = {}) {
+function textToHash(text, unique = {}) {
  return makeUnique(
  encodeURI(
  text
@@ -32,3 +32,5 @@ export default function textToHash(text, unique = {}) {
  unique,
  );
 }
+
+module.exports = textToHash;

docs/src/modules/utils/textToHash.test.js → docs/packages/markdown/textToHash.test.js

@@ -1,5 +1,5 @@
 import { expect } from 'chai';
-import { renderInline as renderInlineMarkdown } from './parseMarkdown';
+import { parseInline as renderInlineMarkdown } from 'marked';
 import textToHash from './textToHash';
 
 describe('textToHash', () => {

docs/pages/blog/2019-developer-survey-results.js

@@ -1,15 +1,7 @@
 import * as React from 'react';
 import TopLayoutBlog from 'docs/src/modules/components/TopLayoutBlog';
-import { prepareMarkdown } from 'docs/src/modules/utils/parseMarkdown';
+import { docs } from '!@material-ui/markdown/loader!./2019-developer-survey-results.md';
 
-const pageFilename = 'blog/2019-developer-survey-results';
-const requireRaw = require.context('!raw-loader!./', false, /2019-developer-survey-results\.md$/);
-
-export default function Page({ docs }) {
+export default function Page() {
  return <TopLayoutBlog docs={docs} />;
 }
-
-Page.getInitialProps = () => {
- const { demos, docs } = prepareMarkdown({ pageFilename, requireRaw });
- return { demos, docs };
-};

docs/pages/blog/2019.js

@@ -1,15 +1,7 @@
 import * as React from 'react';
 import TopLayoutBlog from 'docs/src/modules/components/TopLayoutBlog';
-import { prepareMarkdown } from 'docs/src/modules/utils/parseMarkdown';
+import { docs } from '!@material-ui/markdown/loader!./2019.md';
 
-const pageFilename = 'blog/2019';
-const requireRaw = require.context('!raw-loader!./', false, /2019\.md$/);
-
-export default function Page({ docs }) {
+export default function Page() {
  return <TopLayoutBlog docs={docs} />;
 }
-
-Page.getInitialProps = () => {
- const { demos, docs } = prepareMarkdown({ pageFilename, requireRaw });
- return { demos, docs };
-};

docs/pages/blog/2020-developer-survey-results.js

@@ -1,15 +1,7 @@
 import * as React from 'react';
 import TopLayoutBlog from 'docs/src/modules/components/TopLayoutBlog';
-import { prepareMarkdown } from 'docs/src/modules/utils/parseMarkdown';
+import { docs } from '!@material-ui/markdown/loader!./2020-developer-survey-results.md';
 
-const pageFilename = 'blog/2020-developer-survey-results';
-const requireRaw = require.context('!raw-loader!./', false, /2020-developer-survey-results\.md$/);
-
-export default function Page({ docs }) {
+export default function Page() {
  return <TopLayoutBlog docs={docs} />;
 }
-
-Page.getInitialProps = () => {
- const { demos, docs } = prepareMarkdown({ pageFilename, requireRaw });
- return { demos, docs };
-};