diff --git a/.gitignore b/.gitignore index b96f61af23f923..cbad2d4b1882e3 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ /docs/export /docs/pages/playground/ /docs/public/feed/ +/docs/public/material-ui/ /examples/**/.cache /packages/mui-codemod/lib /packages/mui-envinfo/*.tgz diff --git a/package.json b/package.json index 43894098314ed9..1aa4bd082d2ffc 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,8 @@ "release:pack": "tsx scripts/releasePack.mts", "docs:api": "rimraf --glob ./docs/pages/**/api-docs ./docs/pages/**/api && pnpm docs:api:build", "docs:api:build": "tsx ./scripts/buidApiDocs/index.ts", - "docs:build": "pnpm --filter docs build", + "docs:llms:build": "rimraf --glob ./docs/public/material-ui/ && tsx ./scripts/buildLlmsDocs/index.ts --projectSettings ./packages/api-docs-builder-core/materialUi/projectSettings.ts", + "docs:build": "pnpm docs:llms:build && pnpm --filter docs build", "docs:build-sw": "pnpm --filter docs build-sw", "docs:build-color-preview": "babel-node scripts/buildColorTypes", "docs:deploy": "pnpm --filter docs run deploy", diff --git a/packages-internal/scripts/generate-llms-txt/src/index.ts b/packages-internal/scripts/generate-llms-txt/src/index.ts new file mode 100644 index 00000000000000..e1aed7f312a8c9 --- /dev/null +++ b/packages-internal/scripts/generate-llms-txt/src/index.ts @@ -0,0 +1,3 @@ +// Export all functions from both modules +export * from './processComponent'; +export * from './processApi'; diff --git a/packages-internal/scripts/generate-llms-txt/src/processApi.ts b/packages-internal/scripts/generate-llms-txt/src/processApi.ts new file mode 100644 index 00000000000000..bd06d74566c876 --- /dev/null +++ b/packages-internal/scripts/generate-llms-txt/src/processApi.ts @@ -0,0 +1,331 @@ +import * as fs from 'fs'; + +interface ApiProp { + type: { + name: string; + description?: string; + }; + required?: boolean; + default?: string; + deprecated?: boolean; + deprecationInfo?: string; + signature?: { + type: string; + describedArgs?: string[]; + }; + additionalInfo?: { + cssApi?: boolean; + sx?: boolean; + }; +} + +interface ApiSlot { + name: string; + description: string; + default: string; + class: string | null; +} + +interface ApiClass { + key: string; + className: string; + description: string; + isGlobal: boolean; +} + +interface ApiInheritance { + component: string; + pathname: string; +} + +interface ApiJson { + props: Record; + name: string; + imports: string[]; + slots?: ApiSlot[]; + classes?: ApiClass[]; + spread?: boolean; + themeDefaultProps?: boolean; + muiName?: string; + forwardsRefTo?: string | null; + filename?: string; + inheritance?: ApiInheritance; + demos?: string; + cssComponent?: boolean; + deprecated?: boolean; + deprecationInfo?: string; +} + +/** + * Convert prop type description from HTML format + */ +function formatPropTypeDescription(html: string): string { + // Decode HTML entities + const result = html + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/|/g, '|') + .replace(/ /g, ' ') + .replace(/&/g, '&') + // Replace
tags with space to maintain readability + .replace(//gi, ' ') + // Clean up excessive whitespace + .replace(/\s+/g, ' ') + .trim(); + + return result; +} + +/** + * Convert HTML to markdown + */ +function htmlToMarkdown(html: string): string { + // First pass: decode entities and handle inline elements + let markdown = html + // Decode HTML entities first + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/|/g, '|') + .replace(/ /g, ' ') + .replace(/&/g, '&') + // Convert to backticks + .replace(/([^<]+)<\/code>/gi, '`$1`') + // Convert to markdown links + .replace(/([^<]+)<\/a>/gi, '[$2]($1)'); + + // Handle lists - process them as complete units to avoid extra line breaks + markdown = markdown.replace(/]*>(.*?)<\/ul>/gis, (match, listContent: string) => { + // Process each list item + const items = listContent + .split(/<\/li>/) + .map((item) => item.replace(/]*>/, '').trim()) + .filter((item) => item.length > 0) + .map((item) => `- ${item}`) + .join('\n'); + + return `\n${items}\n`; + }); + + // Handle other block elements + markdown = markdown + // Convert
to newline + .replace(//gi, '\n') + // Convert

to double newline + .replace(/]*>/gi, '\n\n') + .replace(/<\/p>/gi, '') + // Remove any remaining HTML tags + .replace(/<[^>]+>/g, '') + // Clean up excessive whitespace (but preserve intentional line breaks) + .replace(/[ \t]+/g, ' ') + .replace(/ *\n */g, '\n') + .replace(/\n{3,}/g, '\n\n') + .trim(); + + return markdown; +} + +/** + * Format prop type for markdown + */ +function formatPropType(prop: ApiProp): string { + let type = prop.type.name; + + if (prop.type.description) { + // Use specialized function for prop type descriptions + type = formatPropTypeDescription(prop.type.description); + } + + if (prop.signature) { + type = prop.signature.type; + } + + // Escape pipes in union types for better markdown readability + type = type.replace(/\s*\|\s*/g, ' \\| '); + + // Wrap all prop types in backticks to prevent markdown table issues with pipes + return `\`${type}\``; +} + +/** + * Generate props table + */ +function generatePropsTable(props: Record): string { + const propEntries = Object.entries(props); + if (propEntries.length === 0) { + return ''; + } + + let table = '## Props\n\n'; + table += '| Name | Type | Default | Required | Description |\n'; + table += '|------|------|---------|----------|-------------|\n'; + + for (const [propName, prop] of propEntries) { + const name = prop.deprecated ? `${propName} (deprecated)` : propName; + const type = formatPropType(prop); + const defaultValue = prop.default ? `\`${prop.default}\`` : '-'; + const required = prop.required ? 'Yes' : 'No'; + + let description = ''; + if (prop.deprecated && prop.deprecationInfo) { + description = `⚠️ ${htmlToMarkdown(prop.deprecationInfo)}`; + } else if (prop.additionalInfo?.cssApi) { + description = 'Override or extend the styles applied to the component.'; + } else if (prop.additionalInfo?.sx) { + description = + 'The system prop that allows defining system overrides as well as additional CSS styles.'; + } + + table += `| ${name} | ${type} | ${defaultValue} | ${required} | ${description} |\n`; + } + + return table; +} + +/** + * Generate slots table + */ +function generateSlotsTable(slots: ApiSlot[]): string { + if (!slots || slots.length === 0) { + return ''; + } + + let table = '## Slots\n\n'; + table += '| Name | Default | Class | Description |\n'; + table += '|------|---------|-------|-------------|\n'; + + for (const slot of slots) { + const className = slot.class ? `\`.${slot.class}\`` : '-'; + const description = htmlToMarkdown(slot.description); + table += `| ${slot.name} | \`${slot.default}\` | ${className} | ${description} |\n`; + } + + return table; +} + +/** + * Generate classes table + */ +function generateClassesTable(classes: ApiClass[]): string { + if (!classes || classes.length === 0) { + return ''; + } + + let table = '## CSS\n\n'; + table += '### Rule name\n\n'; + table += '| Global class | Rule name | Description |\n'; + table += '|--------------|-----------|-------------|\n'; + + for (const cls of classes) { + const globalClass = cls.isGlobal ? `\`.${cls.className}\`` : '-'; + const ruleName = cls.isGlobal ? '-' : cls.key; + const description = htmlToMarkdown(cls.description); + table += `| ${globalClass} | ${ruleName} | ${description} |\n`; + } + + return table; +} + +/** + * Process API JSON and convert to markdown + */ +export function processApiJson(apiJson: ApiJson | string): string { + const api: ApiJson = typeof apiJson === 'string' ? JSON.parse(apiJson) : apiJson; + + let markdown = `# ${api.name} API\n\n`; + + // Add deprecation warning if applicable + if (api.deprecated) { + const warningText = api.deprecationInfo + ? htmlToMarkdown(api.deprecationInfo) + : 'This component is deprecated. Consider using an alternative component.'; + markdown += `> ⚠️ **Warning**: ${warningText}\n\n`; + } + + // Add demos section + if (api.demos) { + markdown += '## Demos\n\n'; + markdown += + 'For examples and details on the usage of this React component, visit the component demo pages:\n\n'; + markdown += `${htmlToMarkdown(api.demos)}\n\n`; + } + + // Add import section + markdown += '## Import\n\n'; + markdown += '```jsx\n'; + markdown += api.imports.join('\n// or\n'); + markdown += '\n```\n\n'; + + // Add props section + const propsTable = generatePropsTable(api.props); + if (propsTable) { + markdown += `${propsTable}\n`; + } + + // Add ref information + if (api.forwardsRefTo === null) { + markdown += '> **Note**: This component cannot hold a ref.\n\n'; + } else { + markdown += `> **Note**: The \`ref\` is forwarded to the root element${api.forwardsRefTo ? ` (${api.forwardsRefTo})` : ''}.\n\n`; + } + + // Add spread information + if (api.spread) { + const spreadElement = api.inheritance + ? `[${api.inheritance.component}](${api.inheritance.pathname})` + : 'native element'; + markdown += `> Any other props supplied will be provided to the root element (${spreadElement}).\n\n`; + } + + // Add inheritance section + if (api.inheritance) { + markdown += '## Inheritance\n\n'; + markdown += `While not explicitly documented above, the props of the [${api.inheritance.component}](${api.inheritance.pathname}) component are also available on ${api.name}.`; + if (api.inheritance.component === 'Transition') { + markdown += + ' A subset of components support [react-transition-group](https://reactcommunity.org/react-transition-group/transition/) out of the box.'; + } + markdown += '\n\n'; + } + + // Add theme default props section + if (api.themeDefaultProps && api.muiName) { + markdown += '## Theme default props\n\n'; + markdown += `You can use \`${api.muiName}\` to change the default props of this component with the theme.\n\n`; + } + + // Add slots section + const slotsTable = generateSlotsTable(api.slots || []); + if (slotsTable) { + markdown += `${slotsTable}\n`; + } + + // Add classes section + const classesTable = generateClassesTable(api.classes || []); + if (classesTable) { + markdown += `${classesTable}\n`; + } + + // Add CSS component note + if (api.cssComponent) { + markdown += `> **Note**: As a CSS utility, the \`${api.name}\` component also supports all system properties. You can use them as props directly on the component.\n\n`; + } + + // Add source code section + if (api.filename) { + markdown += '## Source code\n\n'; + markdown += `If you did not find the information on this page, consider having a look at the implementation of the component for more detail.\n\n`; + markdown += `- [${api.filename}](https://github.com/mui/material-ui/tree/HEAD${api.filename})\n\n`; + } + + return markdown.trim(); +} + +/** + * Process API JSON file and return markdown + */ +export function processApiFile(filePath: string): string { + const content = fs.readFileSync(filePath, 'utf-8'); + return processApiJson(content); +} diff --git a/packages-internal/scripts/generate-llms-txt/src/processComponent.ts b/packages-internal/scripts/generate-llms-txt/src/processComponent.ts new file mode 100755 index 00000000000000..beede3bec292f5 --- /dev/null +++ b/packages-internal/scripts/generate-llms-txt/src/processComponent.ts @@ -0,0 +1,122 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +interface DemoReplaceOptions { + basePath?: string; + includeTypeScript?: boolean; +} + +/** + * Removes {{"component": ...}} syntax from markdown content + * @param markdownContent - The markdown content to clean + * @returns The cleaned markdown content + */ +export function removeComponentSyntax(markdownContent: string): string { + // Regular expression to match {{"component": "ComponentName"}} pattern + const componentRegex = /\{\{"component":\s*"[^"]+"\}\}/g; + return markdownContent.replace(componentRegex, ''); +} + +/** + * Converts

HTML tags to plain text in markdown + * @param markdownContent - The markdown content to clean + * @returns The cleaned markdown content + */ +export function cleanDescriptionTags(markdownContent: string): string { + // Replace

...

with just the content + return markdownContent.replace(/

([^<]+)<\/p>/g, '$1'); +} + +/** + * Parses markdown content and replaces demo syntax with code snippets + * @param markdownContent - The markdown content to parse + * @param markdownPath - The path to the markdown file (used to resolve relative demo paths) + * @param options - Options for parsing + * @returns The processed markdown with demo code snippets + */ +export function replaceDemoWithSnippet( + markdownContent: string, + markdownPath: string, + options: DemoReplaceOptions = {}, +): string { + const { basePath = '', includeTypeScript = true } = options; + + // Regular expression to match {{"demo": "filename.js"}} pattern + const demoRegex = /\{\{"demo":\s*"([^"]+)"(?:,\s*[^}]+)?\}\}/g; + + return markdownContent.replace(demoRegex, (match, filename) => { + try { + // Extract the base filename without extension + const baseFilename = filename.replace(/\.(js|tsx?)$/, ''); + + // Get the directory of the markdown file + const markdownDir = path.dirname(markdownPath); + + let codeSnippet = ''; + + // Try to read JavaScript file + const jsPath = basePath + ? path.join(basePath, `${baseFilename}.js`) + : path.join(markdownDir, `${baseFilename}.js`); + + if (fs.existsSync(jsPath)) { + const jsContent = fs.readFileSync(jsPath, 'utf-8'); + codeSnippet += `\`\`\`jsx\n${jsContent}\n\`\`\``; + } + + // Try to read TypeScript file if includeTypeScript is true + if (includeTypeScript) { + const tsPath = basePath + ? path.join(basePath, `${baseFilename}.tsx`) + : path.join(markdownDir, `${baseFilename}.tsx`); + + if (fs.existsSync(tsPath)) { + const tsContent = fs.readFileSync(tsPath, 'utf-8'); + if (codeSnippet) { + codeSnippet += '\n\n'; + } + codeSnippet += `\`\`\`tsx\n${tsContent}\n\`\`\``; + } + } + + // If no files found, return original match + if (!codeSnippet) { + if (process.env.NODE_ENV !== 'test') { + console.warn(`Demo file not found: ${filename}`); + } + return match; + } + + return codeSnippet; + } catch (error) { + console.error(`Error processing demo ${filename}:`, error); + return match; + } + }); +} + +/** + * Processes a markdown file and replaces demo syntax with code snippets + * @param filePath - Path to the markdown file + * @param options - Options for parsing + * @returns The processed markdown content + */ +export function processMarkdownFile(filePath: string, options: DemoReplaceOptions = {}): string { + let content = fs.readFileSync(filePath, 'utf-8'); + const dir = path.dirname(filePath); + + // Set basePath relative to markdown file location if not provided + const processOptions = { + ...options, + basePath: options.basePath || dir, + }; + + // First, remove component syntax + content = removeComponentSyntax(content); + + // Clean description HTML tags + content = cleanDescriptionTags(content); + + // Then, replace demo syntax with code snippets + return replaceDemoWithSnippet(content, filePath, processOptions); +} diff --git a/packages-internal/scripts/generate-llms-txt/test/processApi.test.ts b/packages-internal/scripts/generate-llms-txt/test/processApi.test.ts new file mode 100644 index 00000000000000..aac851e1a26f90 --- /dev/null +++ b/packages-internal/scripts/generate-llms-txt/test/processApi.test.ts @@ -0,0 +1,376 @@ +import { expect } from 'chai'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import { processApiJson, processApiFile } from '../src/processApi'; + +describe('processApi', () => { + describe('processApiJson', () => { + it('should generate basic component API markdown', () => { + const apiJson = { + name: 'Button', + imports: [ + "import Button from '@mui/material/Button';", + "import { Button } from '@mui/material';", + ], + props: { + color: { + type: { name: 'string' }, + default: "'primary'", + required: false, + }, + disabled: { + type: { name: 'bool' }, + default: 'false', + required: false, + }, + }, + }; + + const result = processApiJson(apiJson); + + expect(result).to.include('# Button API'); + expect(result).to.include('## Import'); + expect(result).to.include("import Button from '@mui/material/Button';"); + expect(result).to.include('## Props'); + expect(result).to.include("| color | `string` | `'primary'` | No |"); + expect(result).to.include('| disabled | `bool` | `false` | No |'); + }); + + it('should handle deprecated component', () => { + const apiJson = { + name: 'DeprecatedComponent', + imports: ["import DeprecatedComponent from '@mui/material/DeprecatedComponent';"], + props: {}, + deprecated: true, + deprecationInfo: 'Use NewComponent instead.', + }; + + const result = processApiJson(apiJson); + + expect(result).to.include('> ⚠️ **Warning**: Use `NewComponent` instead.'); + }); + + it('should handle deprecated props', () => { + const apiJson = { + name: 'Component', + imports: ["import Component from '@mui/material/Component';"], + props: { + oldProp: { + type: { name: 'string' }, + deprecated: true, + deprecationInfo: 'Use newProp instead.', + }, + }, + }; + + const result = processApiJson(apiJson); + + expect(result).to.include('| oldProp (deprecated) |'); + expect(result).to.include('⚠️ Use `newProp` instead.'); + }); + + it('should handle complex prop types', () => { + const apiJson = { + name: 'Component', + imports: ["import Component from '@mui/material/Component';"], + props: { + onChange: { + type: { name: 'func' }, + signature: { + type: 'function(event: React.SyntheticEvent, value: number) => void', + describedArgs: ['event', 'value'], + }, + }, + slots: { + type: { + name: 'shape', + description: '{ root?: elementType, icon?: elementType }', + }, + }, + sx: { + type: { + name: 'union', + description: + 'Array<func
| object>
| func
| object', + }, + additionalInfo: { sx: true }, + }, + }, + }; + + const result = processApiJson(apiJson); + + expect(result).to.include('`function(event: React.SyntheticEvent, value: number) => void`'); + expect(result).to.include('`{ root?: elementType, icon?: elementType }`'); + expect(result).to.include('`Array \\| func \\| object`'); + expect(result).to.include('The system prop that allows defining system overrides'); + }); + + it('should handle demos section', () => { + const apiJson = { + name: 'Accordion', + imports: ["import Accordion from '@mui/material/Accordion';"], + props: {}, + demos: '

', + }; + + const result = processApiJson(apiJson); + + expect(result).to.include('## Demos'); + expect(result).to.include('- [Accordion](/material-ui/react-accordion/)'); + }); + + it('should handle slots section', () => { + const apiJson = { + name: 'Component', + imports: ["import Component from '@mui/material/Component';"], + props: {}, + slots: [ + { + name: 'root', + description: 'The component that renders the root.', + default: 'Paper', + class: 'MuiComponent-root', + }, + { + name: 'icon', + description: 'The icon element.', + default: 'svg', + class: null, + }, + ], + }; + + const result = processApiJson(apiJson); + + expect(result).to.include('## Slots'); + expect(result).to.include( + '| root | `Paper` | `.MuiComponent-root` | The component that renders the root. |', + ); + expect(result).to.include('| icon | `svg` | - | The icon element. |'); + }); + + it('should handle classes section', () => { + const apiJson = { + name: 'Component', + imports: ["import Component from '@mui/material/Component';"], + props: {}, + classes: [ + { + key: 'disabled', + className: 'Mui-disabled', + description: 'State class applied to the root element if `disabled={true}`.', + isGlobal: true, + }, + { + key: 'root', + className: 'MuiComponent-root', + description: 'Styles applied to the root element.', + isGlobal: false, + }, + ], + }; + + const result = processApiJson(apiJson); + + expect(result).to.include('## CSS'); + expect(result).to.include('### Rule name'); + expect(result).to.include( + '| `.Mui-disabled` | - | State class applied to the root element if `disabled={true}`. |', + ); + expect(result).to.include('| - | root | Styles applied to the root element. |'); + }); + + it('should handle inheritance', () => { + const apiJson = { + name: 'Accordion', + imports: ["import Accordion from '@mui/material/Accordion';"], + props: {}, + inheritance: { + component: 'Paper', + pathname: '/material-ui/api/paper/', + }, + }; + + const result = processApiJson(apiJson); + + expect(result).to.include('## Inheritance'); + expect(result).to.include('[Paper](/material-ui/api/paper/)'); + expect(result).to.include( + 'the props of the [Paper](/material-ui/api/paper/) component are also available on Accordion', + ); + }); + + it('should handle spread props', () => { + const apiJson = { + name: 'Component', + imports: ["import Component from '@mui/material/Component';"], + props: {}, + spread: true, + inheritance: { + component: 'Paper', + pathname: '/material-ui/api/paper/', + }, + }; + + const result = processApiJson(apiJson); + + expect(result).to.include( + 'Any other props supplied will be provided to the root element ([Paper](/material-ui/api/paper/))', + ); + }); + + it('should handle ref forwarding', () => { + const apiJson = { + name: 'Component', + imports: ["import Component from '@mui/material/Component';"], + props: {}, + forwardsRefTo: 'HTMLDivElement', + }; + + const result = processApiJson(apiJson); + + expect(result).to.include('The `ref` is forwarded to the root element (HTMLDivElement)'); + }); + + it('should handle components that cannot hold refs', () => { + const apiJson = { + name: 'Component', + imports: ["import Component from '@mui/material/Component';"], + props: {}, + forwardsRefTo: null, + }; + + const result = processApiJson(apiJson); + + expect(result).to.include('This component cannot hold a ref'); + }); + + it('should handle theme default props', () => { + const apiJson = { + name: 'Button', + imports: ["import Button from '@mui/material/Button';"], + props: {}, + themeDefaultProps: true, + muiName: 'MuiButton', + }; + + const result = processApiJson(apiJson); + + expect(result).to.include('## Theme default props'); + expect(result).to.include('You can use `MuiButton` to change the default props'); + }); + + it('should handle CSS component', () => { + const apiJson = { + name: 'Box', + imports: ["import Box from '@mui/material/Box';"], + props: {}, + cssComponent: true, + }; + + const result = processApiJson(apiJson); + + expect(result).to.include( + 'As a CSS utility, the `Box` component also supports all system properties', + ); + }); + + it('should handle source code section', () => { + const apiJson = { + name: 'Component', + imports: ["import Component from '@mui/material/Component';"], + props: {}, + filename: '/packages/mui-material/src/Component/Component.js', + }; + + const result = processApiJson(apiJson); + + expect(result).to.include('## Source code'); + expect(result).to.include( + 'https://github.com/mui/material-ui/tree/HEAD/packages/mui-material/src/Component/Component.js', + ); + }); + + it('should handle required props', () => { + const apiJson = { + name: 'Component', + imports: ["import Component from '@mui/material/Component';"], + props: { + children: { + type: { name: 'node' }, + required: true, + }, + optional: { + type: { name: 'string' }, + required: false, + }, + }, + }; + + const result = processApiJson(apiJson); + + expect(result).to.include('| children | `node` | - | Yes |'); + expect(result).to.include('| optional | `string` | - | No |'); + }); + }); + + describe('processApiFile', () => { + let tempDir: string; + + beforeEach(() => { + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'api-test-')); + }); + + afterEach(() => { + fs.rmSync(tempDir, { recursive: true, force: true }); + }); + + it('should process API JSON file', () => { + const apiJson = { + name: 'TestComponent', + imports: ["import TestComponent from '@mui/material/TestComponent';"], + props: { + test: { + type: { name: 'bool' }, + default: 'true', + }, + }, + }; + + const filePath = path.join(tempDir, 'test-component.json'); + fs.writeFileSync(filePath, JSON.stringify(apiJson, null, 2)); + + const result = processApiFile(filePath); + + expect(result).to.include('# TestComponent API'); + expect(result).to.include('| test | `bool` | `true` | No |'); + }); + }); + + describe('HTML to Markdown conversion', () => { + it('should convert HTML entities and tags correctly', () => { + const apiJson = { + name: 'Component', + imports: ["import Component from '@mui/material/Component';"], + props: { + complexProp: { + type: { + name: 'union', + description: 'Array<func
| object>
| func', + }, + }, + }, + demos: '

Test paragraph

  • Item 1
  • Item 2
', + }; + + const result = processApiJson(apiJson); + + expect(result).to.include('`Array \\| func`'); + expect(result).to.include('Test paragraph'); + expect(result).to.include('- Item 1'); + expect(result).to.include('- Item 2'); + }); + }); +}); diff --git a/packages-internal/scripts/generate-llms-txt/test/processComponent.test.ts b/packages-internal/scripts/generate-llms-txt/test/processComponent.test.ts new file mode 100644 index 00000000000000..bef716cbf65b4a --- /dev/null +++ b/packages-internal/scripts/generate-llms-txt/test/processComponent.test.ts @@ -0,0 +1,164 @@ +import { expect } from 'chai'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import { replaceDemoWithSnippet, processMarkdownFile } from '../src/index'; + +describe('generate-llms-txt', () => { + let tempDir: string; + + beforeEach(() => { + // Create a temporary directory for test files + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'llms-txt-test-')); + }); + + afterEach(() => { + // Clean up temporary directory + fs.rmSync(tempDir, { recursive: true, force: true }); + }); + + describe('replaceDemoWithSnippet', () => { + it('should replace demo syntax with code snippet', () => { + const markdown = `# Test Component + +Here is a demo: + +{{"demo": "BasicButton.js"}} + +More content here.`; + + const jsContent = `import React from 'react'; + +export default function BasicButton() { + return ; +}`; + + // Create test files + fs.writeFileSync(path.join(tempDir, 'BasicButton.js'), jsContent); + + const result = replaceDemoWithSnippet(markdown, path.join(tempDir, 'test.md'), { + basePath: tempDir, + }); + + expect(result).to.include('```jsx'); + expect(result).to.include(jsContent); + expect(result).to.not.include('{{"demo": "BasicButton.js"}}'); + }); + + it('should include both JS and TS files when includeTypeScript is true', () => { + const markdown = `{{"demo": "Component.js"}}`; + + const jsContent = `// JavaScript version`; + const tsContent = `// TypeScript version`; + + fs.writeFileSync(path.join(tempDir, 'Component.js'), jsContent); + fs.writeFileSync(path.join(tempDir, 'Component.tsx'), tsContent); + + const result = replaceDemoWithSnippet(markdown, path.join(tempDir, 'test.md'), { + basePath: tempDir, + includeTypeScript: true, + }); + + expect(result).to.include('```jsx'); + expect(result).to.include(jsContent); + expect(result).to.include('```tsx'); + expect(result).to.include(tsContent); + }); + + it('should only include JS file when includeTypeScript is false', () => { + const markdown = `{{"demo": "Component.js"}}`; + + const jsContent = `// JavaScript version`; + const tsContent = `// TypeScript version`; + + fs.writeFileSync(path.join(tempDir, 'Component.js'), jsContent); + fs.writeFileSync(path.join(tempDir, 'Component.tsx'), tsContent); + + const result = replaceDemoWithSnippet(markdown, path.join(tempDir, 'test.md'), { + basePath: tempDir, + includeTypeScript: false, + }); + + expect(result).to.include('```jsx'); + expect(result).to.include(jsContent); + expect(result).to.not.include('```tsx'); + expect(result).to.not.include(tsContent); + }); + + it('should handle multiple demos in the same markdown', () => { + const markdown = `# Multiple Demos + +{{"demo": "First.js"}} + +Some text in between. + +{{"demo": "Second.js"}}`; + + fs.writeFileSync(path.join(tempDir, 'First.js'), 'First component'); + fs.writeFileSync(path.join(tempDir, 'Second.js'), 'Second component'); + + const result = replaceDemoWithSnippet(markdown, path.join(tempDir, 'test.md'), { + basePath: tempDir, + }); + + expect(result).to.include('First component'); + expect(result).to.include('Second component'); + expect(result.match(/```jsx/g)).to.have.lengthOf(2); + }); + + it('should return original match when demo file is not found', () => { + const markdown = `{{"demo": "NonExistent.js"}}`; + + const result = replaceDemoWithSnippet(markdown, path.join(tempDir, 'test.md'), { + basePath: tempDir, + }); + + expect(result).to.equal(markdown); + }); + + it('should handle demos with additional properties', () => { + const markdown = `{{"demo": "Button.js", "defaultCodeOpen": false}}`; + + fs.writeFileSync(path.join(tempDir, 'Button.js'), 'Button code'); + + const result = replaceDemoWithSnippet(markdown, path.join(tempDir, 'test.md'), { + basePath: tempDir, + }); + + expect(result).to.include('```jsx'); + expect(result).to.include('Button code'); + }); + }); + + describe('processMarkdownFile', () => { + it('should process a markdown file correctly', () => { + const markdownPath = path.join(tempDir, 'test.md'); + const markdown = `# Test + +{{"demo": "Demo.js"}}`; + + fs.writeFileSync(markdownPath, markdown); + fs.writeFileSync(path.join(tempDir, 'Demo.js'), 'Demo content'); + + const result = processMarkdownFile(markdownPath); + + expect(result).to.include('```jsx'); + expect(result).to.include('Demo content'); + }); + + it('should handle nested directory structures', () => { + const subDir = path.join(tempDir, 'components', 'buttons'); + fs.mkdirSync(subDir, { recursive: true }); + + const markdownPath = path.join(subDir, 'buttons.md'); + const markdown = `{{"demo": "BasicButton.js"}}`; + + fs.writeFileSync(markdownPath, markdown); + fs.writeFileSync(path.join(subDir, 'BasicButton.js'), 'Button component'); + + const result = processMarkdownFile(markdownPath); + + expect(result).to.include('Button component'); + }); + }); +}); diff --git a/packages-internal/scripts/generate-llms-txt/tsconfig.json b/packages-internal/scripts/generate-llms-txt/tsconfig.json new file mode 100644 index 00000000000000..38c3e68e347578 --- /dev/null +++ b/packages-internal/scripts/generate-llms-txt/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "rootDir": "./src", + "outDir": "../build/generate-llms-txt", + "tsBuildInfoFile": "../build/generate-llms-txt/.tsbuildinfo", + "noImplicitAny": false, + "strict": false, + "skipLibCheck": true + }, + "include": ["./src/*"] +} diff --git a/packages-internal/scripts/generate-llms-txt/tsconfig.test.json b/packages-internal/scripts/generate-llms-txt/tsconfig.test.json new file mode 100644 index 00000000000000..a42c4b79a01b78 --- /dev/null +++ b/packages-internal/scripts/generate-llms-txt/tsconfig.test.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "noEmit": true, + "moduleResolution": "", + "types": ["node", "mocha"], + "strict": true, + "esModuleInterop": true, + "isolatedModules": true + }, + "include": ["./src/*.ts", "./test/*.ts"], + "references": [{ "path": "../../docs-utils/tsconfig.build.json" }] +} diff --git a/packages-internal/scripts/package.json b/packages-internal/scripts/package.json index 7844e4d7caedf3..092e507be1bea9 100644 --- a/packages-internal/scripts/package.json +++ b/packages-internal/scripts/package.json @@ -8,6 +8,10 @@ "./typescript-to-proptypes": { "types": "./build/typescript-to-proptypes/index.d.ts", "default": "./build/typescript-to-proptypes/index.js" + }, + "./generate-llms-txt": { + "types": "./build/generate-llms-txt/index.d.ts", + "default": "./build/generate-llms-txt/index.js" } }, "repository": { @@ -21,7 +25,7 @@ "build": "tsc --build tsconfig.json", "release:publish": "pnpm build && pnpm publish --tag latest", "release:publish:dry-run": "pnpm build && pnpm publish --tag latest --registry=\"http://localhost:4873/\"", - "test": "cd ../../ && cross-env NODE_ENV=test mocha --config packages-internal/scripts/typescript-to-proptypes/test/.mocharc.js 'packages-internal/scripts/typescript-to-proptypes/**/*.test.?(c|m)[jt]s?(x)'", + "test": "cd ../../ && cross-env NODE_ENV=test mocha --config packages-internal/scripts/typescript-to-proptypes/test/.mocharc.js 'packages-internal/scripts/typescript-to-proptypes/**/*.test.?(c|m)[jt]s?(x)' 'packages-internal/scripts/generate-llms-txt/**/*.test.?(c|m)[jt]s?(x)'", "typescript": "tsc --build tsconfig.typecheck.json" }, "dependencies": { diff --git a/packages-internal/scripts/tsconfig.json b/packages-internal/scripts/tsconfig.json index beb8da4f93e065..777e079ed618c2 100644 --- a/packages-internal/scripts/tsconfig.json +++ b/packages-internal/scripts/tsconfig.json @@ -1,5 +1,5 @@ { "files": [], "include": [], - "references": [{ "path": "./typescript-to-proptypes" }] + "references": [{ "path": "./typescript-to-proptypes" }, { "path": "./generate-llms-txt" }] } diff --git a/packages/api-docs-builder-core/joyUi/getJoyUiComponentInfo.ts b/packages/api-docs-builder-core/joyUi/getJoyUiComponentInfo.ts index f0d84cd52c4b9a..d2358007899f11 100644 --- a/packages/api-docs-builder-core/joyUi/getJoyUiComponentInfo.ts +++ b/packages/api-docs-builder-core/joyUi/getJoyUiComponentInfo.ts @@ -60,6 +60,7 @@ export function getJoyUiComponentInfo(filename: string): ComponentInfo { return allMarkdowns .filter((page) => page.pathname.startsWith('/joy') && page.components.includes(name)) .map((page) => ({ + filePath: page.filename, // pathname of the markdown file demoPageTitle: getTitle(page.markdownContent), demoPathname: fixPathname(page.pathname), })); diff --git a/packages/api-docs-builder-core/materialUi/getMaterialUiComponentInfo.test.ts b/packages/api-docs-builder-core/materialUi/getMaterialUiComponentInfo.test.ts index b065ccd0451b1d..c9247f6a58eca5 100644 --- a/packages/api-docs-builder-core/materialUi/getMaterialUiComponentInfo.test.ts +++ b/packages/api-docs-builder-core/materialUi/getMaterialUiComponentInfo.test.ts @@ -30,16 +30,20 @@ describe('getMaterialUiComponentInfo', () => { // eslint-disable-next-line no-empty } catch (error) {} if (existed) { - expect(componentInfo.getDemos()).to.deep.equal([ - { - demoPageTitle: 'Button Group', - demoPathname: '/material-ui/react-button-group/', - }, - { - demoPageTitle: 'Button', - demoPathname: '/material-ui/react-button/', - }, - ]); + const demos = componentInfo.getDemos(); + expect(demos).to.have.lengthOf(2); + + expect(demos[0]).to.deep.include({ + demoPageTitle: 'Button Group', + demoPathname: '/material-ui/react-button-group/', + }); + expect(demos[0].filePath).to.include('button-group/button-group.md'); + + expect(demos[1]).to.deep.include({ + demoPageTitle: 'Button', + demoPathname: '/material-ui/react-button/', + }); + expect(demos[1].filePath).to.include('buttons/buttons.md'); } }); }); diff --git a/packages/api-docs-builder-core/materialUi/getMaterialUiComponentInfo.ts b/packages/api-docs-builder-core/materialUi/getMaterialUiComponentInfo.ts index 3090b60d23053a..91735722c979dd 100644 --- a/packages/api-docs-builder-core/materialUi/getMaterialUiComponentInfo.ts +++ b/packages/api-docs-builder-core/materialUi/getMaterialUiComponentInfo.ts @@ -62,6 +62,7 @@ export function getMaterialUiComponentInfo(filename: string): ComponentInfo { return allMarkdowns .filter((page) => page.pathname.startsWith('/material') && page.components.includes(name)) .map((page) => ({ + filePath: page.filename, // pathname of the markdown file demoPageTitle: getTitle(page.markdownContent), demoPathname: fixPathname(page.pathname), })); diff --git a/packages/api-docs-builder-core/materialUi/projectSettings.ts b/packages/api-docs-builder-core/materialUi/projectSettings.ts index b443e8664e983e..c8391266a79c36 100644 --- a/packages/api-docs-builder-core/materialUi/projectSettings.ts +++ b/packages/api-docs-builder-core/materialUi/projectSettings.ts @@ -46,4 +46,12 @@ export const projectSettings: ProjectSettings = { isGlobalClassName: isGlobalState, // #host-reference baseApiUrl: 'https://mui.com', + nonComponentFolders: [ + 'material/getting-started', + 'material/customization', + 'material/experimental-api', + 'material/guides', + 'material/integrations', + 'material/migration', + ], }; diff --git a/packages/api-docs-builder-core/muiSystem/getSystemComponentInfo.ts b/packages/api-docs-builder-core/muiSystem/getSystemComponentInfo.ts index 4d8b8a580c8424..c6bfffe8ec841d 100644 --- a/packages/api-docs-builder-core/muiSystem/getSystemComponentInfo.ts +++ b/packages/api-docs-builder-core/muiSystem/getSystemComponentInfo.ts @@ -76,6 +76,7 @@ export function getSystemComponentInfo(filename: string): ComponentInfo { return allMarkdowns .filter((page) => page.components.includes(name)) .map((page) => ({ + filePath: page.filename, // pathname of the markdown file demoPageTitle: pathToSystemTitle({ ...page, title: getTitle(page.markdownContent), diff --git a/packages/api-docs-builder/ProjectSettings.ts b/packages/api-docs-builder/ProjectSettings.ts index e814a25d7ee428..c5b752bd153cae 100644 --- a/packages/api-docs-builder/ProjectSettings.ts +++ b/packages/api-docs-builder/ProjectSettings.ts @@ -113,4 +113,9 @@ export interface ProjectSettings { * Determines the base API URL for generated JSDocs */ baseApiUrl?: string; + /** + * Determines the non-component folders to be included in the llms.txt file. + * The folders are relative to the `docs/data` directory. + */ + nonComponentFolders?: string[]; } diff --git a/packages/api-docs-builder/index.ts b/packages/api-docs-builder/index.ts index 5bc030987a977b..f504480f844334 100644 --- a/packages/api-docs-builder/index.ts +++ b/packages/api-docs-builder/index.ts @@ -10,4 +10,4 @@ export type { HookApiContent, ComponentClassDefinition, } from './types/ApiBuilder.types'; -export type { Slot } from './types/utils.types'; +export type { Slot, ComponentInfo } from './types/utils.types'; diff --git a/packages/api-docs-builder/types/utils.types.ts b/packages/api-docs-builder/types/utils.types.ts index 6cd4f256bdd028..58c3c940cee044 100644 --- a/packages/api-docs-builder/types/utils.types.ts +++ b/packages/api-docs-builder/types/utils.types.ts @@ -47,7 +47,7 @@ export type ComponentInfo = { */ apiPathname: string; }; - getDemos: () => Array<{ demoPageTitle: string; demoPathname: string }>; + getDemos: () => Array<{ demoPageTitle: string; demoPathname: string; filePath: string }>; apiPagesDirectory: string; /** * The path to import specific layout config of the page if needed. diff --git a/scripts/buildLlmsDocs/index.ts b/scripts/buildLlmsDocs/index.ts new file mode 100644 index 00000000000000..59b8bbe36d0acd --- /dev/null +++ b/scripts/buildLlmsDocs/index.ts @@ -0,0 +1,599 @@ +/** + * LLM Documentation Generator + * + * This script generates LLM-optimized documentation by processing MUI component markdown files + * and non-component documentation files to create comprehensive, standalone documentation. + * + * ## Main Workflow: + * + * 1. **Component Processing**: + * - Discovers all components using the API docs builder infrastructure + * - For each component, finds its markdown documentation and API JSON + * - Processes markdown by replacing `{{"demo": "filename.js"}}` syntax with actual code snippets + * - Appends API documentation (props, slots, CSS classes) to the markdown + * - Outputs to files like `material-ui/react-accordion.md` + * + * 2. **Non-Component Processing** (optional): + * - Processes markdown files from specified folders (e.g., `system`, `material/customization`) + * - Applies the same demo replacement logic + * - Uses URL transformation logic to maintain consistent paths with components + * - Outputs to files like `system/borders.md`, `material-ui/customization/color.md` + * + * 3. **Index Generation** (llms.txt): + * - Generates `llms.txt` index files for each top-level directory + * - Groups files by category (components, customization, getting-started, etc.) + * - Creates markdown-formatted lists with relative paths and descriptions + * - Outputs to files like `material-ui/llms.txt`, `system/llms.txt` + * + * ## Key Features: + * + * - **Demo Replacement**: Converts `{{"demo": "filename.js"}}` to actual JSX/TSX code snippets + * - **API Integration**: Automatically includes component API documentation (props, slots, CSS) + * - **Reusable**: Accepts project settings via CLI to work across different repositories + * - **Filtering**: Supports grep patterns to process specific components/files + * - **Path Consistency**: Uses existing URL transformation logic for consistent output structure + * - **Auto-indexing**: Generates llms.txt files with categorized documentation listings + * + * ## Usage Examples: + * + * ```bash + * # Process all Material UI components + * pnpm tsx scripts/buildLlmsDocs/index.ts --projectSettings ./packages/api-docs-builder-core/materialUi/projectSettings.ts + * + * # Process specific components with non-component docs + * pnpm tsx scripts/buildLlmsDocs/index.ts \ + * --projectSettings ./packages/api-docs-builder-core/materialUi/projectSettings.ts \ + * --nonComponentFolders system material/customization \ + * --grep "Button|borders" + * ``` + * + * ## Output Structure: + * + * - **Components**: `material-ui/react-{component}.md` (e.g., `material-ui/react-button.md`) + * - **Customization**: `material-ui/customization/{topic}.md` (e.g., `material-ui/customization/color.md`) + * - **Getting Started**: `material-ui/getting-started/{topic}.md` (e.g., `material-ui/getting-started/installation.md`) + * - **Index Files**: `{directory}/llms.txt` (e.g., `material-ui/llms.txt`, `system/llms.txt`) + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import yargs, { ArgumentsCamelCase } from 'yargs'; +import kebabCase from 'lodash/kebabCase'; +import { processMarkdownFile, processApiFile } from '@mui/internal-scripts/generate-llms-txt'; +import { ComponentInfo, ProjectSettings } from '@mui-internal/api-docs-builder'; +import { getHeaders } from '@mui/internal-markdown'; +import { fixPathname } from '@mui-internal/api-docs-builder/buildApiUtils'; +import replaceUrl from '@mui-internal/api-docs-builder/utils/replaceUrl'; +import findComponents from '@mui-internal/api-docs-builder/utils/findComponents'; +import findPagesMarkdown from '@mui-internal/api-docs-builder/utils/findPagesMarkdown'; + +interface ComponentDocInfo { + name: string; + componentInfo: ComponentInfo; + demos: Array<{ demoPageTitle: string; demoPathname: string }>; + markdownPath?: string; + apiJsonPath?: string; +} + +interface GeneratedFile { + outputPath: string; + title: string; + description: string; + originalMarkdownPath: string; + category: string; + orderIndex?: number; // Track the order for non-component folders +} + +type CommandOptions = { + grep?: string; + outputDir?: string; + projectSettings?: string; +}; + +/** + * Find all components using the API docs builder infrastructure + */ +async function findComponentsToProcess( + projectSettings: ProjectSettings, + grep: RegExp | null, +): Promise { + const components: ComponentDocInfo[] = []; + + // Iterate through TypeScript projects, using the same logic as buildApi.ts + for (const project of projectSettings.typeScriptProjects) { + const projectComponents = findComponents(path.join(project.rootPath, 'src')).filter( + (component) => { + if (projectSettings.skipComponent(component.filename)) { + return false; + } + + if (grep === null) { + return true; + } + + return grep.test(component.filename); + }, + ); + + for (const component of projectComponents) { + try { + // Get component info using the API docs builder + const componentInfo = projectSettings.getComponentInfo(component.filename); + + // Skip if component should be skipped (internal, etc.) + const fileInfo = componentInfo.readFile(); + if (fileInfo.shouldSkip) { + continue; + } + + // Get demos for this component + const demos = componentInfo.getDemos(); + + // Skip if no demos found (likely not a public component) + if (demos.length === 0) { + continue; + } + + // Get the markdown file path from the first demo + const firstDemo = demos[0]; + const markdownPath = firstDemo ? firstDemo.filePath : undefined; + + // Get API JSON path + const apiJsonPath = path.join( + componentInfo.apiPagesDirectory, + `${path.basename(componentInfo.apiPathname)}.json`, + ); + + components.push({ + name: componentInfo.name, + componentInfo, + demos, + markdownPath, + apiJsonPath: fs.existsSync(apiJsonPath) ? apiJsonPath : undefined, + }); + } catch (error) { + // Skip components that can't be processed + continue; + } + } + } + + return components; +} + +/** + * Extract title and description from markdown content + */ +function extractMarkdownInfo(markdownPath: string): { title: string; description: string } { + try { + const content = fs.readFileSync(markdownPath, 'utf-8'); + const headers = getHeaders(content); + + // Get title from frontmatter or first h1 + const title = + headers.title || content.match(/^# (.+)$/m)?.[1] || path.basename(markdownPath, '.md'); + + // Extract description from the first paragraph with class="description" + const descriptionMatch = content.match(/

([^<]+)<\/p>/); + let description = ''; + + if (descriptionMatch) { + description = descriptionMatch[1].trim(); + } else { + // Fallback: get first paragraph after title (excluding headers) + const paragraphMatch = content.match(/^# .+\n\n(?!#)(.+?)(?:\n\n|$)/m); + if (paragraphMatch && !paragraphMatch[1].startsWith('#')) { + description = paragraphMatch[1].trim(); + } + } + + return { title, description }; + } catch (error) { + return { + title: path.basename(markdownPath, '.md'), + description: '', + }; + } +} + +/** + * Find all non-component markdown files from specified folders + */ +function findNonComponentMarkdownFiles( + folders: string[], + grep: RegExp | null, +): Array<{ markdownPath: string; outputPath: string }> { + // Get all markdown files using the existing findPagesMarkdown utility + const allMarkdownFiles = findPagesMarkdown(); + + const files: Array<{ markdownPath: string; outputPath: string }> = []; + + for (const page of allMarkdownFiles) { + // Check if the page belongs to one of the specified folders + const belongsToFolder = folders.some((folder) => page.pathname.startsWith(`/${folder}`)); + if (!belongsToFolder) { + continue; + } + + // Apply grep filter if specified + if (grep) { + const fileName = path.basename(page.filename); + if (!grep.test(fileName) && !grep.test(page.pathname)) { + continue; + } + } + + // Apply fixPathname first, then replaceUrl to get the proper output structure (like components) + const afterFixPathname = fixPathname(page.pathname); + const fixedPathname = replaceUrl(afterFixPathname, '/material-ui/'); + const outputPath = `${fixedPathname.replace(/^\//, '').replace(/\/$/, '')}.md`; + + files.push({ + markdownPath: page.filename, + outputPath, + }); + } + + return files; +} + +/** + * Process a single component + */ +function processComponent(component: ComponentDocInfo): string | null { + // Processing component: ${component.name} + + // Skip if no markdown file found + if (!component.markdownPath) { + console.error(`Warning: No markdown file found for component: ${component.name}`); + return null; + } + + // Process the markdown file with demo replacement + let processedMarkdown = processMarkdownFile(component.markdownPath); + + // Read the frontmatter to get all components listed in this markdown file + const markdownContent = fs.readFileSync(component.markdownPath, 'utf-8'); + const headers = getHeaders(markdownContent); + const componentsInPage = headers.components || []; + + // Add API sections for all components listed in the frontmatter + if (componentsInPage.length > 0) { + for (const componentName of componentsInPage) { + // Construct the API JSON path based on the project settings + const apiJsonPath = path.join( + component.componentInfo.apiPagesDirectory, + `${kebabCase(componentName)}.json`, + ); + + if (fs.existsSync(apiJsonPath)) { + try { + const apiMarkdown = processApiFile(apiJsonPath); + processedMarkdown += `\n\n${apiMarkdown}`; + } catch (error) { + console.error(`Warning: Could not process API for ${componentName}:`, error); + } + } else { + console.warn(`Warning: API JSON file not found for ${componentName}: ${apiJsonPath}`); + } + } + } else if (component.apiJsonPath) { + // Fallback: Add API section for the primary component if no frontmatter components found + try { + const apiMarkdown = processApiFile(component.apiJsonPath); + processedMarkdown += `\n\n${apiMarkdown}`; + } catch (error) { + console.error(`Warning: Could not process API for ${component.name}:`, error); + } + } + + return processedMarkdown; +} + +/** + * Convert kebab-case to Title Case + */ +function toTitleCase(kebabCaseStr: string): string { + return kebabCaseStr + .split('-') + .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) + .join(' '); +} + +/** + * Generate llms.txt content for a specific directory + */ +function generateLlmsTxt( + generatedFiles: GeneratedFile[], + projectName: string, + baseDir: string, +): string { + // Group files by category + const groupedByCategory: Record = {}; + + for (const file of generatedFiles) { + const category = file.category; + if (!groupedByCategory[category]) { + groupedByCategory[category] = []; + } + groupedByCategory[category].push(file); + } + + // Generate content + let content = `# ${projectName}\n\n`; + content += `This is the documentation for the ${projectName} package.\n`; + content += `It contains comprehensive guides, components, and utilities for building user interfaces.\n\n`; + + // Add sections for each category + // Sort categories to ensure components appear first, then by orderIndex for non-component folders + const sortedCategories = Object.keys(groupedByCategory).sort((a, b) => { + if (a === 'components') { + return -1; + } + if (b === 'components') { + return 1; + } + + // For non-component categories, check if they have orderIndex + const filesA = groupedByCategory[a]; + const filesB = groupedByCategory[b]; + const orderIndexA = filesA[0]?.orderIndex ?? Number.MAX_SAFE_INTEGER; + const orderIndexB = filesB[0]?.orderIndex ?? Number.MAX_SAFE_INTEGER; + + if (orderIndexA !== orderIndexB) { + return orderIndexA - orderIndexB; + } + + return a.localeCompare(b); + }); + + for (const category of sortedCategories) { + const files = groupedByCategory[category]; + if (files.length === 0) { + continue; + } + + const sectionTitle = toTitleCase(category); + content += `## ${sectionTitle}\n\n`; + + // Sort files by title (components) or maintain original order (non-components) + if (category === 'components') { + files.sort((a, b) => a.title.localeCompare(b.title)); + } + // Non-component files are already in the order they were discovered + + for (const file of files) { + // Calculate relative path from the baseDir to the file + const relativePath = file.outputPath.startsWith(`${baseDir}/`) + ? `/${baseDir}/${file.outputPath.substring(baseDir.length + 1)}` + : `/${file.outputPath}`; + content += `- [${file.title}](${relativePath})`; + if (file.description) { + content += `: ${file.description}`; + } + content += '\n'; + } + content += '\n'; + } + + return content.trim(); +} + +/** + * Main build function + */ +async function buildLlmsDocs(argv: ArgumentsCamelCase): Promise { + const grep = argv.grep ? new RegExp(argv.grep) : null; + const outputDir = argv.outputDir || path.join(process.cwd(), 'docs/public'); + + // Load project settings from the specified path + if (!argv.projectSettings) { + throw new Error('--projectSettings is required'); + } + + let projectSettings: ProjectSettings; + try { + const settingsPath = path.resolve(argv.projectSettings); + const settingsModule = await import(settingsPath); + projectSettings = settingsModule.projectSettings || settingsModule.default || settingsModule; + } catch (error) { + throw new Error(`Failed to load project settings from ${argv.projectSettings}: ${error}`); + } + + // Building LLMs docs... + // Project settings: ${argv.projectSettings} + // Output directory: ${outputDir} + if (grep) { + // Filter pattern: ${grep} + } + + // Find all components + const components = await findComponentsToProcess(projectSettings, grep); + + // Found ${components.length} components to process + + // Find non-component markdown files if specified in project settings + let nonComponentFiles: Array<{ markdownPath: string; outputPath: string }> = []; + const nonComponentFolders = (projectSettings as any).nonComponentFolders; + if (nonComponentFolders && nonComponentFolders.length > 0) { + nonComponentFiles = findNonComponentMarkdownFiles(nonComponentFolders, grep); + // Found ${nonComponentFiles.length} non-component markdown files to process + } + + // Track generated files for llms.txt + const generatedFiles: GeneratedFile[] = []; + + // Process each component + let processedCount = 0; + for (const component of components) { + try { + const processedMarkdown = processComponent(component); + + if (!processedMarkdown) { + continue; + } + + // Use the component's demo pathname to create the output structure + // e.g., /material-ui/react-accordion/ -> material-ui/react-accordion.md + const outputFileName = component.demos[0] + ? `${component.demos[0].demoPathname.replace(/^\//, '').replace(/\/$/, '')}.md` + : `${component.componentInfo.apiPathname.replace(/^\//, '').replace(/\/$/, '')}.md`; + + const outputPath = path.join(outputDir, outputFileName); + + // Check if this file has already been generated (avoid duplicates for components that share the same markdown file) + const existingFile = generatedFiles.find((f) => f.outputPath === outputFileName); + if (!existingFile) { + // Ensure the directory exists + const outputDirPath = path.dirname(outputPath); + if (!fs.existsSync(outputDirPath)) { + fs.mkdirSync(outputDirPath, { recursive: true }); + } + + fs.writeFileSync(outputPath, processedMarkdown, 'utf-8'); + // ✓ Generated: ${outputFileName} + processedCount += 1; + + // Track this file for llms.txt + if (component.markdownPath) { + const { title, description } = extractMarkdownInfo(component.markdownPath); + generatedFiles.push({ + outputPath: outputFileName, + title, + description, + originalMarkdownPath: component.markdownPath, + category: 'components', + }); + } + } + } catch (error) { + console.error(`✗ Error processing ${component.name}:`, error); + } + } + + // Process non-component markdown files + for (const file of nonComponentFiles) { + try { + // Processing non-component file: ${path.relative(process.cwd(), file.markdownPath)} + + // Process the markdown file with demo replacement + const processedMarkdown = processMarkdownFile(file.markdownPath); + + const outputPath = path.join(outputDir, file.outputPath); + + // Ensure the directory exists + const outputDirPath = path.dirname(outputPath); + if (!fs.existsSync(outputDirPath)) { + fs.mkdirSync(outputDirPath, { recursive: true }); + } + + fs.writeFileSync(outputPath, processedMarkdown, 'utf-8'); + // ✓ Generated: ${file.outputPath} + processedCount += 1; + + // Track this file for llms.txt + const { title, description } = extractMarkdownInfo(file.markdownPath); + + // Extract category from the file path + // e.g., "material-ui/customization/color.md" -> "customization" + // e.g., "getting-started/installation.md" -> "getting-started" + const pathParts = file.outputPath.split('/'); + const category = pathParts.reverse()[1]; + + // Find the order index based on which folder this file belongs to + let orderIndex = -1; + if (nonComponentFolders) { + for (let i = 0; i < nonComponentFolders.length; i += 1) { + if (file.markdownPath.includes(`/${nonComponentFolders[i]}/`)) { + orderIndex = i; + break; + } + } + } + + generatedFiles.push({ + outputPath: file.outputPath, + title, + description, + originalMarkdownPath: file.markdownPath, + category, + orderIndex, + }); + } catch (error) { + console.error(`✗ Error processing ${file.markdownPath}:`, error); + } + } + + // Generate llms.txt files + if (generatedFiles.length > 0) { + const groupedByFirstDir: Record = {}; + + for (const file of generatedFiles) { + const firstDir = file.outputPath.split('/')[0]; + if (!groupedByFirstDir[firstDir]) { + groupedByFirstDir[firstDir] = []; + } + groupedByFirstDir[firstDir].push(file); + } + + for (const [dirName, files] of Object.entries(groupedByFirstDir)) { + let projectName; + if (dirName === 'material-ui') { + projectName = 'Material UI'; + } else if (dirName === 'system') { + projectName = 'MUI System'; + } else { + projectName = dirName.charAt(0).toUpperCase() + dirName.slice(1); + } + + const llmsContent = generateLlmsTxt(files, projectName, dirName); + const llmsPath = path.join(outputDir, dirName, 'llms.txt'); + + // Ensure directory exists + const llmsDirPath = path.dirname(llmsPath); + if (!fs.existsSync(llmsDirPath)) { + fs.mkdirSync(llmsDirPath, { recursive: true }); + } + + fs.writeFileSync(llmsPath, llmsContent, 'utf-8'); + // ✓ Generated: ${dirName}/llms.txt + processedCount += 1; + } + } + + // eslint-disable-next-line no-console + console.log(`\nCompleted! Generated ${processedCount} files in ${outputDir}`); +} + +/** + * CLI setup + */ +yargs(process.argv.slice(2)) + .command({ + command: '$0', + describe: 'Generates LLM-optimized documentation for MUI components.', + builder: (command: any) => { + return command + .option('grep', { + description: + 'Only generate files for components matching the pattern. The string is treated as a RegExp.', + type: 'string', + }) + .option('outputDir', { + description: 'Output directory for generated markdown files.', + type: 'string', + default: './docs/public', + }) + .option('projectSettings', { + description: + 'Path to the project settings module that exports ProjectSettings interface.', + type: 'string', + demandOption: true, + }); + }, + handler: buildLlmsDocs, + }) + .help() + .strict(true) + .version(false) + .parse(); diff --git a/scripts/buildLlmsDocs/tsconfig.json b/scripts/buildLlmsDocs/tsconfig.json new file mode 100644 index 00000000000000..2b56dc8d607113 --- /dev/null +++ b/scripts/buildLlmsDocs/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "./build", + "allowJs": false, + "noEmit": false + }, + "include": ["./index.ts"], + "exclude": [] +} diff --git a/tsconfig.json b/tsconfig.json index 676b002f9dbf7b..7552e3d446288f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -46,6 +46,9 @@ "@mui/internal-scripts/typescript-to-proptypes": [ "./packages-internal/scripts/typescript-to-proptypes/src" ], + "@mui/internal-scripts/generate-llms-txt": [ + "./packages-internal/scripts/generate-llms-txt/src" + ], "@mui/internal-test-utils": ["./packages-internal/test-utils/src"], "@mui/internal-test-utils/*": ["./packages-internal/test-utils/src/*"], "@mui/stylis-plugin-rtl": ["./packages/mui-stylis-plugin-rtl/src"],