From d8fc7746e3572e15f0fb8f058ae019352b13849c Mon Sep 17 00:00:00 2001 From: siriwatknp Date: Mon, 9 Jun 2025 16:17:34 +0700 Subject: [PATCH 01/29] init generate-llms-txt --- .../scripts/generate-llms-txt/src/index.ts | 95 +++++++++++ .../generate-llms-txt/test/index.test.ts | 156 ++++++++++++++++++ .../scripts/generate-llms-txt/tsconfig.json | 12 ++ .../generate-llms-txt/tsconfig.test.json | 12 ++ packages-internal/scripts/package.json | 6 +- packages-internal/scripts/tsconfig.json | 2 +- tsconfig.json | 3 + 7 files changed, 284 insertions(+), 2 deletions(-) create mode 100755 packages-internal/scripts/generate-llms-txt/src/index.ts create mode 100644 packages-internal/scripts/generate-llms-txt/test/index.test.ts create mode 100644 packages-internal/scripts/generate-llms-txt/tsconfig.json create mode 100644 packages-internal/scripts/generate-llms-txt/tsconfig.test.json diff --git a/packages-internal/scripts/generate-llms-txt/src/index.ts b/packages-internal/scripts/generate-llms-txt/src/index.ts new file mode 100755 index 00000000000000..c4ab2ec042f715 --- /dev/null +++ b/packages-internal/scripts/generate-llms-txt/src/index.ts @@ -0,0 +1,95 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +interface DemoReplaceOptions { + basePath?: string; + includeTypeScript?: boolean; +} + +/** + * Parses markdown content and replaces demo syntax with code snippets + * @param markdownContent - The markdown content to parse + * @param markdownPath - The path to the markdown file (used to resolve relative demo paths) + * @param options - Options for parsing + * @returns The processed markdown with demo code snippets + */ +export function replaceDemoWithSnippet( + markdownContent: string, + markdownPath: string, + options: DemoReplaceOptions = {} +): string { + const { basePath = '', includeTypeScript = true } = options; + + // Regular expression to match {{"demo": "filename.js"}} pattern + const demoRegex = /\{\{"demo":\s*"([^"]+)"(?:,\s*[^}]+)?\}\}/g; + + return markdownContent.replace(demoRegex, (match, filename) => { + try { + // Extract the base filename without extension + const baseFilename = filename.replace(/\.(js|tsx?)$/, ''); + + // Get the directory of the markdown file + const markdownDir = path.dirname(markdownPath); + + let codeSnippet = ''; + + // Try to read JavaScript file + const jsPath = basePath ? + path.join(basePath, `${baseFilename}.js`) : + path.join(markdownDir, `${baseFilename}.js`); + + if (fs.existsSync(jsPath)) { + const jsContent = fs.readFileSync(jsPath, 'utf-8'); + codeSnippet += `\`\`\`jsx\n${jsContent}\n\`\`\``; + } + + // Try to read TypeScript file if includeTypeScript is true + if (includeTypeScript) { + const tsPath = basePath ? + path.join(basePath, `${baseFilename}.tsx`) : + path.join(markdownDir, `${baseFilename}.tsx`); + + if (fs.existsSync(tsPath)) { + const tsContent = fs.readFileSync(tsPath, 'utf-8'); + if (codeSnippet) { + codeSnippet += '\n\n'; + } + codeSnippet += `\`\`\`tsx\n${tsContent}\n\`\`\``; + } + } + + // If no files found, return original match + if (!codeSnippet) { + console.warn(`Demo file not found: ${filename}`); + return match; + } + + return codeSnippet; + } catch (error) { + console.error(`Error processing demo ${filename}:`, error); + return match; + } + }); +} + +/** + * Processes a markdown file and replaces demo syntax with code snippets + * @param filePath - Path to the markdown file + * @param options - Options for parsing + * @returns The processed markdown content + */ +export function processMarkdownFile( + filePath: string, + options: DemoReplaceOptions = {} +): string { + const content = fs.readFileSync(filePath, 'utf-8'); + const dir = path.dirname(filePath); + + // Set basePath relative to markdown file location if not provided + const processOptions = { + ...options, + basePath: options.basePath || dir, + }; + + return replaceDemoWithSnippet(content, filePath, processOptions); +} \ No newline at end of file diff --git a/packages-internal/scripts/generate-llms-txt/test/index.test.ts b/packages-internal/scripts/generate-llms-txt/test/index.test.ts new file mode 100644 index 00000000000000..062068c86b8462 --- /dev/null +++ b/packages-internal/scripts/generate-llms-txt/test/index.test.ts @@ -0,0 +1,156 @@ +import { expect } from 'chai'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import { replaceDemoWithSnippet, processMarkdownFile } from '../src/index'; + +describe('generate-llms-txt', () => { + let tempDir: string; + + beforeEach(() => { + // Create a temporary directory for test files + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'llms-txt-test-')); + }); + + afterEach(() => { + // Clean up temporary directory + fs.rmSync(tempDir, { recursive: true, force: true }); + }); + + describe('replaceDemoWithSnippet', () => { + it('should replace demo syntax with code snippet', () => { + const markdown = `# Test Component + +Here is a demo: + +{{"demo": "BasicButton.js"}} + +More content here.`; + + const jsContent = `import React from 'react'; + +export default function BasicButton() { + return ; +}`; + + // Create test files + fs.writeFileSync(path.join(tempDir, 'BasicButton.js'), jsContent); + + const result = replaceDemoWithSnippet(markdown, path.join(tempDir, 'test.md'), { basePath: tempDir }); + + expect(result).to.include('```jsx'); + expect(result).to.include(jsContent); + expect(result).to.not.include('{{"demo": "BasicButton.js"}}'); + }); + + it('should include both JS and TS files when includeTypeScript is true', () => { + const markdown = `{{"demo": "Component.js"}}`; + + const jsContent = `// JavaScript version`; + const tsContent = `// TypeScript version`; + + fs.writeFileSync(path.join(tempDir, 'Component.js'), jsContent); + fs.writeFileSync(path.join(tempDir, 'Component.tsx'), tsContent); + + const result = replaceDemoWithSnippet(markdown, path.join(tempDir, 'test.md'), { + basePath: tempDir, + includeTypeScript: true + }); + + expect(result).to.include('```jsx'); + expect(result).to.include(jsContent); + expect(result).to.include('```tsx'); + expect(result).to.include(tsContent); + }); + + it('should only include JS file when includeTypeScript is false', () => { + const markdown = `{{"demo": "Component.js"}}`; + + const jsContent = `// JavaScript version`; + const tsContent = `// TypeScript version`; + + fs.writeFileSync(path.join(tempDir, 'Component.js'), jsContent); + fs.writeFileSync(path.join(tempDir, 'Component.tsx'), tsContent); + + const result = replaceDemoWithSnippet(markdown, path.join(tempDir, 'test.md'), { + basePath: tempDir, + includeTypeScript: false + }); + + expect(result).to.include('```jsx'); + expect(result).to.include(jsContent); + expect(result).to.not.include('```tsx'); + expect(result).to.not.include(tsContent); + }); + + it('should handle multiple demos in the same markdown', () => { + const markdown = `# Multiple Demos + +{{"demo": "First.js"}} + +Some text in between. + +{{"demo": "Second.js"}}`; + + fs.writeFileSync(path.join(tempDir, 'First.js'), 'First component'); + fs.writeFileSync(path.join(tempDir, 'Second.js'), 'Second component'); + + const result = replaceDemoWithSnippet(markdown, path.join(tempDir, 'test.md'), { basePath: tempDir }); + + expect(result).to.include('First component'); + expect(result).to.include('Second component'); + expect(result.match(/```jsx/g)).to.have.lengthOf(2); + }); + + it('should return original match when demo file is not found', () => { + const markdown = `{{"demo": "NonExistent.js"}}`; + + const result = replaceDemoWithSnippet(markdown, path.join(tempDir, 'test.md'), { basePath: tempDir }); + + expect(result).to.equal(markdown); + }); + + it('should handle demos with additional properties', () => { + const markdown = `{{"demo": "Button.js", "defaultCodeOpen": false}}`; + + fs.writeFileSync(path.join(tempDir, 'Button.js'), 'Button code'); + + const result = replaceDemoWithSnippet(markdown, path.join(tempDir, 'test.md'), { basePath: tempDir }); + + expect(result).to.include('```jsx'); + expect(result).to.include('Button code'); + }); + }); + + describe('processMarkdownFile', () => { + it('should process a markdown file correctly', () => { + const markdownPath = path.join(tempDir, 'test.md'); + const markdown = `# Test + +{{"demo": "Demo.js"}}`; + + fs.writeFileSync(markdownPath, markdown); + fs.writeFileSync(path.join(tempDir, 'Demo.js'), 'Demo content'); + + const result = processMarkdownFile(markdownPath); + + expect(result).to.include('```jsx'); + expect(result).to.include('Demo content'); + }); + + it('should handle nested directory structures', () => { + const subDir = path.join(tempDir, 'components', 'buttons'); + fs.mkdirSync(subDir, { recursive: true }); + + const markdownPath = path.join(subDir, 'buttons.md'); + const markdown = `{{"demo": "BasicButton.js"}}`; + + fs.writeFileSync(markdownPath, markdown); + fs.writeFileSync(path.join(subDir, 'BasicButton.js'), 'Button component'); + + const result = processMarkdownFile(markdownPath); + + expect(result).to.include('Button component'); + }); + }); +}); \ No newline at end of file diff --git a/packages-internal/scripts/generate-llms-txt/tsconfig.json b/packages-internal/scripts/generate-llms-txt/tsconfig.json new file mode 100644 index 00000000000000..38c3e68e347578 --- /dev/null +++ b/packages-internal/scripts/generate-llms-txt/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "rootDir": "./src", + "outDir": "../build/generate-llms-txt", + "tsBuildInfoFile": "../build/generate-llms-txt/.tsbuildinfo", + "noImplicitAny": false, + "strict": false, + "skipLibCheck": true + }, + "include": ["./src/*"] +} diff --git a/packages-internal/scripts/generate-llms-txt/tsconfig.test.json b/packages-internal/scripts/generate-llms-txt/tsconfig.test.json new file mode 100644 index 00000000000000..a42c4b79a01b78 --- /dev/null +++ b/packages-internal/scripts/generate-llms-txt/tsconfig.test.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "noEmit": true, + "moduleResolution": "", + "types": ["node", "mocha"], + "strict": true, + "esModuleInterop": true, + "isolatedModules": true + }, + "include": ["./src/*.ts", "./test/*.ts"], + "references": [{ "path": "../../docs-utils/tsconfig.build.json" }] +} diff --git a/packages-internal/scripts/package.json b/packages-internal/scripts/package.json index fef8cc072a2092..8e66bb3c7314f1 100644 --- a/packages-internal/scripts/package.json +++ b/packages-internal/scripts/package.json @@ -8,6 +8,10 @@ "./typescript-to-proptypes": { "types": "./build/typescript-to-proptypes/index.d.ts", "default": "./build/typescript-to-proptypes/index.js" + }, + "./generate-llms-txt": { + "types": "./build/generate-llms-txt/index.d.ts", + "default": "./build/generate-llms-txt/index.js" } }, "repository": { @@ -21,7 +25,7 @@ "build": "tsc --build tsconfig.json", "release:publish": "pnpm build && pnpm publish --tag latest", "release:publish:dry-run": "pnpm build && pnpm publish --tag latest --registry=\"http://localhost:4873/\"", - "test": "cd ../../ && cross-env NODE_ENV=test mocha --config packages-internal/scripts/typescript-to-proptypes/test/.mocharc.js 'packages-internal/scripts/typescript-to-proptypes/**/*.test.?(c|m)[jt]s?(x)'", + "test": "cd ../../ && cross-env NODE_ENV=test mocha --config packages-internal/scripts/typescript-to-proptypes/test/.mocharc.js 'packages-internal/scripts/typescript-to-proptypes/**/*.test.?(c|m)[jt]s?(x)' 'packages-internal/scripts/generate-llms-txt/**/*.test.?(c|m)[jt]s?(x)'", "typescript": "tsc --build tsconfig.typecheck.json" }, "dependencies": { diff --git a/packages-internal/scripts/tsconfig.json b/packages-internal/scripts/tsconfig.json index beb8da4f93e065..777e079ed618c2 100644 --- a/packages-internal/scripts/tsconfig.json +++ b/packages-internal/scripts/tsconfig.json @@ -1,5 +1,5 @@ { "files": [], "include": [], - "references": [{ "path": "./typescript-to-proptypes" }] + "references": [{ "path": "./typescript-to-proptypes" }, { "path": "./generate-llms-txt" }] } diff --git a/tsconfig.json b/tsconfig.json index 676b002f9dbf7b..7552e3d446288f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -46,6 +46,9 @@ "@mui/internal-scripts/typescript-to-proptypes": [ "./packages-internal/scripts/typescript-to-proptypes/src" ], + "@mui/internal-scripts/generate-llms-txt": [ + "./packages-internal/scripts/generate-llms-txt/src" + ], "@mui/internal-test-utils": ["./packages-internal/test-utils/src"], "@mui/internal-test-utils/*": ["./packages-internal/test-utils/src/*"], "@mui/stylis-plugin-rtl": ["./packages/mui-stylis-plugin-rtl/src"], From 1b98e4a1422e467a8c2ee82c3b97c8701b9591a2 Mon Sep 17 00:00:00 2001 From: siriwatknp Date: Mon, 9 Jun 2025 18:21:14 +0700 Subject: [PATCH 02/29] able to generate components and api --- .../scripts/generate-llms-txt/src/index.ts | 98 +---- .../generate-llms-txt/src/processApi.ts | 288 ++++++++++++++ .../generate-llms-txt/src/processComponent.ts | 95 +++++ .../generate-llms-txt/test/processApi.test.ts | 363 ++++++++++++++++++ ...index.test.ts => processComponent.test.ts} | 0 scripts/buildLlmsDocs/index.ts | 205 ++++++++++ scripts/buildLlmsDocs/tsconfig.json | 10 + 7 files changed, 964 insertions(+), 95 deletions(-) mode change 100755 => 100644 packages-internal/scripts/generate-llms-txt/src/index.ts create mode 100644 packages-internal/scripts/generate-llms-txt/src/processApi.ts create mode 100755 packages-internal/scripts/generate-llms-txt/src/processComponent.ts create mode 100644 packages-internal/scripts/generate-llms-txt/test/processApi.test.ts rename packages-internal/scripts/generate-llms-txt/test/{index.test.ts => processComponent.test.ts} (100%) create mode 100644 scripts/buildLlmsDocs/index.ts create mode 100644 scripts/buildLlmsDocs/tsconfig.json diff --git a/packages-internal/scripts/generate-llms-txt/src/index.ts b/packages-internal/scripts/generate-llms-txt/src/index.ts old mode 100755 new mode 100644 index c4ab2ec042f715..85d2b6af02fe09 --- a/packages-internal/scripts/generate-llms-txt/src/index.ts +++ b/packages-internal/scripts/generate-llms-txt/src/index.ts @@ -1,95 +1,3 @@ -import * as fs from 'fs'; -import * as path from 'path'; - -interface DemoReplaceOptions { - basePath?: string; - includeTypeScript?: boolean; -} - -/** - * Parses markdown content and replaces demo syntax with code snippets - * @param markdownContent - The markdown content to parse - * @param markdownPath - The path to the markdown file (used to resolve relative demo paths) - * @param options - Options for parsing - * @returns The processed markdown with demo code snippets - */ -export function replaceDemoWithSnippet( - markdownContent: string, - markdownPath: string, - options: DemoReplaceOptions = {} -): string { - const { basePath = '', includeTypeScript = true } = options; - - // Regular expression to match {{"demo": "filename.js"}} pattern - const demoRegex = /\{\{"demo":\s*"([^"]+)"(?:,\s*[^}]+)?\}\}/g; - - return markdownContent.replace(demoRegex, (match, filename) => { - try { - // Extract the base filename without extension - const baseFilename = filename.replace(/\.(js|tsx?)$/, ''); - - // Get the directory of the markdown file - const markdownDir = path.dirname(markdownPath); - - let codeSnippet = ''; - - // Try to read JavaScript file - const jsPath = basePath ? - path.join(basePath, `${baseFilename}.js`) : - path.join(markdownDir, `${baseFilename}.js`); - - if (fs.existsSync(jsPath)) { - const jsContent = fs.readFileSync(jsPath, 'utf-8'); - codeSnippet += `\`\`\`jsx\n${jsContent}\n\`\`\``; - } - - // Try to read TypeScript file if includeTypeScript is true - if (includeTypeScript) { - const tsPath = basePath ? - path.join(basePath, `${baseFilename}.tsx`) : - path.join(markdownDir, `${baseFilename}.tsx`); - - if (fs.existsSync(tsPath)) { - const tsContent = fs.readFileSync(tsPath, 'utf-8'); - if (codeSnippet) { - codeSnippet += '\n\n'; - } - codeSnippet += `\`\`\`tsx\n${tsContent}\n\`\`\``; - } - } - - // If no files found, return original match - if (!codeSnippet) { - console.warn(`Demo file not found: ${filename}`); - return match; - } - - return codeSnippet; - } catch (error) { - console.error(`Error processing demo ${filename}:`, error); - return match; - } - }); -} - -/** - * Processes a markdown file and replaces demo syntax with code snippets - * @param filePath - Path to the markdown file - * @param options - Options for parsing - * @returns The processed markdown content - */ -export function processMarkdownFile( - filePath: string, - options: DemoReplaceOptions = {} -): string { - const content = fs.readFileSync(filePath, 'utf-8'); - const dir = path.dirname(filePath); - - // Set basePath relative to markdown file location if not provided - const processOptions = { - ...options, - basePath: options.basePath || dir, - }; - - return replaceDemoWithSnippet(content, filePath, processOptions); -} \ No newline at end of file +// Export all functions from both modules +export * from './processComponent'; +export * from './processApi'; \ No newline at end of file diff --git a/packages-internal/scripts/generate-llms-txt/src/processApi.ts b/packages-internal/scripts/generate-llms-txt/src/processApi.ts new file mode 100644 index 00000000000000..e3f608e0dbb738 --- /dev/null +++ b/packages-internal/scripts/generate-llms-txt/src/processApi.ts @@ -0,0 +1,288 @@ +import * as fs from 'fs'; + +interface ApiProp { + type: { + name: string; + description?: string; + }; + required?: boolean; + default?: string; + deprecated?: boolean; + deprecationInfo?: string; + signature?: { + type: string; + describedArgs?: string[]; + }; + additionalInfo?: { + cssApi?: boolean; + sx?: boolean; + }; +} + +interface ApiSlot { + name: string; + description: string; + default: string; + class: string | null; +} + +interface ApiClass { + key: string; + className: string; + description: string; + isGlobal: boolean; +} + +interface ApiInheritance { + component: string; + pathname: string; +} + +interface ApiJson { + props: Record; + name: string; + imports: string[]; + slots?: ApiSlot[]; + classes?: ApiClass[]; + spread?: boolean; + themeDefaultProps?: boolean; + muiName?: string; + forwardsRefTo?: string | null; + filename?: string; + inheritance?: ApiInheritance; + demos?: string; + cssComponent?: boolean; + deprecated?: boolean; + deprecationInfo?: string; +} + +/** + * Convert HTML to markdown + */ +function htmlToMarkdown(html: string): string { + return html + // Decode HTML entities first + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/|/g, '|') + .replace(/ /g, ' ') + .replace(/&/g, '&') + // Convert
to space for inline content (not newline for table cells) + .replace(//gi, ' ') + // Convert to backticks + .replace(/([^<]+)<\/code>/gi, '`$1`') + // Convert to markdown links + .replace(/([^<]+)<\/a>/gi, '[$2]($1)') + // Convert