diff --git a/index.js b/index.js index f78b11d..b9d01fb 100644 --- a/index.js +++ b/index.js @@ -235,6 +235,157 @@ function convertRequirementsToMarkdown(content) { ); } +// Temporarily replace fenced code blocks so downstream regexes cannot corrupt them. +// The step-9 JSX stripper matches /<[A-Z].../ and treats heredoc markers like +// `<) because `[A-Z]` matches the E in EOF. +// It then deletes everything until the next self-closing JSX tag (e.g. +// ``), leaving an unclosed ``` fence in the emitted .md file. +function protectFencedCodeBlocks(content) { + const blocks = []; + const protectedContent = content.replace( + /(^|\n)(```[\s\S]*?\n```)/g, + (match, prefix, block) => { + const token = `\n__FENCE_${blocks.length}__\n`; + blocks.push(block); + return prefix + token; + } + ); + return { content: protectedContent, blocks }; +} + +function restoreFencedCodeBlocks(content, blocks) { + blocks.forEach((block, index) => { + content = content.replace(`__FENCE_${index}__`, block.trim()); + }); + return content; +} + +// Convert Docusaurus CodeBlock components to fenced code blocks +function convertCodeBlockToMarkdown(content, substitutions = {}) { + content = content.replace( + /\s*\{`([\s\S]*?)`\}\s*<\/CodeBlock>/g, + (match, language, title, code) => { + let cleanCode = code; + for (const [key, value] of Object.entries(substitutions)) { + cleanCode = cleanCode.replace(new RegExp(`\\$\\{${key}\\}`, 'g'), value); + } + const opener = title + ? '```' + language + ' title="' + title + '"' + : '```' + language; + return opener + '\n' + cleanCode + '\n```'; + } + ); + return content.replace( + /]*>\s*\n?\s*([^<{][\s\S]*?)\s*<\/CodeBlock>/g, + (match, language, code) => '```' + language + '\n' + code.trim() + '\n```' + ); +} + +// Include both nightly and stable branches so agents see all install paths +function convertConditionalVersionDocsToMarkdown(content) { + content = content.replace( + /\s*([\s\S]*?)<\/ConditionalVersionDocs>/gi, + (match, inner) => '**Nightly:**\n\n' + inner.trim() + '\n\n' + ); + return content.replace( + /\s*([\s\S]*?)<\/ConditionalVersionDocs>/gi, + (match, inner) => '**Stable:**\n\n' + inner.trim() + '\n\n' + ); +} + +function convertAdmonitionToMarkdown(content) { + return content.replace( + /\s*([\s\S]*?)<\/Admonition>/g, + (match, type, inner) => { + const cleanInner = inner + .replace(/

/g, '') + .replace(/<\/p>/g, '\n') + .replace(/([^<]*)<\/b>/g, '**$1**') + .replace(/([^<]*)<\/code>/g, '`$1`') + .replace(/<[^>]+>/g, '') + .trim(); + return '> **' + type + ':** ' + cleanInner + '\n'; + } + ); +} + +function convertFigureToMarkdown(content, fileDir, imgUrlBase) { + return content.replace( + /

\s*]*alt="([^"]*)"[^>]*\/>\s*
([\s\S]*?)<\/figcaption>\s*<\/figure>/g, + (match, imagePath, alt, caption) => { + const relativePath = imagePath.replace(/^\.\//, ''); + const normalizedBase = imgUrlBase.endsWith('/') ? imgUrlBase : imgUrlBase + '/'; + const imageUrl = normalizedBase + fileDir + relativePath; + const cleanCaption = caption.replace(/<\/?b>/g, '**').replace(/<\/?strong>/g, '**'); + return '![' + alt + '](' + imageUrl + ')\n\n*' + cleanCaption + '*'; + } + ); +} + +function extractIncludeComponentBody(includeContent) { + let body = includeContent.replace(/^import[\s\S]*?(?=^export default)/m, ''); + const returnMatch = body.match(/return\s*\(\s*([\s\S]*)\s*\)\s*;\s*\}\s*$/); + return returnMatch ? returnMatch[1] : body; +} + +function processIncludeTemplate(includeContent, substitutions) { + let body = extractIncludeComponentBody(includeContent); + body = body.replace(/const tooltipNightly[\s\S]*?;\s*/g, ''); + body = body.replace(/const tooltipStable[\s\S]*?;\s*/g, ''); + body = body.replace(/\(\{tooltipNightly\}\)/g, '(nightly)'); + body = body.replace(/\(\{tooltipStable\}\)/g, '(stable)'); + body = body.replace(//g, ''); + for (const [key, value] of Object.entries(substitutions)) { + body = body.replace(new RegExp(`\\$\\{${key}\\}`, 'g'), value); + } + body = convertConditionalVersionDocsToMarkdown(body); + body = convertCodeBlockToMarkdown(body, substitutions); + body = convertAdmonitionToMarkdown(body); + body = body.replace(/([\s\S]*?)<\/Link>/g, '[$2]($1)'); + body = body.replace(/([^<]*)<\/code><\/a>/g, '[`$2`]($1)'); + body = removeDivTags(body); + body = convertTabsToMarkdown(body); + body = body.replace(/<[^>]+>/g, ''); + return body.trim(); +} + +function convertInstallModularToMarkdown(content, pluginContext) { + if (!pluginContext) return content; + const includeTemplate = pluginContext.includeTemplates?.['install-modular']; + if (!includeTemplate) return content; + + return content.replace(/]*?)\/>/gi, (match) => { + const folder = extractAttribute(match, 'folder') || 'example-project'; + const extraMatch = match.match(/extraLibraries=\{(\[[^\]]*\])\}/); + let extraLibs = []; + if (extraMatch) { + try { + extraLibs = JSON.parse(extraMatch[1].replace(/'/g, '"')); + } catch (error) { + extraLibs = []; + } + } + const extraLibsString = extraLibs.length > 0 ? ' ' + extraLibs.join(' ') : ''; + const extraLibsDescription = + extraLibs.length > 0 ? ' and other required packages' : ''; + return processIncludeTemplate(includeTemplate, { + folder, + extraLibsString, + extraLibsDescription, + }); + }); +} + +function convertInstallOpenAIToMarkdown(content, pluginContext) { + if (!pluginContext) return content; + const includeTemplate = pluginContext.includeTemplates?.['install-openai']; + if (!includeTemplate) return content; + + return content.replace(//gi, () => + processIncludeTemplate(includeTemplate, {}) + ); +} + // Unwrap MDX components by removing their tags but preserving inner content function unwrapMdxComponents(content) { // List of MDX components to unwrap (keeps growing as we find more) @@ -243,7 +394,7 @@ function unwrapMdxComponents(content) { const components = [ 'ModelSelector', 'ModelDropdownTabs', - 'InstallModular' + 'InstallModular', ]; for (const comp of components) { @@ -315,7 +466,8 @@ function cleanMarkdownForDisplay( content, filepath, docsPath = '/docs/', - assetBasePath = undefined + assetBasePath = undefined, + pluginContext = undefined ) { const imgUrlBase = assetBasePath !== undefined ? assetBasePath : docsPath; @@ -346,6 +498,13 @@ function cleanMarkdownForDisplay( // 6. Convert Requirements component to link content = convertRequirementsToMarkdown(content); + // Expand shared MDX includes into markdown (improves .md/HTML parity for agents) + content = convertInstallModularToMarkdown(content, pluginContext); + content = convertInstallOpenAIToMarkdown(content, pluginContext); + content = convertConditionalVersionDocsToMarkdown(content); + content = convertCodeBlockToMarkdown(content); + content = convertAdmonitionToMarkdown(content); + // Convert Button components with DocLink to markdown links content = content.replace( /\s*([\s\S]*?)\s*<\/Button>/g, @@ -400,10 +559,20 @@ function cleanMarkdownForDisplay( // 8. Convert details/summary components to readable markdown (preserve content) content = convertDetailsToMarkdown(content); + // Convert figure blocks before stripping remaining JSX components + content = convertFigureToMarkdown(content, fileDir, imgUrlBase); + // 9. Remove custom React/MDX components (FAQStructuredData, etc.) // Matches both self-closing and paired tags: or ... - // This runs AFTER Tabs/details conversion to preserve their content - content = content.replace(/<[A-Z][a-zA-Z]*[\s\S]*?(?:\/>|<\/[A-Z][a-zA-Z]*>)/g, ''); + // This runs AFTER Tabs/details conversion to preserve their content. + // Fenced code blocks are protected first so heredoc syntax is not corrupted. + const { content: protectedContent, blocks: fencedBlocks } = + protectFencedCodeBlocks(content); + content = protectedContent.replace( + /<[A-Z][a-zA-Z]*[\s\S]*?(?:\/>|<\/[A-Z][a-zA-Z]*>)/g, + '' + ); + content = restoreFencedCodeBlocks(content, fencedBlocks); // 10. Convert relative image paths to absolute paths from docs root // Matches: ![alt](./img/file.png) or ![alt](img/file.png) @@ -677,6 +846,7 @@ async function writeProcessedMarkdownToBuild({ docsPrefix, extraLinkPrefixes = [], fileContent, + pluginContext, }) { await fs.ensureDir(path.dirname(destPath)); const content = fileContent ?? (await fs.readFile(sourcePath, 'utf8')); @@ -684,7 +854,8 @@ async function writeProcessedMarkdownToBuild({ content, mdFileRelativeForCleaning, docsPath, - assetBasePath + assetBasePath, + pluginContext ); if (directive) { cleanedContent = directive + '\n\n' + cleanedContent; @@ -747,7 +918,7 @@ async function copyImageDirectories(docsDir, buildDir) { return copiedCount; } -module.exports = function markdownSourcePlugin(context, options = {}) { +function markdownSourcePlugin(context, options = {}) { // Configurable options with defaults for backwards compatibility const docsPath = options.docsPath || '/docs/'; const docsDirName = options.docsDir || 'docs'; @@ -817,6 +988,21 @@ module.exports = function markdownSourcePlugin(context, options = {}) { } } + const includeTemplates = {}; + const includesDir = path.join(context.siteDir, docsDirName, '_includes'); + for (const [key, file] of [['install-modular', 'install-modular.mdx'], ['install-openai', 'install-openai.mdx']]) { + const p = path.join(includesDir, file); + if (fs.existsSync(p)) { + includeTemplates[key] = fs.readFileSync(p, 'utf8'); + } + } + + const pluginContext = { + siteDir: context.siteDir, + docsDirName, + includeTemplates, + }; + console.log('[markdown-source-plugin] Copying markdown source files...'); // Find all markdown files in docs directory @@ -864,6 +1050,7 @@ module.exports = function markdownSourcePlugin(context, options = {}) { docsPrefix, extraLinkPrefixes: blogLinkPrefixes, fileContent: raw, + pluginContext, }); copiedCount++; @@ -965,6 +1152,7 @@ module.exports = function markdownSourcePlugin(context, options = {}) { docsPrefix, extraLinkPrefixes: blogLinkPrefixes, fileContent: raw, + pluginContext, }); blogCopied++; copiedCount++; @@ -992,4 +1180,19 @@ module.exports = function markdownSourcePlugin(context, options = {}) { ); }, }; +} + +module.exports = markdownSourcePlugin; + +module.exports._internals = { + cleanMarkdownForDisplay, + protectFencedCodeBlocks, + restoreFencedCodeBlocks, + convertCodeBlockToMarkdown, + convertConditionalVersionDocsToMarkdown, + convertAdmonitionToMarkdown, + convertFigureToMarkdown, + convertInstallModularToMarkdown, + convertInstallOpenAIToMarkdown, + unwrapMdxComponents, }; diff --git a/tests/test-code-fence-protection.js b/tests/test-code-fence-protection.js new file mode 100644 index 0000000..5e61619 --- /dev/null +++ b/tests/test-code-fence-protection.js @@ -0,0 +1,368 @@ +/** + * Tests for fenced code block protection, MDX component converters, + * and the JSX stripper regression that motivated the protect/restore mechanism. + * + * Run with: node tests/test-code-fence-protection.js + */ +const assert = require('assert'); +const { + cleanMarkdownForDisplay, + protectFencedCodeBlocks, + restoreFencedCodeBlocks, + convertCodeBlockToMarkdown, + convertConditionalVersionDocsToMarkdown, + convertAdmonitionToMarkdown, + convertFigureToMarkdown, + convertInstallModularToMarkdown, + convertInstallOpenAIToMarkdown, + unwrapMdxComponents, +} = require('../index')._internals; + +let passed = 0; +let failed = 0; + +function test(name, fn) { + try { + fn(); + passed++; + console.log(` ✓ ${name}`); + } catch (e) { + failed++; + console.error(` ✗ ${name}`); + console.error(` ${e.message}`); + } +} + +// --------------------------------------------------------------------------- +// protectFencedCodeBlocks / restoreFencedCodeBlocks +// --------------------------------------------------------------------------- +console.log('\nprotectFencedCodeBlocks / restoreFencedCodeBlocks:'); + +test('round-trips a simple fenced block', () => { + const input = 'before\n```python\nprint("hi")\n```\nafter'; + const { content, blocks } = protectFencedCodeBlocks(input); + assert(!content.includes('```python'), 'fence should be replaced by token'); + assert(content.includes('__FENCE_0__'), 'token should be present'); + const restored = restoreFencedCodeBlocks(content, blocks); + assert(restored.includes('```python\nprint("hi")\n```'), 'fence should be restored'); + assert(restored.includes('before'), 'surrounding text preserved'); + assert(restored.includes('after'), 'surrounding text preserved'); +}); + +test('round-trips multiple fenced blocks', () => { + const input = '```js\na()\n```\ntext\n```bash\nb()\n```'; + const { content, blocks } = protectFencedCodeBlocks(input); + assert.strictEqual(blocks.length, 2); + assert(content.includes('__FENCE_0__')); + assert(content.includes('__FENCE_1__')); + const restored = restoreFencedCodeBlocks(content, blocks); + assert(restored.includes('```js\na()\n```')); + assert(restored.includes('```bash\nb()\n```')); +}); + +test('preserves heredoc syntax inside fenced blocks', () => { + const input = '```bash\ncat < config.yaml\nkey: value\nEOF\n```'; + const { content, blocks } = protectFencedCodeBlocks(input); + assert(!content.includes('< { + const input = [ + '---', + 'title: Test', + '---', + '', + 'Some text', + '', + '```bash', + 'cat < config.yaml', + 'key: value', + 'EOF', + '```', + '', + '', + '', + 'More text', + ].join('\n'); + + const result = cleanMarkdownForDisplay(input, 'test.md'); + assert(result.includes('```bash'), 'bash fence opener should survive'); + assert(result.includes('< { + const input = [ + '---', + 'title: Test', + '---', + '', + '```python', + 'X = SomeClass()', + '```', + '', + '', + '', + '```bash', + 'export MY_VAR="hello"', + '```', + ].join('\n'); + + const result = cleanMarkdownForDisplay(input, 'test.md'); + assert(result.includes('X = SomeClass()'), 'python code preserved'); + assert(result.includes('export MY_VAR="hello"'), 'bash code preserved'); + assert(!result.includes(' { + const input = '{`print("hello")`}'; + const result = convertCodeBlockToMarkdown(input); + assert.strictEqual(result, '```python\nprint("hello")\n```'); +}); + +test('converts CodeBlock with title attribute', () => { + const input = '{`const x = 1;`}'; + const result = convertCodeBlockToMarkdown(input); + assert(result.includes('```js title="example.js"')); + assert(result.includes('const x = 1;')); +}); + +test('applies substitutions to CodeBlock content', () => { + const input = '{`cd ${folder}`}'; + const result = convertCodeBlockToMarkdown(input, { folder: 'my-project' }); + assert(result.includes('cd my-project')); +}); + +test('converts CodeBlock with plain text content', () => { + const input = '\nHello world\n'; + const result = convertCodeBlockToMarkdown(input); + assert(result.includes('```text')); + assert(result.includes('Hello world')); +}); + +test('leaves non-matching content unchanged', () => { + const input = 'regular markdown text\n\n```python\ncode\n```'; + const result = convertCodeBlockToMarkdown(input); + assert.strictEqual(result, input); +}); + +// --------------------------------------------------------------------------- +// convertConditionalVersionDocsToMarkdown +// --------------------------------------------------------------------------- +console.log('\nconvertConditionalVersionDocsToMarkdown:'); + +test('converts nightly version docs', () => { + const input = '\nInstall nightly build\n'; + const result = convertConditionalVersionDocsToMarkdown(input); + assert(result.includes('**Nightly:**')); + assert(result.includes('Install nightly build')); +}); + +test('converts stable version docs', () => { + const input = '\nInstall stable build\n'; + const result = convertConditionalVersionDocsToMarkdown(input); + assert(result.includes('**Stable:**')); + assert(result.includes('Install stable build')); +}); + +test('converts both nightly and stable in same content', () => { + const input = [ + 'nightly content', + 'stable content', + ].join('\n'); + const result = convertConditionalVersionDocsToMarkdown(input); + assert(result.includes('**Nightly:**')); + assert(result.includes('nightly content')); + assert(result.includes('**Stable:**')); + assert(result.includes('stable content')); +}); + +test('leaves unrecognized content unchanged', () => { + const input = 'just some text'; + assert.strictEqual(convertConditionalVersionDocsToMarkdown(input), input); +}); + +// --------------------------------------------------------------------------- +// convertAdmonitionToMarkdown +// --------------------------------------------------------------------------- +console.log('\nconvertAdmonitionToMarkdown:'); + +test('converts Admonition with type to blockquote', () => { + const input = '\nRemember this.\n'; + const result = convertAdmonitionToMarkdown(input); + assert(result.includes('> **note:**')); + assert(result.includes('Remember this.')); +}); + +test('converts HTML within Admonition content', () => { + const input = '

Use caution with rm

'; + const result = convertAdmonitionToMarkdown(input); + assert(result.includes('**caution**')); + assert(result.includes('`rm`')); + assert(!result.includes('

')); + assert(!result.includes('')); +}); + +test('leaves non-Admonition content unchanged', () => { + const input = 'plain text'; + assert.strictEqual(convertAdmonitionToMarkdown(input), input); +}); + +// --------------------------------------------------------------------------- +// convertFigureToMarkdown +// --------------------------------------------------------------------------- +console.log('\nconvertFigureToMarkdown:'); + +test('converts figure with require() image to markdown', () => { + const input = '

\nArchitecture\n
System overview
\n
'; + const result = convertFigureToMarkdown(input, 'getting-started/', '/docs/'); + assert(result.includes('![Architecture]')); + assert(result.includes('/docs/getting-started/img/diagram.png')); + assert(result.includes('*System overview*')); +}); + +test('converts bold in figcaption to markdown bold', () => { + const input = '
\nChart\n
Figure 1: Results
\n
'; + const result = convertFigureToMarkdown(input, '', '/docs/'); + assert(result.includes('**Figure 1:**')); +}); + +test('leaves non-matching figure HTML unchanged', () => { + const input = '
test
'; + const result = convertFigureToMarkdown(input, '', '/docs/'); + assert.strictEqual(result, input); +}); + +// --------------------------------------------------------------------------- +// convertInstallModularToMarkdown +// --------------------------------------------------------------------------- +console.log('\nconvertInstallModularToMarkdown:'); + +test('returns content unchanged when pluginContext is undefined', () => { + const input = ''; + assert.strictEqual(convertInstallModularToMarkdown(input, undefined), input); +}); + +test('returns content unchanged when template is not cached', () => { + const input = ''; + const ctx = { includeTemplates: {} }; + assert.strictEqual(convertInstallModularToMarkdown(input, ctx), input); +}); + +test('expands self-closing InstallModular when template is available', () => { + const template = [ + 'import Something from "somewhere";', + '', + 'export default function Install({ folder }) {', + ' return (', + '
', + ' Install into ${folder}', + '
', + ' );', + '}', + ].join('\n'); + const ctx = { includeTemplates: { 'install-modular': template } }; + const input = 'Before\n\nAfter'; + const result = convertInstallModularToMarkdown(input, ctx); + assert(!result.includes(' { + const input = ''; + assert.strictEqual(convertInstallOpenAIToMarkdown(input, undefined), input); +}); + +test('returns content unchanged when template is not cached', () => { + const input = ''; + const ctx = { includeTemplates: {} }; + assert.strictEqual(convertInstallOpenAIToMarkdown(input, ctx), input); +}); + +// --------------------------------------------------------------------------- +// unwrapMdxComponents — InstallModular fallback for non-self-closing usage +// --------------------------------------------------------------------------- +console.log('\nunwrapMdxComponents (InstallModular fallback):'); + +test('unwraps non-self-closing InstallModular preserving children', () => { + const input = 'Some child content'; + const result = unwrapMdxComponents(input); + assert(!result.includes(' { + const input = 'Before After'; + const result = unwrapMdxComponents(input); + assert(!result.includes(' { + const input = '---\ntitle: Test\n---\n\nHello world\n\n'; + const result = cleanMarkdownForDisplay(input, 'test.md', '/docs/'); + assert(result.includes('# Test')); + assert(result.includes('Hello world')); + assert(!result.includes(' { + const input = '---\ntitle: Test\n---\n\n\n\nMore text'; + const result = cleanMarkdownForDisplay(input, 'test.md', '/docs/', undefined, undefined); + assert(!result.includes(' { + const input = '---\ntitle: Test\n---\n\n\n\nMore text'; + const result = cleanMarkdownForDisplay(input, 'test.md', '/docs/'); + assert(!result.includes(' { + const input = '---\ntitle: Test\n---\n\n{`x = 1`}'; + const result = cleanMarkdownForDisplay(input, 'test.md'); + assert(result.includes('```python')); + assert(result.includes('x = 1')); +}); + +test('converts Admonition globally even without pluginContext', () => { + const input = '---\ntitle: Test\n---\n\n\nA tip.\n'; + const result = cleanMarkdownForDisplay(input, 'test.md'); + assert(result.includes('> **tip:**')); + assert(result.includes('A tip.')); +}); + +// --------------------------------------------------------------------------- +// Summary +// --------------------------------------------------------------------------- +console.log(`\nResults: ${passed} passed, ${failed} failed\n`); +process.exit(failed > 0 ? 1 : 0);