fix(site) Repair native reference context and tmux documentation

CONTRIBUTING.md

@@ -113,6 +113,19 @@ Chrome:
 $ LIBTMUX_DOCS_BROWSER_CHANNEL=chrome pnpm test
 ```
 
+To keep local browser checks on a fixed port, stop the preview using that
+port, then run:
+
+```console
+$ LIBTMUX_DOCS_BROWSER_PORT=53971 pnpm test
+```
+
+The value must be an integer from 1 to 65535. An occupied explicit port fails
+the check instead of selecting another port. This setting also applies to
+direct `check-dev.mjs` invocations, including `--serve`; without it, automated
+checks select an available port and `--serve` uses 53971. Restart the shared
+preview after the checks finish.
+
 Install the pinned Chromium build for local browser checks with
 `pnpm --filter @libtmux/site exec playwright install chromium`. CI installs
 that build and its system dependencies before the publication audit.

packages/api-model/src/builtins.ts

@@ -58,6 +58,7 @@ export const TYPESCRIPT_UTILITY_TYPES: Record<string, string> = Object.fromEntri
 )
 const kotlin = (page: string) => `https://kotlinlang.org/api/core/kotlin-stdlib/${page}/`
 const scala = (page: string) => `https://www.scala-lang.org/api/3.x/${page}.html`
+const scalaUsing = 'https://www.scala-lang.org/api/3.9.0/scala/util/Using$.html'
 const fsharp = (page: string) => `https://fsharp.github.io/fsharp-core-docs/reference/${page}.html`
 const coroutines = (page: string) => `https://kotlinlang.org/api/kotlinx.coroutines/kotlinx-coroutines-core/${page}`
 const catsEffect = (name: string) => `https://typelevel.org/cats-effect/api/3.x/cats/effect/kernel/${name}.html`
@@ -160,7 +161,9 @@ export const BUILTINS: Record<string, Record<string, string>> = {
     IterableOnce: scala('scala/collection/IterableOnce'),
     'scala.concurrent.duration.FiniteDuration': scala('scala/concurrent/duration/FiniteDuration'),
     ExecutionContext: scala('scala/concurrent/ExecutionContext'),
-    'scala.util.Using.resource': scala('scala/util/Using$'),
+    Using: scalaUsing, 'scala.util.Using': scalaUsing,
+    'Using.resource': `${scalaUsing}#resource-fffffb81`,
+    'scala.util.Using.resource': `${scalaUsing}#resource-fffffb81`,
     IO: 'https://typelevel.org/cats-effect/api/3.x/cats/effect/IO.html',
     Resource: 'https://typelevel.org/cats-effect/api/3.x/cats/effect/kernel/Resource.html',
   },

packages/api-model/src/doc/go.ts

@@ -0,0 +1,75 @@
+import type { Parser } from 'web-tree-sitter'
+import type { ParsedMarkdownDoc } from './markdown.ts'
+
+const LIST = /^[ \t]*(?:[-+*•]|\d+[.)])[ \t]+\S/
+const FENCE = /^ {0,3}(`{3,}|~{3,})([^`~]*)$/
+
+/** Native Go documentation removes a common literal space/tab prefix. */
+function unindent(source: string[]): string[] {
+  const lines = [...source]
+  while (lines.length && !lines[0].trim()) lines.shift()
+  while (lines.length && !lines.at(-1)!.trim()) lines.pop()
+  let prefix = /^[ \t]*/.exec(lines[0] ?? '')![0]
+  for (const line of lines.filter((line) => line.trim())) {
+    while (!line.startsWith(prefix)) prefix = prefix.slice(0, -1)
+  }
+  return lines.map((line) => line.trim() ? line.slice(prefix.length) : '')
+}
+
+/** Syntax classification preserves non-Go preformatted output as plain text. */
+function codeLanguage(code: string, parser: Parser): 'go' | 'text' {
+  for (const source of [`package doc\n${code}`, `package doc\nfunc example() {\n${code}\n}`]) {
+    const tree = parser.parse(source)
+    const valid = tree && !tree.rootNode.hasError
+    tree?.delete()
+    if (valid) return 'go'
+  }
+  return 'text'
+}
+
+/** Preserve native block boundaries in Markdown without moving code past prose. */
+export function parseGoDoc(raw: string, parser: Parser): ParsedMarkdownDoc {
+  const lines = unindent(raw.replaceAll('\r\n', '\n').split('\n'))
+  const blocks: { kind: 'prose' | 'other'; text: string }[] = []
+  for (let index = 0; index < lines.length;) {
+    const line = lines[index]
+    if (!line.trim()) { index++; continue }
+    const fence = FENCE.exec(line)
+    if (fence) {
+      const code = [fence[2].trim() ? line : `${fence[1]}go`]
+      const end = new RegExp(`^ {0,3}${fence[1][0]}{${fence[1].length},}\\s*$`)
+      while (++index < lines.length) {
+        code.push(lines[index])
+        if (end.test(lines[index])) { index++; break }
+      }
+      blocks.push({ kind: 'other', text: code.join('\n') })
+      continue
+    }
+    if (/^[ \t]/.test(line)) {
+      const indented: string[] = []
+      while (index < lines.length && (!lines[index].trim() || /^[ \t]/.test(lines[index]))) {
+        indented.push(lines[index++])
+      }
+      const content = unindent(indented).join('\n')
+      if (LIST.test(line)) {
+        blocks.push({ kind: 'other', text: content.replace(/^([ \t]*)•(?=[ \t])/gm, '$1-')
+          .replace(/^([ \t]*\d+)\)(?=[ \t])/gm, '$1.') })
+      } else {
+        const marker = '`'.repeat(Math.max(3, ...[...content.matchAll(/`+/g)].map((match) => match[0].length + 1)))
+        blocks.push({ kind: 'other', text: `${marker}${codeLanguage(content, parser)}\n${content}\n${marker}` })
+      }
+      continue
+    }
+    if (/^#\s+/.test(line)) {
+      blocks.push({ kind: 'other', text: line }); index++; continue
+    }
+    const prose: string[] = []
+    while (index < lines.length && lines[index].trim() && !/^[ \t]|^#\s+/.test(lines[index]) && !FENCE.test(lines[index])) {
+      prose.push(lines[index++])
+    }
+    blocks.push({ kind: LIST.test(line) ? 'other' : 'prose', text: prose.join('\n') })
+  }
+  const first = blocks[0]?.kind === 'prose' ? blocks.shift()!.text : ''
+  return { doc: { summary: first.replace(/\s+/g, ' ').trim(), body: blocks.map((block) => block.text).join('\n\n') || undefined },
+    params: new Map(), raises: [] }
+}

packages/api-model/src/doc/roles.ts

@@ -1,3 +1,5 @@
+import { fromMarkdown } from 'mdast-util-from-markdown'
+
 /**
  * reST cross-reference roles, which are most of the interlinking.
  *
@@ -57,6 +59,13 @@ export type DocSpan =
  *   "search upward", and the label drops it.
  */
 const ROLE_RE = /:([a-z:]+):`([^`]+)`/g
+/** Embedded reST hyperlinks, including anonymous links with two underscores. */
+const REST_LINK_RE = /`([^`<>]+?)(\s*)<((?:\\[\s\S]|[^\\<>`])+)>`_{1,2}/g
+/** reST distinguishes opening and closing ASCII delimiters from Unicode punctuation. */
+/* eslint-disable no-control-regex -- Docutils also treats U+001C through U+001F as whitespace. */
+const REST_LINK_START_RE = /(?:[\p{White_Space}\u001c-\u001f"'(<[{/:-]|(?=[^\x00-\x7f])[\p{Ps}\p{Pi}\p{Pf}\p{Pd}\p{Po}])$/u
+const REST_LINK_END_RE = /^(?:[\p{White_Space}\u001c-\u001f"')>\]}/:.,;!?\\-]|(?=[^\x00-\x7f])[\p{Pe}\p{Pi}\p{Pf}\p{Pd}\p{Po}])/u
+/* eslint-enable no-control-regex */
 
 /**
  * A citation reference: `[server_manual]_`.
@@ -169,9 +178,35 @@ const EMPHASIS_RE = /(?<![\w*\\])\*(?![\s*])([^*\n]+?)(?<![\s\\])\*(?![\w*])/g
 
 /** A bare double-backtick literal, which is never a link. */
 const LITERAL_RE = /``([^`]+)``/g
+/** reST literals can contain single backticks without enabling nested markup. */
+const REST_LITERAL_RE = /``((?:[^`]|`(?!`))+)``/g
 /** A single-backtick reference, which napoleon also resolves. */
 const BACKTICK_RE = /`([^`<>]+)`/g
 
+/** Match equal-length Markdown delimiters without interpreting their code as reST. */
+function markdownCode(text: string, masked: string): { start: number; end: number; text: string; ticks: number }[] {
+  const runs = [...masked.matchAll(/`+/g)]
+  const next = new Map<number, number>()
+  const closing = runs.map(() => -1)
+  for (let i = runs.length - 1; i >= 0; i--) {
+    closing[i] = next.get(runs[i][0].length) ?? -1
+    next.set(runs[i][0].length, i)
+  }
+  const spans: ReturnType<typeof markdownCode> = []
+  for (let i = 0; i < runs.length; i++) {
+    const open = runs[i]
+    let slashes = 0
+    for (let at = open.index - 1; at >= 0 && text[at] === '\\'; at--) slashes++
+    if (slashes % 2 || closing[i] < 0) continue
+    const close = runs[closing[i]]
+    let code = text.slice(open.index + open[0].length, close.index).replace(/\r\n?|\n/g, ' ')
+    if (code.startsWith(' ') && code.endsWith(' ') && /[^ ]/.test(code)) code = code.slice(1, -1)
+    spans.push({ start: open.index, end: close.index + close[0].length, text: code, ticks: open[0].length })
+    i = closing[i]
+  }
+  return spans
+}
+
 function labelFor(raw: string): { target: string; label: string } {
   const titled = /^(.*?)\s*<([^>]+)>$/.exec(raw.trim())
   if (titled) return { target: titled[2].trim(), label: titled[1].trim() }
@@ -213,7 +248,7 @@ const SYNTAX: Record<string, ReadonlySet<string>> = {
   rs: new Set(['intra-doc']),
   go: new Set(['go-link']),
   java: new Set(['jsdoc']),
-  ts: new Set(['jsdoc']),
+  ts: new Set(['jsdoc', 'markdown-link']),
   csharp: new Set(['xml-doc']),
   // DocC gives double backticks a meaning reST reserves for a literal: in a
   // Swift doc comment ``Server`` is a symbol link, not code.
@@ -249,20 +284,38 @@ export function tokenizeDoc(text: string, lang?: string): DocSpan[] {
   type Hit = { start: number; end: number; span: DocSpan }
   const hits: Hit[] = []
 
-  if (syntax.has('markdown-link')) {
-    // A code label can contain brackets (generic types, regexes, array values)
-    // and use a longer backtick fence when the code itself contains backticks.
-    for (const match of text.matchAll(/\[(?:(`+)([^\n]*?)\1|([^\]\n]+))\]\(([^\n)]+)\)/g)) {
-      const label = match[1] ? match[2] : match[3]
-      const target = match[4].trim()
-      if (syntax.has('documenter') && /^@ref(?:\s|$)/.test(target)) {
-        hits.push({ start: match.index, end: match.index + match[0].length,
-          span: { kind: 'ref', role: 'any', target: target.slice(4).trim() || label, label } })
-      } else if (/^(?:https?:\/\/|\/(?!\/)|#)/.test(target)) {
-        hits.push({ start: match.index, end: match.index + match[0].length,
-          span: { kind: 'link', text: label, href: target, ...(match[1] ? { code: true } : {}) } })
+  if (syntax.has('documenter')) {
+    // Documenter's named @ref destination contains spaces that Markdown URLs cannot.
+    for (const match of text.matchAll(/\[(?:(`+)([^\n]*?)\1|([^\]\n]+))\]\(@ref\s+([^\n)]+)\)/g)) {
+      hits.push({ start: match.index, end: match.index + match[0].length,
+        span: { kind: 'ref', role: 'any', target: match[4].trim(), label: match[1] ? match[2] : match[3] } })
+    }
+  }
+  if (syntax.has('markdown-link') && text.includes('](')) {
+    // Authored links own their labels, including generic code and balanced URI parentheses.
+    type MarkdownNode = ReturnType<typeof fromMarkdown> | ReturnType<typeof fromMarkdown>['children'][number]
+    const labelText = (node: MarkdownNode): string => 'value' in node ? node.value
+      : 'children' in node ? node.children.map((child) => labelText(child as MarkdownNode)).join('')
+        : 'alt' in node ? node.alt ?? '' : ''
+    const visit = (node: MarkdownNode) => {
+      if (node.type === 'link') {
+        const start = node.position!.start.offset!
+        const end = node.position!.end.offset!
+        const label = labelText(node)
+        const target = node.url
+        let span: DocSpan = { kind: 'text', text: text.slice(start, end) }
+        if (syntax.has('documenter') && /^@ref(?:\s|$)/.test(target)) {
+          span = { kind: 'ref', role: 'any', target: target.slice(4).trim() || label, label }
+        } else if (/^https?:\/\//i.test(target) || !/^(?:[a-z][\w+.-]*:|\/\/)/i.test(target)) {
+          span = { kind: 'link', text: label, href: target,
+            ...(node.children.length === 1 && node.children[0].type === 'inlineCode' ? { code: true } : {}) }
+        }
+        hits.push({ start, end, span })
+        return
       }
+      if ('children' in node) for (const child of node.children) visit(child as Parameters<typeof visit>[0])
     }
+    visit(fromMarkdown(text))
   }
 
   for (const match of text.matchAll(/<kbd>([^<\n]+)<\/kbd>/g)) {
@@ -392,35 +445,78 @@ export function tokenizeDoc(text: string, lang?: string): DocSpan[] {
       span: { kind: 'ref', role, target, label },
     })
   }
-  for (const m of text.matchAll(LITERAL_RE)) {
+  if (syntax.has('rest')) for (const m of text.matchAll(REST_LITERAL_RE)) {
     hits.push({
       start: m.index,
       end: m.index + m[0].length,
-      // DocC's ``Symbol`` is a link where reST's ``literal`` is code.
-      span: syntax.has('docc')
-        ? { kind: 'ref', role: 'any', target: m[1].replace(/\(.*$/, ''), label: m[1] }
-        : { kind: 'code', text: m[1] },
+      span: { kind: 'code', text: m[1] },
     })
   }
+  if (syntax.has('rest')) {
+    for (const match of text.matchAll(REST_LINK_RE)) {
+      const start = match.index
+      const end = start + match[0].length
+      if (hits.some((hit) => hit.start < end && hit.end > start)) continue
+      const rawLabel = match[1]
+      const separator = match[2]
+      const rawTarget = match[3]
+      let span: DocSpan = { kind: 'text', text: match[0] }
+      const delimited = (start === 0 || REST_LINK_START_RE.test(text.slice(Math.max(0, start - 2), start)))
+        && (end === text.length || REST_LINK_END_RE.test(text.slice(end, end + 2)))
+      /* eslint-disable no-control-regex -- Match Docutils whitespace and its NUL escape markers. */
+      const validWhitespace = !/^[\p{White_Space}\u001c-\u001f]/u.test(rawLabel)
+        && /^[ \t\r\n]+$/.test(separator)
+        && !/^[\p{White_Space}\u001c-\u001f]|[\p{White_Space}\u001c-\u001f]$/u.test(rawTarget)
+      if (delimited && validWhitespace) {
+        const label = rawLabel.replace(/\\([ \n])|\\([\s\S]?)/g, (_match, _space, char: string | undefined) => char ?? '')
+        // Docutils folds URI whitespace before removing native escape markers.
+        const target = rawTarget.replace(/\\([\s\S]?)/g, '\u0000$1').split(/\u0000[ \n]/)
+          .map((part) => part.replace(/[\p{White_Space}\u001c-\u001f]/gu, '')).join(' ')
+          .replaceAll('\u0000', '').replace(/\\_$/, '_')
+        try {
+          const destination = new URL(target)
+          if (/^https?:\/\//i.test(target) && (destination.protocol === 'http:' || destination.protocol === 'https:')) {
+            span = { kind: 'link', text: label, href: target }
+          }
+        } catch {
+          // Malformed destinations remain literal.
+        }
+      }
+      /* eslint-enable no-control-regex */
+      hits.push({ start, end, span })
+    }
+  }
   // Single-backtick references are scanned over a *masked* copy, with every
   // range already claimed by a role or a literal blanked out. Filtering
   // matches afterwards is not equivalent and quietly loses references: the
   // pairing is greedy and left-to-right, so an unclaimed backtick can pair
   // with one inside a role and swallow everything between them. In
   // ``:meth:`send keys <Pane.send_keys>` plus `Window`.`` that consumed the
   // `Window` reference entirely.
+  // Native tags can contain unmatched ticks; they cannot consume a later code opener.
   // Code units, not code points: `hits` are regex indices into the string.
   const masked = text.split('')
   for (const hit of hits) {
     for (let i = hit.start; i < hit.end; i++) masked[i] = ' '
   }
-  for (const m of masked.join('').matchAll(BACKTICK_RE)) {
-    const { target, label } = labelFor(m[1])
-    hits.push({
-      start: m.index,
-      end: m.index + m[0].length,
-      span: { kind: 'ref', role: 'any', target, label },
-    })
+  if (syntax.has('rest')) {
+    for (const m of masked.join('').matchAll(BACKTICK_RE)) {
+      const { target, label } = labelFor(m[1])
+      hits.push({
+        start: m.index,
+        end: m.index + m[0].length,
+        span: { kind: 'ref', role: 'any', target, label },
+      })
+    }
+  } else {
+    for (const code of markdownCode(text, masked.join(''))) {
+      const span: DocSpan = code.ticks === 1
+        ? { kind: 'ref', role: 'any', target: code.text, label: code.text }
+        : syntax.has('docc') && code.ticks === 2
+          ? { kind: 'ref', role: 'any', target: code.text.replace(/\(.*$/, ''), label: code.text }
+          : { kind: 'code', text: code.text }
+      hits.push({ start: code.start, end: code.end, span })
+    }
   }
 
   for (const match of text.matchAll(EMPHASIS_RE)) {

packages/api-model/src/javadoc.ts

@@ -0,0 +1,52 @@
+interface JavadocPackage {
+  l: string
+  m?: string
+}
+
+interface JavadocType {
+  l: string
+  p?: string
+  m?: string
+  u?: string
+}
+
+interface JavadocMember extends JavadocType {
+  c?: string
+}
+
+/** Convert Javadoc search indexes without guessing a package's module. */
+export function javadocSymbols(
+  packages: JavadocPackage[],
+  types: JavadocType[],
+  members: JavadocMember[],
+) {
+  const owners = new Map<string, Set<string>>()
+  for (const pkg of packages) {
+    if (!pkg.m) continue
+    const modules = owners.get(pkg.l) ?? new Set<string>()
+    modules.add(pkg.m)
+    owners.set(pkg.l, modules)
+  }
+  const moduleOf = (entry: JavadocType) => {
+    const modules = owners.get(entry.p!)
+    if (entry.m && modules?.has(entry.m)) return entry.m
+    if (!entry.m && modules?.size === 1) return [...modules][0]
+    throw new Error(`Javadoc package ${entry.p}: ${entry.m ? `unknown module ${entry.m}` : 'missing or ambiguous module'}`)
+  }
+  const symbols: { id: string; name: string; kind: 'class' | 'method'; uri: string }[] = []
+  for (const type of types) {
+    if (!type.p || !type.l) continue
+    const path = `${moduleOf(type)}/${type.p.replace(/\./g, '/')}/${type.u ?? `${type.l}.html`}`
+    symbols.push({ id: `${type.p}.${type.l}`, name: type.l, kind: 'class', uri: path })
+    symbols.push({ id: type.l, name: type.l, kind: 'class', uri: path })
+  }
+  for (const member of members) {
+    if (!member.p || !member.c || !member.l) continue
+    const bare = member.l.replace(/\(.*$/, '')
+    if (!bare || /[^A-Za-z0-9_$]/.test(bare)) continue
+    const path = `${moduleOf(member)}/${member.p.replace(/\./g, '/')}/${member.c}.html#${member.u ?? member.l}`
+    symbols.push({ id: `${member.c}.${bare}`, name: bare, kind: 'method', uri: path })
+    symbols.push({ id: `${member.p}.${member.c}.${bare}`, name: bare, kind: 'method', uri: path })
+  }
+  return symbols
+}