diff --git a/app/app.config.ts b/app/app.config.ts index d9c0554..b90a00f 100644 --- a/app/app.config.ts +++ b/app/app.config.ts @@ -185,7 +185,7 @@ export default defineAppConfig({ // info: 'i-tabler-info-square-rounded-filled', }, }, - // `seo.siteName`, `header.title` and `github.*` are deliberately NOT defaulted here: modules/config.ts seeds + // `seo.siteName`, `header.title` and `github.*` are deliberately NOT defaulted here: modules/config/ seeds // them into `nuxt.options.appConfig`, and app.config values — even empty strings — would win over those. header: { to: '/', diff --git a/app/app.vue b/app/app.vue index a508c15..4798242 100644 --- a/app/app.vue +++ b/app/app.vue @@ -1,6 +1,5 @@ + + diff --git a/app/components/AssistantChat.vue b/app/components/AssistantChat.vue index 4adcc67..591c212 100644 --- a/app/components/AssistantChat.vue +++ b/app/components/AssistantChat.vue @@ -3,7 +3,7 @@ import { DefaultChatTransport, isReasoningUIPart, isTextUIPart, isToolUIPart, ge import { useChat } from '@ai-sdk/vue' import { isPartStreaming, isToolStreaming } from '@nuxt/ui/utils/ai' import rangi from 'comark/plugins/rangi' -import { geistTheme } from '../../utils/geist-theme' +import { geistTheme } from '../../utils/geist' const MAX_INPUT = 1000 diff --git a/app/components/landing/LandingHeroDemo.vue b/app/components/landing/LandingHeroDemo.vue index da3aaef..96cddfa 100644 --- a/app/components/landing/LandingHeroDemo.vue +++ b/app/components/landing/LandingHeroDemo.vue @@ -1,6 +1,6 @@ @@ -32,11 +29,6 @@ provide('navigation', navigation) - - - + diff --git a/app/utils/navigation.ts b/app/utils/navigation.ts index dc718d0..3f14e41 100644 --- a/app/utils/navigation.ts +++ b/app/utils/navigation.ts @@ -23,7 +23,7 @@ function walk(items: NavigationItem[], path: string): boolean { } // Shared with the server-side `/raw/**` mirror (server/routes/raw/[...slug].md.get.ts). -export { findFirstLeaf } from '../../utils/first-leaf' +export { findFirstLeaf } from '../../utils/navigation' export interface BreadcrumbItem { title: string diff --git a/app/utils/search-sections.ts b/app/utils/search-sections.ts deleted file mode 100644 index 4aa65d9..0000000 --- a/app/utils/search-sections.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { defineContentClientPlugin } from 'comark-content/client' -import { joinURL } from 'ufo' - -/** One search entry per document heading — consumed by `UContentSearch`. */ -export interface SearchSection { - id: string - title: string - titles: string[] - level: number - content: string -} - -interface SearchSectionsClientMethods { - searchSections(): Promise -} - -/** Client half of the `search-sections` serve handler (`server/utils/content.ts`); adds `content.searchSections()`. */ -export const searchSectionsClient = defineContentClientPlugin, SearchSectionsClientMethods>(() => ({ - name: 'search-sections', - setup: ({ options }) => ({ - searchSections: () => options.fetch(joinURL(options.baseURL, options.basePath, 'search-sections')), - }), -})) diff --git a/app/workers/internal/search-logger.ts b/app/workers/internal/search-logger.ts new file mode 100644 index 0000000..8917943 --- /dev/null +++ b/app/workers/internal/search-logger.ts @@ -0,0 +1,69 @@ +/** + * Logging for the search worker. + * + * Triggered by `?debug=search` param. + */ +import type { ContentFile, Logger, RelationalDatabase } from 'comark-content' + +const PREFIX = '[search:worker]' + +let debug = false + +/** Called on every `warmup`; once on, it stays on for the life of the worker. */ +export function setDebug(value: boolean): void { + debug = debug || value +} + +export function isDebug(): boolean { + return debug +} + +export function log(...args: unknown[]): void { + if (debug) console.info(PREFIX, ...args) +} + +/** Milliseconds since `from`, for log lines. */ +export function since(from: number): string { + return `${(performance.now() - from).toFixed(1)}ms` +} + +/** + * Warn and error are deliberately ungated: the FTS plugin reports a missing snapshot through this + * channel, and that failure is otherwise indistinguishable from "the query matched nothing". + */ +export const logger: Logger = { + debug: (tag, ...args) => log(`${tag}:`, ...args), + info: (tag, ...args) => log(`${tag}:`, ...args), + warn: (tag, ...args) => console.warn(`${PREFIX} ${tag}:`, ...args), + error: (tag, ...args) => console.error(`${PREFIX} ${tag}:`, ...args), +} + +/** + * What a decoded artifact holds: a snapshot decodes to the source's items, the manifest to an object + * keyed by path. `with nodes` is the number that matters — the FTS plugin indexes + * `kind === 'document' && nodes?.length`, so a bodies-less (partial) snapshot builds an empty index. + */ +export function describeArtifact(decoded: unknown): string { + if (Array.isArray(decoded)) { + const items = decoded as ContentFile[] + const documents = items.filter((item) => item.meta.kind === 'document') + const withNodes = documents.filter((item) => item.nodes?.length) + return `${items.length} item(s), ${documents.length} document(s), ${withNodes.length} with nodes` + } + const items = (decoded as { items?: Record } | null)?.items + return `${items ? Object.keys(items).length : 0} manifest item(s)` +} + +/** + * Rows in the FTS plugin's index — the one number that separates "nothing was indexed" from "the + * query found nothing", since `search()` catches SQL errors and returns `[]` either way. Reads the + * plugin's private table, so it is a diagnostic, not something to build on. + */ +export async function indexedRows(database: RelationalDatabase, source: string): Promise { + try { + const rows = await database.all<{ n: number }>('SELECT count(*) as n FROM __fts_search WHERE source = ?', [source]) + return rows?.[0]?.n ?? 'unknown' + } catch (error) { + return `unknown (${error instanceof Error ? error.message : String(error)})` + } +} diff --git a/app/workers/search.ts b/app/workers/search.ts new file mode 100644 index 0000000..46d4394 --- /dev/null +++ b/app/workers/search.ts @@ -0,0 +1,118 @@ +/** + * Search worker: owns the browser-standalone `comark-content` instance (sqlite-wasm FTS5). + * + * Hydrated from the per-commit snapshot artifacts. + */ +import { comarkContent, DEFAULT_CONTENT_NAME, readArtifact } from 'comark-content' +import sqliteWasm from 'comark-content/database/sqlite-wasm' +import snapshot from 'comark-content/sources/snapshot' +import sqliteFullTextSearch from 'comark-content/plugins/sqlite-full-text-search' +import { ofetch } from 'ofetch' +import { describeArtifact, indexedRows, isDebug, log, logger, setDebug, since } from './internal/search-logger' +import type { CacheArtifact, SearchOptions, SearchResult } from 'comark-content' + +/** + * Factored out so `SearchInstance` can be derived from its return type instead of annotated — + * `ComarkContent`'s instance-name parameter reaches `get()`'s argument type, so a bare + * `ComarkContent & SqliteFullTextSearchMethods` annotation isn't a supertype of a concrete + * instance (fails under `strictFunctionTypes`, same reason as `DocsContent` in + * `server/utils/content.ts`). + */ +function createSearchInstance(fetchArtifact: (path: string) => Promise, apiBase: string) { + const database = sqliteWasm() + return { + database, + content: comarkContent({ + // The first (full-body) tier is what the index is built from; the second (manifest) tier + // is the light one `init()` prefers, so a bare `init()` below doesn't download bodies that + // `search()` is about to fetch anyway via the snapshot tier. + source: snapshot( + () => fetchArtifact(`${apiBase}/snapshot/${DEFAULT_CONTENT_NAME}.json`), + () => fetchArtifact(`${apiBase}/manifest.json`) + ), + plugins: [sqliteFullTextSearch({ database })], + logger, + }), + } +} + +type SearchInstance = ReturnType['content'] + +let instance: SearchInstance | undefined + +/** + * The in-flight hydration. + * + * Ensures only one hydration runs at a time. + */ +let hydration: Promise | undefined + +/** Loads the database. No-op once ready; retries after a failure. */ +export function warmupSearch(apiBase: string, origin: string, debug: boolean): Promise { + setDebug(debug) + if (instance) { + log('warmup ignored — already ready') + return Promise.resolve() + } + hydration ||= loadDatabase(apiBase, origin).catch((error) => { + hydration = undefined // clears the guard so the next warmup can retry + throw error + }) + return hydration +} + +async function loadDatabase(apiBase: string, origin: string): Promise { + const started = performance.now() + try { + const fetchArtifact = async (path: string): Promise => { + const url = new URL(path, origin).href + const fetchStarted = performance.now() + try { + const artifact = await ofetch(url) + if (isDebug()) { + let contents: string + try { + contents = describeArtifact(await readArtifact(artifact)) + } catch (error) { + contents = `undecodable: ${error instanceof Error ? error.message : String(error)}` + } + log(`fetched ${path} in ${since(fetchStarted)} — ${artifact?.size ?? 0} bytes, ${contents}`) + } + return artifact + } catch (error) { + log(`failed ${path} after ${since(fetchStarted)}`, error) + throw error + } + } + + const { database, content } = createSearchInstance(fetchArtifact, apiBase) + + await content.init() + + const indexStarted = performance.now() + await content.search('') // pulls the snapshot in and builds the FTS index + log(`index built in ${since(indexStarted)} — ${await indexedRows(database, DEFAULT_CONTENT_NAME)} row(s)`) + + instance = content + log(`ready in ${since(started)}`) + } catch (error) { + log(`hydration failed after ${since(started)}`, error) + throw error + } +} + +/** Empty until hydration lands. */ +export async function searchContent(query: string, opts?: SearchOptions): Promise { + if (!instance) { + log(`dropped query "${query}" — no instance yet`) + return [] + } + const queryStarted = performance.now() + const results = await instance.search(query, { + limit: 25, + snippet: { columns: ['content'] }, + ...opts, + }) + log(`query "${query}" -> ${results.length} result(s) in ${since(queryStarted)}`) + return results +} diff --git a/modules/config.ts b/modules/config/index.ts similarity index 90% rename from modules/config.ts rename to modules/config/index.ts index 35aaa70..dda9f24 100644 --- a/modules/config.ts +++ b/modules/config/index.ts @@ -1,10 +1,9 @@ import { existsSync, readdirSync } from 'node:fs' import { defineNuxtModule, useLogger } from '@nuxt/kit' import { defu } from 'defu' -import { resolveContentDir } from '../utils/content-dir' -import { getGitBranch, getGitEnv, getGitRoot, getLocalGitInfo } from '../utils/git' -import { LAYER_ICON_COLLECTIONS } from '../utils/icons' -import { getPackageJsonMetadata, inferSiteURL } from '../utils/meta' +import { getGitBranch, getGitEnv, getGitRoot, getLocalGitInfo } from '../../utils/git' +import { LAYER_ICON_COLLECTIONS } from '../../utils/icons' +import { getPackageJsonMetadata, inferSiteURL, resolveContentDir } from './utils' const logger = useLogger('comark-docs') @@ -160,7 +159,7 @@ export default defineNuxtModule({ // Previews are served live (SSR) off Runtime Cache; `/blob/**` is immutable commit HTML. // `/pr/**` follows the PR's head like `/tree/**` follows a branch, so it shares the short TTL. '/tree/**': { isr, robots: 'noindex, nofollow' }, - '/blob/**': { isr: true, robots: 'noindex, nofollow' }, + '/blob/**': { isr: true, robots: 'noindex, nofollow' }, // Immutable since SHA-pinned '/pr/**': { isr, robots: 'noindex, nofollow' }, // Raw markdown mirrors of every page, for agents. '/raw/**': { isr, robots: 'noindex' }, @@ -168,11 +167,14 @@ export default defineNuxtModule({ '/llms.txt': { isr }, '/llms-full.txt': { isr }, '/rss.xml': { isr }, - // Fetched on every page hydration (see app.vue) and parses every doc body, so cache it. - '/api/content/blob/*/search-sections': { isr: true }, - '/api/content/tree/*/search-sections': { isr }, - '/api/content/pr/*/search-sections': { isr }, - '/api/content/search-sections': { isr }, + // Per-commit artifacts hydrating the client-side search database (see `useSearch`) + '/api/content/blob/*/manifest.json': { isr: true }, // Immutable since SHA-pinned + '/api/content/blob/*/snapshot/*': { isr: true }, // Immutable since SHA-pinned + '/api/content/tree/*/manifest.json': { isr }, + '/api/content/tree/*/snapshot/*': { isr }, + // `/pr/*` follows the PR head, so it gets the short TTL like `/tree/*`. + '/api/content/pr/*/manifest.json': { isr }, + '/api/content/pr/*/snapshot/*': { isr }, '/api/code-explorer/**': { isr }, '/_payload.json': { headers: { 'cache-control': `public, max-age=${isr}, s-maxage=${isr}, stale-while-revalidate=60` }, diff --git a/test/content-dir.test.ts b/modules/config/test/config.test.ts similarity index 60% rename from test/content-dir.test.ts rename to modules/config/test/config.test.ts index 993b7cf..be00c54 100644 --- a/test/content-dir.test.ts +++ b/modules/config/test/config.test.ts @@ -1,5 +1,5 @@ -import { describe, expect, it } from 'vitest' -import { resolveContentDir } from '../utils/content-dir' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { inferSiteURL, resolveContentDir } from '../utils' describe('resolveContentDir', () => { it('relativises against the git root for an app in a subdirectory', () => { @@ -64,3 +64,54 @@ describe('resolveContentDir', () => { }) }) }) + +describe('inferSiteURL', () => { + const keys = [ + 'NUXT_PUBLIC_SITE_URL', + 'NUXT_SITE_URL', + 'VERCEL_PROJECT_PRODUCTION_URL', + 'VERCEL_BRANCH_URL', + 'VERCEL_URL', + 'URL', + 'CI_PAGES_URL', + 'CF_PAGES_URL', + ] + let saved: Record + + // `Reflect.deleteProperty` rather than `delete process.env[key]`: same effect, + // without tripping `no-dynamic-delete`. + const unset = (key: string) => Reflect.deleteProperty(process.env, key) + + beforeEach(() => { + saved = Object.fromEntries(keys.map((key) => [key, process.env[key]])) + for (const key of keys) unset(key) + }) + + afterEach(() => { + for (const [key, value] of Object.entries(saved)) { + if (value === undefined) unset(key) + else process.env[key] = value + } + }) + + it('returns undefined when nothing is set', () => { + expect(inferSiteURL()).toBeUndefined() + }) + + it('adds https to a bare Vercel host', () => { + process.env.VERCEL_URL = 'my-app-abc123.vercel.app' + expect(inferSiteURL()).toBe('https://my-app-abc123.vercel.app') + }) + + it('prefers the explicit override over the platform value', () => { + process.env.VERCEL_URL = 'my-app-abc123.vercel.app' + process.env.NUXT_PUBLIC_SITE_URL = 'https://docs.example.com' + expect(inferSiteURL()).toBe('https://docs.example.com') + }) + + it('prefers the production URL over the per-branch one', () => { + process.env.VERCEL_BRANCH_URL = 'branch.vercel.app' + process.env.VERCEL_PROJECT_PRODUCTION_URL = 'docs.comark.dev' + expect(inferSiteURL()).toBe('https://docs.comark.dev') + }) +}) diff --git a/utils/content-dir.ts b/modules/config/utils.ts similarity index 61% rename from utils/content-dir.ts rename to modules/config/utils.ts index 1dcc9f1..f0c884c 100644 --- a/utils/content-dir.ts +++ b/modules/config/utils.ts @@ -1,4 +1,6 @@ -import { join, normalize, relative } from 'pathe' +import { readFile } from 'node:fs/promises' +import { join, normalize, relative, resolve } from 'pathe' +import { withHttps } from 'ufo' export interface ContentDirInput { rootDir: string @@ -37,3 +39,28 @@ export function resolveContentDir({ rootDir, gitRoot, explicit }: ContentDirInpu return { contentPath, contentDir: 'content', source: 'assumed' } } + +/** Infer the public site URL from the deployment platform env. */ +export function inferSiteURL(): string | undefined { + // https://github.com/unjs/std-env/issues/59 + const url = + process.env.NUXT_PUBLIC_SITE_URL || + process.env.NUXT_SITE_URL || + process.env.VERCEL_PROJECT_PRODUCTION_URL || + process.env.VERCEL_BRANCH_URL || + process.env.VERCEL_URL || + process.env.URL || // Netlify + process.env.CI_PAGES_URL || // GitLab Pages + process.env.CF_PAGES_URL // Cloudflare Pages + + return url ? withHttps(url) : undefined +} + +export async function getPackageJsonMetadata(dir: string): Promise<{ name?: string; description?: string }> { + try { + const parsed = JSON.parse(await readFile(resolve(dir, 'package.json'), 'utf-8')) + return { name: parsed.name, description: parsed.description } + } catch { + return {} + } +} diff --git a/modules/markdown-rewrite.ts b/modules/markdown-rewrite/index.ts similarity index 95% rename from modules/markdown-rewrite.ts rename to modules/markdown-rewrite/index.ts index 463a1a0..c1513e4 100644 --- a/modules/markdown-rewrite.ts +++ b/modules/markdown-rewrite/index.ts @@ -1,7 +1,7 @@ import { readFile, writeFile } from 'node:fs/promises' import { defineNuxtModule, useLogger } from '@nuxt/kit' import { resolve } from 'pathe' -import { buildMarkdownRewriteRoutes } from '../utils/markdown-rewrite' +import { buildMarkdownRewriteRoutes } from './utils' const logger = useLogger('comark-docs') diff --git a/test/markdown-rewrite.test.ts b/modules/markdown-rewrite/test/markdown-rewrite.test.ts similarity index 99% rename from test/markdown-rewrite.test.ts rename to modules/markdown-rewrite/test/markdown-rewrite.test.ts index 107845c..2b1db74 100644 --- a/test/markdown-rewrite.test.ts +++ b/modules/markdown-rewrite/test/markdown-rewrite.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest' -import { buildMarkdownRewriteRoutes, type VercelRoute } from '../utils/markdown-rewrite' +import { buildMarkdownRewriteRoutes, type VercelRoute } from '../utils' // Vercel resolves `$n` in `headers.Location` from the capture groups of `src` — replicate that to // assert on the final redirect target rather than on regex internals. diff --git a/utils/markdown-rewrite.ts b/modules/markdown-rewrite/utils.ts similarity index 100% rename from utils/markdown-rewrite.ts rename to modules/markdown-rewrite/utils.ts diff --git a/modules/snapshot/index.ts b/modules/snapshot/index.ts new file mode 100644 index 0000000..64ee422 --- /dev/null +++ b/modules/snapshot/index.ts @@ -0,0 +1,73 @@ +import { cp, mkdir } from 'node:fs/promises' +import { defineNuxtModule, useLogger } from '@nuxt/kit' +import { writeSnapshots } from 'comark-content' +import fs from 'comark-content/sources/fs' +import { join } from 'pathe' +import { createBuildContentInstance } from '../../utils/content' +import { resolveSeedRefs } from './utils' +import { getPinnedSha } from '../../server/utils/global-config' + +const logger = useLogger('comark-docs') + +/** Where the seed lives in the build, and the server-asset namespace it is read back through. */ +const ASSET_BASE = 'comark-content' + +/** + * Writes a build-time content seed into the function bundle, so a cold start hydrates from the generated snapshot. + */ +export default defineNuxtModule({ + meta: { name: 'comark-docs:snapshot' }, + setup(_options, nuxt) { + // Do not run in dev or prepare. + if (nuxt.options.dev || nuxt.options._prepare) return + + const dir = join(nuxt.options.buildDir, ASSET_BASE) + + nuxt.hook('modules:done', async () => { + await mkdir(dir, { recursive: true }) + nuxt.options.nitro.serverAssets = [ + ...(nuxt.options.nitro.serverAssets ?? []), + { baseName: ASSET_BASE, dir }, + ] + }) + + nuxt.hook('build:before', async () => { + const { docs } = nuxt.options.runtimeConfig + const { repoRoot, contentDir, contentPath, github } = docs + + const refs = await resolveSeedRefs({ + repoRoot, + contentDir, + repo: `${github.owner}/${github.repo}`, + token: docs.githubToken || process.env.GITHUB_TOKEN, + pinnedSha: await getPinnedSha(), + warn: (message) => logger.warn(message), + }) + if (!refs.length) { + logger.warn( + 'No commit in this checkout could be confirmed to hold the content being built, ' + + 'so no seed is shipped — cold starts will walk the content repository.' + ) + return + } + + // A throwaway instance over the local files, sharing the runtime's parser: a seed parsed by + // a different plugin set is silently different content, not a cache miss. + const content = createBuildContentInstance({ source: fs(contentPath) }) + + try { + const [primary, ...rest] = refs as [string, ...string[]] + await writeSnapshots(content, { dir: join(dir, primary) }) + // Copied rather than re-written: `writeSnapshots()` reparses from the source each time, and + // every ref here was verified to hold the same content anyway. + for (const ref of rest) await cp(join(dir, primary), join(dir, ref), { recursive: true }) + + logger.success(`Content seed: ${refs.map((ref) => ref.slice(0, 7)).join(', ')}`) + } catch (error) { + // Never fail the build over an optimization. An empty asset directory reads as "no seed" + // and the deployment falls back to GitHub. + logger.warn('Could not write the content seed — cold starts will walk the content repository.', error) + } + }) + }, +}) diff --git a/modules/snapshot/test/snapshot.test.ts b/modules/snapshot/test/snapshot.test.ts new file mode 100644 index 0000000..98849e8 --- /dev/null +++ b/modules/snapshot/test/snapshot.test.ts @@ -0,0 +1,124 @@ +import { execFileSync } from 'node:child_process' +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { resolveSeedRefs } from '../utils' + +const SHA = (char: string) => char.repeat(40) + +describe('resolveSeedRefs', () => { + let repo: string + let contentCommit: string + let head: string + + const run = (...args: string[]) => + execFileSync('git', args, { cwd: repo, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim() + + const write = async (file: string, body: string) => { + await mkdir(dirname(join(repo, file)), { recursive: true }) + await writeFile(join(repo, file), body, 'utf8') + } + + /** Answers the commits query with `sha` per requested ref; `null` means 404. */ + function stubApi(bySha: Record) { + return vi.fn(async (url: string | URL) => { + const ref = new URL(String(url)).searchParams.get('sha') ?? '' + const answer = bySha[ref] + if (answer === undefined || answer === null) return new Response('[]', { status: 404 }) + return new Response(JSON.stringify([{ sha: answer }]), { status: 200 }) + }) + } + + beforeEach(async () => { + repo = await mkdtemp(join(tmpdir(), 'comark-seedrefs-')) + run('init', '-q', '-b', 'main') + run('config', 'user.email', 'test@example.com') + run('config', 'user.name', 'Test') + + await write('content/index.md', '# one\n') + run('add', '-A') + run('commit', '-qm', 'add content') + contentCommit = run('rev-parse', 'HEAD') + + // A later commit that leaves `content/` alone, so HEAD is not the last content commit. + await write('src/app.ts', 'export const a = 1\n') + run('add', '-A') + run('commit', '-qm', 'add code') + head = run('rev-parse', 'HEAD') + }) + + afterEach(async () => { + await rm(repo, { recursive: true, force: true }) + vi.unstubAllGlobals() + }) + + const input = () => ({ repoRoot: repo, contentDir: 'content', repo: 'owner/name', token: 'tok' }) + + it('walks from the built commit, not the branch', async () => { + // The distinction that keeps a mid-build push (or a redeploy of an older commit) from labelling + // the seed with content it does not hold. + const fetchMock = stubApi({ [head]: contentCommit, main: SHA('f') }) + vi.stubGlobal('fetch', fetchMock) + + expect(await resolveSeedRefs(input())).toEqual([contentCommit]) + + const requested = new URL(String(fetchMock.mock.calls[0]![0])).searchParams + expect(requested.get('sha')).toBe(head) + expect(requested.get('path')).toBe('content') + expect(requested.get('per_page')).toBe('1') + }) + + it('adds the pin when it resolves to the same content commit', async () => { + const pinnedSha = SHA('a') + vi.stubGlobal('fetch', stubApi({ [head]: contentCommit, [pinnedSha]: contentCommit })) + + expect(await resolveSeedRefs({ ...input(), pinnedSha })).toEqual([contentCommit, pinnedSha]) + }) + + it('drops a pin that resolves elsewhere', async () => { + // A pin on older content: the seed holds this build's content, so it must not be labelled with it. + const pinnedSha = SHA('a') + vi.stubGlobal('fetch', stubApi({ [head]: contentCommit, [pinnedSha]: SHA('b') })) + + expect(await resolveSeedRefs({ ...input(), pinnedSha })).toEqual([contentCommit]) + }) + + it('falls back to a tree-verified git answer when the API fails', async () => { + vi.stubGlobal('fetch', stubApi({})) + + // Full history here, so git finds the true commit and its content tree matches HEAD's. + expect(await resolveSeedRefs(input())).toEqual([contentCommit]) + }) + + it('ships nothing when neither the API nor git can name the content', async () => { + vi.stubGlobal('fetch', stubApi({})) + + expect(await resolveSeedRefs({ ...input(), contentDir: 'nope' })).toEqual([]) + }) + + it('warns on the git fallback when the answer is a shallow boundary', async () => { + vi.stubGlobal('fetch', stubApi({})) + const warn = vi.fn() + + // A one-commit repo: its only commit is parentless, which is what a depth-1 clone looks like. + const shallow = await mkdtemp(join(tmpdir(), 'comark-shallow-')) + try { + const at = (...args: string[]) => execFileSync('git', args, { cwd: shallow, stdio: 'ignore' }) + at('init', '-q', '-b', 'main') + at('config', 'user.email', 'test@example.com') + at('config', 'user.name', 'Test') + await mkdir(join(shallow, 'content'), { recursive: true }) + await writeFile(join(shallow, 'content/index.md'), '# one\n', 'utf8') + at('add', '-A') + at('commit', '-qm', 'init') + + const refs = await resolveSeedRefs({ ...input(), repoRoot: shallow, warn }) + expect(refs).toHaveLength(1) + expect(warn).toHaveBeenCalledOnce() + expect(warn.mock.calls[0]![0]).toContain('shallow clone boundary') + } finally { + await rm(shallow, { recursive: true, force: true }) + } + }) +}) diff --git a/modules/snapshot/utils.ts b/modules/snapshot/utils.ts new file mode 100644 index 0000000..09ccd33 --- /dev/null +++ b/modules/snapshot/utils.ts @@ -0,0 +1,79 @@ +import { getLastCommit, getTreeSha, hasParent, headCommit } from '../../utils/git' +import { fetchLastContentCommit } from '../../utils/github' + +export interface SeedRefsInput { + /** Repository root of the checkout being built. */ + repoRoot: string + /** Content directory, relative to the repository root. */ + contentDir: string + /** `owner/name` of the content repository. */ + repo: string + /** GitHub token, if the build has one. Without it only the git fallback runs. */ + token?: string + /** Global Config pin, if one is set — production serves it in preference to the branch head. */ + pinnedSha?: string + /** Reported to the caller; defaults to `console.warn`. */ + warn?: (message: string) => void +} + +/** {@link fetchLastContentCommit}, but never throwing: a build-time optimization must not fail a build. */ +async function lastContentCommit( + repo: string, + contentDir: string, + ref: string, + token?: string +): Promise { + try { + const sha = await fetchLastContentCommit({ repo, path: contentDir, ref, token }) + // Validated here rather than in the shared query: this one names a directory in the build. + return sha && /^[0-9a-f]{40}$/.test(sha) ? sha : undefined + } catch { + return undefined + } +} + +/** + * The refs a build-time content seed may be stored under: commits whose `contentDir` holds exactly + * the content being parsed. + * + * Aligned with `resolveContentSha()` runs at runtime. + */ +export async function resolveSeedRefs(input: SeedRefsInput): Promise { + const { repoRoot, contentDir, repo, token, pinnedSha } = input + const warn = input.warn ?? ((message: string) => console.warn(message)) + + const head = headCommit(repoRoot) + const refs = new Set() + + const fromApi = head && repo ? await lastContentCommit(repo, contentDir, head, token) : undefined + if (fromApi) { + refs.add(fromApi) + + // The pin earns a label only if its content is the same tree — same question, asked of the pin's + // own history. A pin on older content resolves elsewhere and is dropped. + if (pinnedSha && pinnedSha !== fromApi) { + const fromPin = await lastContentCommit(repo, contentDir, pinnedSha, token) + if (fromPin === fromApi) refs.add(pinnedSha) + } + return [...refs] + } + + // No API answer: fall back to git, which needs the tree check to be trustworthy. + const parsed = getTreeSha(repoRoot, 'HEAD', contentDir) + const fromGit = getLastCommit(repoRoot, contentDir) + if (!parsed || !fromGit) return [] + + if (getTreeSha(repoRoot, fromGit, contentDir) !== parsed) return [] + + if (!hasParent(repoRoot, fromGit)) { + warn( + `Could not reach the GitHub API, and git labels the content seed ${fromGit.slice(0, 7)}, ` + + `which has no parent in this checkout — a shallow clone boundary.\n` + + ` The seed is safe, but probably will not be looked up under that commit at runtime.` + ) + } + + refs.add(fromGit) + if (pinnedSha && getTreeSha(repoRoot, pinnedSha, contentDir) === parsed) refs.add(pinnedSha) + return [...refs] +} diff --git a/nuxt.config.ts b/nuxt.config.ts index ee65cd7..e73e9a1 100644 --- a/nuxt.config.ts +++ b/nuxt.config.ts @@ -14,6 +14,7 @@ export default defineNuxtConfig({ 'nuxt-og-image', '@nuxtjs/mcp-toolkit', 'nuxt-llms', + 'nuxt-workers', ], ignore: ['content/**'], ui: { content: true, prose: true }, @@ -33,12 +34,15 @@ export default defineNuxtConfig({ resolve: { alias: { 'beautiful-mermaid': resolveModulePath('beautiful-mermaid', { from: import.meta.url }) }, }, + worker: { format: 'es' }, optimizeDeps: { include: [ 'beautiful-mermaid', 'comark-docs > ai > @ai-sdk/gateway > @vercel/oidc', 'js-yaml' ], + // Pre-bundling would break the wasm/worker assets sqlite loads relative to its module URL. + exclude: ['@sqlite.org/sqlite-wasm'], }, }, nitro: { diff --git a/package.json b/package.json index fe53c21..d7e2bdf 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "@octokit/webhooks-methods": "^6.0.0", "@opentelemetry/api": "^1.9.1", "@resvg/resvg-js": "^2.6.2", + "@sqlite.org/sqlite-wasm": "3.53.0-build1", "@vercel/analytics": "^2.0.1", "@vercel/functions": "^3.9.5", "@vercel/global-config": "^1.5.1", @@ -58,7 +59,7 @@ "ai": "^7.0.77", "beautiful-mermaid": "^1.1.3", "comark": "^0.6.2", - "comark-content": "https://pkg.pr.new/comark-content@baefd4d", + "comark-content": "https://pkg.pr.new/comark-content@63ffc3f", "defu": "^6.1.7", "exsolve": "^1.1.1", "js-yaml": "^5.3.0", @@ -66,6 +67,7 @@ "nuxt-llms": "^0.2.0", "nuxt-og-image": "^6.7.8", "nuxt-seo-utils": "^8.4.2", + "nuxt-workers": "^0.1.0", "pathe": "^2.0.3", "rangi": "^2.2.0", "satori": "^0.29.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 40b81d2..a06fe2c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -56,6 +56,9 @@ importers: '@resvg/resvg-js': specifier: ^2.6.2 version: 2.6.2 + '@sqlite.org/sqlite-wasm': + specifier: 3.53.0-build1 + version: 3.53.0-build1 '@vercel/analytics': specifier: ^2.0.1 version: 2.0.1(nuxt@4.5.2(@babel/plugin-syntax-jsx@7.29.7(@babel/core@7.29.7(supports-color@10.2.2)))(@babel/plugin-syntax-typescript@7.29.7(@babel/core@7.29.7(supports-color@10.2.2)))(@oxc-project/types@0.146.0)(@types/node@26.2.0)(@vercel/functions@3.9.5(ws@8.21.3))(@vue/compiler-sfc@3.5.41)(db0@0.3.4)(esbuild@0.28.2)(eslint@10.9.0(jiti@2.7.0)(supports-color@10.2.2))(ioredis@5.11.1(supports-color@10.2.2))(lightningcss@1.33.0)(magicast@0.5.4)(optionator@0.9.4)(oxc-parser@0.143.0)(rolldown@1.2.5)(rollup-plugin-visualizer@7.1.1(rolldown@1.2.5)(rollup@4.62.5))(rollup@4.62.5)(srvx@0.11.22)(supports-color@10.2.2)(terser@5.50.0)(typescript@6.0.3)(vite@8.2.2(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(yaml@2.9.0))(vue-tsc@3.3.11(typescript@6.0.3))(yaml@2.9.0))(vue@3.5.41(typescript@6.0.3)) @@ -84,8 +87,8 @@ importers: specifier: ^0.6.2 version: 0.6.2(beautiful-mermaid@1.1.3)(rangi@2.2.0)(shiki@4.4.3) comark-content: - specifier: https://pkg.pr.new/comark-content@baefd4d - version: https://pkg.pr.new/comark-content@baefd4d(@vercel/functions@3.9.5(ws@8.21.3))(beautiful-mermaid@1.1.3)(db0@0.3.4)(ioredis@5.11.1(supports-color@10.2.2))(rangi@2.2.0)(shiki@4.4.3) + specifier: https://pkg.pr.new/comark-content@63ffc3f + version: https://pkg.pr.new/comark-content@63ffc3f(@vercel/functions@3.9.5(ws@8.21.3))(beautiful-mermaid@1.1.3)(db0@0.3.4)(ioredis@5.11.1(supports-color@10.2.2))(rangi@2.2.0)(shiki@4.4.3) defu: specifier: ^6.1.7 version: 6.1.7 @@ -107,6 +110,9 @@ importers: nuxt-seo-utils: specifier: ^8.4.2 version: 8.4.2(6c2732a7424285fd39f1e74aab3b66f7) + nuxt-workers: + specifier: ^0.1.0 + version: 0.1.0(magicast@0.5.4) pathe: specifier: ^2.0.3 version: 2.0.3 @@ -1173,6 +1179,10 @@ packages: '@nuxt/icon@2.5.1': resolution: {integrity: sha512-zBP72Po7BS+tXzoeDRA/Y9TTY77OIcNCyzfgXsOmep7zZShTEoe4p1WZBXce/9oJDK3YXmbYC3ILQ7XvqM4/XA==} + '@nuxt/kit@3.21.11': + resolution: {integrity: sha512-0Xi3tgwN77w43Q8GCPIrvWmF1J7Peehkts44E0uKNIml9lB8WoUn8YxyUjxBv47XtVR86NWoWALAT+/IEMHJEA==} + engines: {node: '>=18.12.0'} + '@nuxt/kit@4.5.2': resolution: {integrity: sha512-l66LU9DcJYjmNwqwAj2I5UGRrUbnG2DOKGChnN70zIGtn0eq/z87gi/FRgha6eMb9/FmB1PFHgtx6PWVml1C2Q==} engines: {node: '>=18.12.0'} @@ -2069,6 +2079,10 @@ packages: '@speed-highlight/core@1.2.24': resolution: {integrity: sha512-qeW2e1l78afw8VhRPfPQ1Gjj+KU5XFQ/OFV5ti6eTa9bruO7mJyZtA4vw0ofqmA3tKCkROE9xLk3VZoeRc98nw==} + '@sqlite.org/sqlite-wasm@3.53.0-build1': + resolution: {integrity: sha512-PfWPWN2n+/37doa8oh2/oUXk4OOsRYZsxc1W1sDXIGb/Pu5Yrb+f2eyYpgQMGITVX7HVgxhs9P18Rc6I97ym/g==} + engines: {node: '>=22'} + '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} @@ -3335,8 +3349,8 @@ packages: colortranslator@5.0.0: resolution: {integrity: sha512-Z3UPUKasUVDFCDYAjP2fmlVRf1jFHJv1izAmPjiOa0OCIw1W7iC8PZ2GsoDa8uZv+mKyWopxxStT9q05+27h7w==} - comark-content@https://pkg.pr.new/comark-content@baefd4d: - resolution: {integrity: sha512-7E/3OIIKPBI5XJ3s/aEtptAP49Of9W/EQVjzsbGnAZgCxvtlpqeSq1l+8GUbMDYqNy20rLWXlk7JdXTd8rsj/g==, tarball: https://pkg.pr.new/comark-content@baefd4d} + comark-content@https://pkg.pr.new/comark-content@63ffc3f: + resolution: {integrity: sha512-F0jDyRaJqfASydFGik1UpAlfqa9SmrDV+Sj2QfjoO6vJHtH8xZiAf0XSmOFfPcGI5QMRKnrqqyZy1ei5RJ6IWQ==, tarball: https://pkg.pr.new/comark-content@63ffc3f} version: 0.3.0 hasBin: true @@ -5050,6 +5064,9 @@ packages: peerDependencies: vue: ^3.5.30 + nuxt-workers@0.1.0: + resolution: {integrity: sha512-npsxy72FRQZkxHV1Y+KCkuSvb2Y/7Tcp7xGXPHXXVQ5/oIZ5+69VAudfdhpuwPN1JZJn9ULr01vKRLamENsTew==} + nuxt@4.5.2: resolution: {integrity: sha512-tR3fcqeHlHmmkLMpIg3V7Y+1ltr302lW8djMw/iy+myfo7QSSz+BVJDuQhg5j73b9oteSyBfOKTDYTgvMtj6TA==} engines: {node: ^22.19.0 || ^24.11.0 || >=26.0.0} @@ -7856,6 +7873,32 @@ snapshots: - vite - vue + '@nuxt/kit@3.21.11(magicast@0.5.4)': + dependencies: + c12: 3.3.4(magicast@0.5.4) + consola: 3.4.2 + defu: 6.1.7 + destr: 2.0.5 + errx: 0.1.2 + exsolve: 1.1.1 + ignore: 7.0.6 + jiti: 2.7.0 + klona: 2.0.6 + knitwork: 1.3.0 + mlly: 1.8.2 + ohash: 2.0.12 + pathe: 2.0.3 + pkg-types: 2.3.1 + rc9: 3.0.1 + scule: 1.3.0 + semver: 7.8.5 + tinyglobby: 0.2.17 + ufo: 1.6.4 + unctx: 2.5.0 + untyped: 2.0.0 + transitivePeerDependencies: + - magicast + '@nuxt/kit@4.5.2(magic-string@1.2.3)(magicast@0.5.4)(oxc-parser@0.143.0)(rolldown@1.2.5)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.5)(rollup@4.62.5)(vite@8.2.2(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(yaml@2.9.0)))': dependencies: c12: 3.3.4(magicast@0.5.4) @@ -8864,6 +8907,8 @@ snapshots: '@speed-highlight/core@1.2.24': {} + '@sqlite.org/sqlite-wasm@3.53.0-build1': {} + '@standard-schema/spec@1.1.0': {} '@stylistic/eslint-plugin@5.10.0(eslint@10.9.0(jiti@2.7.0)(supports-color@10.2.2))': @@ -10093,7 +10138,7 @@ snapshots: colortranslator@5.0.0: {} - comark-content@https://pkg.pr.new/comark-content@baefd4d(@vercel/functions@3.9.5(ws@8.21.3))(beautiful-mermaid@1.1.3)(db0@0.3.4)(ioredis@5.11.1(supports-color@10.2.2))(rangi@2.2.0)(shiki@4.4.3): + comark-content@https://pkg.pr.new/comark-content@63ffc3f(@vercel/functions@3.9.5(ws@8.21.3))(beautiful-mermaid@1.1.3)(db0@0.3.4)(ioredis@5.11.1(supports-color@10.2.2))(rangi@2.2.0)(shiki@4.4.3): dependencies: citty: 0.2.2 comark: 0.6.2(beautiful-mermaid@1.1.3)(rangi@2.2.0)(shiki@4.4.3) @@ -12010,6 +12055,17 @@ snapshots: - vite - zod + nuxt-workers@0.1.0(magicast@0.5.4): + dependencies: + '@nuxt/kit': 3.21.11(magicast@0.5.4) + magic-string: 0.30.21 + mlly: 1.8.2 + pathe: 2.0.3 + ufo: 1.6.4 + unplugin: 2.3.11 + transitivePeerDependencies: + - magicast + nuxt@4.5.2(@babel/plugin-syntax-jsx@7.29.7(@babel/core@7.29.7(supports-color@10.2.2)))(@babel/plugin-syntax-typescript@7.29.7(@babel/core@7.29.7(supports-color@10.2.2)))(@oxc-project/types@0.146.0)(@types/node@26.2.0)(@vercel/functions@3.9.5(ws@8.21.3))(@vue/compiler-sfc@3.5.41)(db0@0.3.4)(esbuild@0.28.2)(eslint@10.9.0(jiti@2.7.0)(supports-color@10.2.2))(ioredis@5.11.1(supports-color@10.2.2))(lightningcss@1.33.0)(magicast@0.5.4)(optionator@0.9.4)(oxc-parser@0.143.0)(rolldown@1.2.5)(rollup-plugin-visualizer@7.1.1(rolldown@1.2.5)(rollup@4.62.5))(rollup@4.62.5)(srvx@0.11.22)(supports-color@10.2.2)(terser@5.50.0)(typescript@6.0.3)(vite@8.2.2(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(yaml@2.9.0))(vue-tsc@3.3.11(typescript@6.0.3))(yaml@2.9.0): dependencies: '@dxup/nuxt': 0.5.10(esbuild@0.28.2)(magicast@0.5.4)(oxc-parser@0.143.0)(rolldown@1.2.5)(rollup@4.62.5)(vite@8.2.2(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(yaml@2.9.0)) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 669fb7c..8a4e480 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -17,7 +17,7 @@ minimumReleaseAgeExclude: - '@comark/nuxt@0.6.0 || 0.6.1 || 0.6.2' - '@comark/vue@0.6.0 || 0.6.1 || 0.6.2' - comark@0.6.0 || 0.6.1 || 0.6.2 - - comark-content@0.3.0 + - comark-content@0.4.0 overrides: # Keep the workspace on a single h3 major (v1), matching comark-content. diff --git a/server/api/code-explorer/[...path].get.ts b/server/api/code-explorer/[...path].get.ts index 6edb4bb..1bd84ed 100644 --- a/server/api/code-explorer/[...path].get.ts +++ b/server/api/code-explorer/[...path].get.ts @@ -3,7 +3,7 @@ import { parseMarkdown, type MarkdownDocument } from 'comark' import rangi from 'comark/plugins/rangi' import fs from 'comark-content/sources/fs' import github from 'comark-content/sources/github' -import { geistTheme } from '../../../utils/geist-theme.ts' +import { geistTheme } from '../../../utils/geist.ts' // A read source for one example directory. Dev: working tree. Prod: authenticated GitHub — the repo may be // private, so jsDelivr / unauthenticated raw are out. Mirrors {@link contentSource}'s dev/prod split. diff --git a/server/api/content/[...path].get.ts b/server/api/content/[...path].get.ts index e10db39..c4f39d7 100644 --- a/server/api/content/[...path].get.ts +++ b/server/api/content/[...path].get.ts @@ -1,6 +1,6 @@ /** - * Single data endpoint: `content.handler()` dispatches `get`, `navigation`, `list` and custom handlers - * (e.g. `search-sections`). Cached per-URL — see `routeRules`. + * Single data endpoint: `content.handler()` dispatches `get`, `navigation`, `list`, `manifest` + * and `snapshot`. Must be cached per-URL by layer consumer. */ export default defineEventHandler(async (event) => { const content = await getProdContent() diff --git a/server/api/content/blob/[sha]/[...path].get.ts b/server/api/content/blob/[sha]/[...path].get.ts index a72ef7f..623d140 100644 --- a/server/api/content/blob/[sha]/[...path].get.ts +++ b/server/api/content/blob/[sha]/[...path].get.ts @@ -20,6 +20,19 @@ export default defineEventHandler(async (event) => { // Also resolves short SHAs so one commit pins one content instance. const fullSha = await authorizePreviewSha(sha) + // Head-of-branch requests reuse the shared prod instance (same source ref, same per-SHA cache + // namespace) instead of minting a duplicate preview instance that would pin an LRU slot with a + // clone of production. Re-checked after `getProdContent()`, which may advance the head. + if (fullSha === getHeadRef()) { + const prod = await getProdContent() + if (fullSha === getHeadRef()) { + const request = toWebRequest(event) + const url = new URL(request.url) + url.pathname = url.pathname.replace(`/blob/${rawSha}`, '') + return await prod.handler(new Request(url, request)) + } + } + const content = await getPreviewContent(fullSha, `/api/content/blob/${sha}`) return await content.handler(toWebRequest(event)) diff --git a/server/api/content/head.get.ts b/server/api/content/head.get.ts new file mode 100644 index 0000000..493c5bb --- /dev/null +++ b/server/api/content/head.get.ts @@ -0,0 +1,10 @@ +/** + * The commit SHA production content is pinned to, or `null` in dev. + */ +export default defineEventHandler(async () => { + if (import.meta.dev) return { sha: null } + + // Same resolution as the pages (`getProdContent`), so the search artifacts the client hydrates + // from can't come from a different commit than the rendered content — notably under a pin. + return { sha: await resolveProdSha() } +}) diff --git a/server/api/revalidate.post.ts b/server/api/revalidate.post.ts index 51de3d5..4d3ff88 100644 --- a/server/api/revalidate.post.ts +++ b/server/api/revalidate.post.ts @@ -1,30 +1,25 @@ -import type { ContentListFile } from 'comark-content' import { verify } from '@octokit/webhooks-methods' +import { DEFAULT_CONTENT_NAME } from 'comark-content' import { waitUntil } from '@vercel/functions' /** Each re-render hits this same deployment, so the ceiling is about not stampeding ourselves. */ const REVALIDATE_CONCURRENCY = 8 -/** `Promise.allSettled` over `items`, at most `size` in flight. */ -async function settleInBatches( - items: T[], - size: number, - fn: (item: T) => Promise -): Promise[]> { - const results: PromiseSettledResult[] = [] - for (let i = 0; i < items.length; i += size) { - // Not `.map(fn)` — `map` passes the index, which lands in the callee's optional parameter. - results.push(...(await Promise.allSettled(items.slice(i, i + size).map((item) => fn(item))))) - } - return results -} +/** Why a route was purged — one value per `addPath` call site below. */ +type PurgeReason = 'page' | 'payload' | 'raw' | 'nav' | 'global' + +/** Display/response order. */ +const REASON_ORDER: PurgeReason[] = ['page', 'payload', 'raw', 'nav', 'global'] + +/** `nav` is the only unbounded reason (the whole site can be thousands of pages) — cap what the log prints. */ +const MAX_LOGGED_PATHS_PER_REASON = 5 export default defineEventHandler(async (event) => { const { docs } = useRuntimeConfig(event) const secret = docs.webhookSecret || process.env.WEBHOOK_SECRET const bypassToken = docs.bypassToken || process.env.VERCEL_BYPASS_TOKEN if (!secret || !bypassToken) { - throw createError({ statusCode: 500, statusMessage: 'Webhook not configured' }) + throw createError({ statusCode: 501, statusMessage: 'Revalidation webhook is not configured' }) } const signature = getHeader(event, 'x-hub-signature-256') @@ -41,181 +36,220 @@ export default defineEventHandler(async (event) => { throw createError({ statusCode: 401, statusMessage: 'Invalid signature' }) } + const requestId = getHeader(event, 'x-vercel-id') ?? getHeader(event, 'x-request-id') ?? 'local' + const deliveryId = getHeader(event, 'x-github-delivery') + const tag = `[revalidate:${requestId}${deliveryId ? `:${deliveryId}` : ''}]` + const timings = createTimings() + + const githubEvent = getHeader(event, 'x-github-event') + if (githubEvent !== 'push') { + console.log(`${tag} skipped: ${githubEvent ?? 'unknown'} event`) + return { ok: true, skipped: 'not-a-push-event', event: githubEvent } + } + const payload = JSON.parse(raw) as GitHubPushPayload const branch = targetBranch() const contentDir = docs.contentDir const expectedRef = `refs/heads/${branch}` - if (payload.ref !== expectedRef) { - console.log(`[content] revalidate push skipped (ref=${payload.ref} !== expected=${expectedRef})`) - return { - ok: true, - skipped: 'non-target-branch', - expected: expectedRef, - received: payload.ref, - } + const repo = githubRepo() + if (payload.repository?.full_name && payload.repository.full_name !== repo) { + console.log(`${tag} skipped: repo=${payload.repository.full_name} !== expected=${repo}`) + return { ok: true, skipped: 'wrong-repo', expected: repo, received: payload.repository.full_name } } - // Classify changed content files. A file added in one commit and modified in - // another counts as added; `.navigation.*` config files always touch navigation. - const added = new Set() - const removed = new Set() - const modified = new Set() - let navConfigTouched = false - for (const commit of payload.commits ?? []) { - for (const f of commit.added ?? []) { - if (isContentMd(f)) added.add(f) - else if (isNavConfig(f)) navConfigTouched = true - } - for (const f of commit.modified ?? []) { - if (isContentMd(f)) modified.add(f) - else if (isNavConfig(f)) navConfigTouched = true - } - for (const f of commit.removed ?? []) { - if (isContentMd(f)) removed.add(f) - else if (isNavConfig(f)) navConfigTouched = true - } + if (payload.ref !== expectedRef) { + console.log(`${tag} skipped: ref=${payload.ref} !== expected=${expectedRef}`) + return { ok: true, skipped: 'non-target-branch', expected: expectedRef, received: payload.ref } } - for (const f of added) modified.delete(f) - const changedFiles = [...added, ...modified, ...removed] - if (changedFiles.length === 0 && !navConfigTouched) { + const changes = changesForPush(contentDir, payload.commits ?? []) + if (!changes.upserted.length && !changes.removed.length && !changes.navTouched) { return { ok: true, skipped: 'no-content-changes' } } - const protocol = getRequestProtocol(event) - const host = getRequestHost(event, { xForwardedHost: true }) - const baseURL = `${protocol}://${host}` + const buildId = useRuntimeConfig(event).app.buildId + const pathsToPurge = new Set() + const byReason = new Map>() - // `x-vercel-protection-bypass` bypasses the SSO wall when the handler calls itself - const readHeaders: Record = {} - if (process.env.VERCEL_AUTOMATION_BYPASS_SECRET) { - readHeaders['x-vercel-protection-bypass'] = process.env.VERCEL_AUTOMATION_BYPASS_SECRET + /** Add a path to the purge set, and track it by reason for the breakdown log. */ + const addPath = (reason: PurgeReason, path: string): void => { + if (pathsToPurge.has(path)) return + pathsToPurge.add(path) + const paths = byReason.get(reason) ?? new Set() + paths.add(path) + byReason.set(reason, paths) } - // `x-prerender-revalidate` purges the ISR cache - const headers: Record = { - ...readHeaders, - 'x-prerender-revalidate': bypassToken, - } + // Diffed against the live prod instance, already warm + const { headSha, newItems, pagePaths, navChanged } = await timings.time('rebuild', async () => { + const outdated = await getProdContent() + await outdated.init() + const oldItems = { ...(await outdated.manifest()).items } - const headSha = payload.head_commit?.id - if (!headSha) { - throw createError({ statusCode: 400, statusMessage: 'Missing head commit SHA' }) - } + // Refresh the content SHA + const headSha = await resolveContentSha(branch, contentDir, { refresh: true }) + const freshContent = await createSourceContent(headSha, { cache: { driver: cacheDriver(headSha) } }) + // Partial init: the diff needs the index (cache will be reused by the warm below) + await freshContent.init() + const newItems = (await freshContent.manifest()).items - // Bypass the short ref cache and write the canonical path-filtered revision before the purge fan-out, - // so a freshly-purged page cannot re-render against a stale or payload-order-dependent content SHA. - const contentSha = await resolveContentSha(branch, contentDir, { refresh: true }) + return { headSha, newItems, ...diffContent(changes, oldItems, newItems) } + }) - console.log(`[content] revalidate push headSha=${headSha} contentSha=${contentSha}`) + for (const path of pagePaths) { + addPath('page', path) + addPath('payload', payloadUrlForPage(path, buildId)) + addPath('raw', rawUrlForPage(path)) + } - const requestId = getHeader(event, 'x-vercel-id') ?? getHeader(event, 'x-request-id') ?? 'local' - const tag = `[revalidate:${requestId}]` - - // Planning before we respond costs two `init()` passes against GitHub's ~10s delivery timeout, - // in exchange for diagnostics in the webhook body. Safe because everything here is idempotent, - // so a retried delivery only repeats work. If it gets slow, move this into `waitUntil`. - const beforeSha = payload.before - let oldItems: Record = {} - if (beforeSha && !/^0+$/.test(beforeSha)) { - try { - const oldContent = await createSourceContent(beforeSha) - await oldContent.init() - oldItems = (await oldContent.manifest()).items - } catch (err) { - const message = err instanceof Error ? err.message : err - console.warn(`${tag} no before-manifest (${beforeSha}) — treating as full revalidate:`, message) + // Navigation renders on every page, so a change to it re-renders all of them. + if (navChanged) { + for (const item of Object.values(newItems)) { + if (item.meta.kind !== 'document') continue + addPath('nav', item.path) + addPath('nav', payloadUrlForPage(item.path, buildId)) + addPath('nav', rawUrlForPage(item.path)) } } - // The head snapshot has the same content directory as `contentSha`; populate the namespace that - // production instances will read even when later commits in this push only changed code. - const headContent = await createSourceContent(headSha, { cache: { driver: cacheDriver(contentSha) } }) - await headContent.init() - const newItems = (await headContent.manifest()).items - - const oldPaths = new Set(Object.keys(oldItems)) - const newPaths = Object.keys(newItems) - const addedPaths = newPaths.filter((p) => !oldPaths.has(p)) - const removedPaths = [...oldPaths].filter((p) => !(p in newItems)) + // Per-commit search artifacts (ISR, immutable). + const artifactBase = `/api/content/blob/${headSha}` + const pathsToWarm = [`${artifactBase}/manifest.json`, `${artifactBase}/snapshot/${DEFAULT_CONTENT_NAME}.json`] - const metaChangedPaths: string[] = [] - for (const p of newPaths) { - if (oldPaths.has(p) && hashManifestItem(oldItems[p]) !== hashManifestItem(newItems[p])) metaChangedPaths.push(p) + // Any content change invalidates the global indexes: each is rebuilt from the whole tree. + for (const path of ['/llms.txt', '/llms-full.txt', '/rss.xml', '/sitemap.xml']) { + addPath('global', path) } - const navChanged = navConfigTouched || addedPaths.length > 0 || removedPaths.length > 0 || metaChangedPaths.length > 0 - // Payload routes are keyed by the build-id query on some deployments, so purge the exact - // URL the browser loads (`…/_payload.json?`). - const buildId = useRuntimeConfig(event).app.buildId + console.log( + `${tag} navChanged=${navChanged} ` + + `(upserted=${changes.upserted.length}, removed=${changes.removed.length}, navConfig=${changes.navTouched}) | ` + + `${pathsToPurge.size} to purge, ${pathsToWarm.length} to warm | ${timings.format()}` + ) - // Any content change invalidates the llms indexes, the feed, and the body-derived search index. - const paths = new Set(['/llms.txt', '/llms-full.txt', '/rss.xml', '/api/content/search-sections']) - for (const f of changedFiles) { - const pageUrl = pageUrlForPath(f) - if (pageUrl) { - paths.add(payloadUrlForRoute(pageUrl, buildId)) - paths.add(pageUrl) - } - const rawUrl = rawUrlForPath(f) - if (rawUrl) paths.add(rawUrl) - } + logBreakdown(tag, byReason) + for (const path of pathsToWarm) console.log(`${tag} warm\t${path}`) - // Navigation renders on every page, so a change to it re-renders all of them. - if (navChanged) { - for (const item of Object.values(newItems)) { - if (item.meta.kind === 'document') { - paths.add(item.path) - paths.add(payloadUrlForRoute(item.path, buildId)) - } + // Dev has no ISR cache to purge + if (import.meta.dev) { + return { + ok: true, + requestId, + deliveryId, + navChanged, + routes: routesBreakdown(byReason), + warm: pathsToWarm.length, + dev: true, } } - console.log( - `${tag} navChanged=${navChanged} ` + - `(added=${addedPaths.length}, removed=${removedPaths.length}, meta=${metaChangedPaths.length}, navConfig=${navConfigTouched}) | ` + - `files: +${added.size} ~${modified.size} -${removed.size} | ${paths.size} route(s)` - ) - if (metaChangedPaths.length) console.log(`${tag} meta changed: ${metaChangedPaths.join(', ')}`) - if (addedPaths.length) console.log(`${tag} added: ${addedPaths.join(', ')}`) - if (removedPaths.length) console.log(`${tag} removed: ${removedPaths.join(', ')}`) + const protocol = getRequestProtocol(event) + const host = getRequestHost(event, { xForwardedHost: true }) + const baseURL = `${protocol}://${host}` + + // Lets the deployment call itself while Vercel Authentication is on (preview deploys). + const selfCall: Record = process.env.VERCEL_AUTOMATION_BYPASS_SECRET + ? { 'x-vercel-protection-bypass': process.env.VERCEL_AUTOMATION_BYPASS_SECRET } + : {} + + // `x-prerender-revalidate` regenerates the ISR entry for the URL being fetched. + const purgeHeaders = { ...selfCall, 'x-prerender-revalidate': bypassToken } // Vercel's native waitUntil, not Nitro's `event.waitUntil` — that one can orphan async work here. waitUntil( (async () => { - const revalidate = (path: string, extra: Record = {}) => - $fetch(path, { baseURL, method: 'GET', headers: { ...headers, ...extra } }).catch((err) => { - console.error(`${tag} ✗ ${path}`, err?.statusCode ?? err?.message ?? err) - throw err - }) - - // Warm the per-SHA body cache so cold instances skip re-parsing from GitHub. - // `metaOnly` became `partial` in comark-content 0.2.0 with no alias and consumers straddle both, - // so send both keys — each version ignores the other's. Not inlined: as a literal, - // excess-property checking rejects whichever key the installed types don't declare. - const full = { partial: false, metaOnly: false } - await headContent.init(full).catch((err) => { - console.error(`${tag} cache warm failed`, err?.message ?? err) - }) - - await useStorage('cache:nuxt:payload').clear() - - // Bounded: a nav change queues two URLs per page, and every one re-enters this function. - const results = await settleInBatches([...paths], REVALIDATE_CONCURRENCY, revalidate) - const ok = results.filter((r) => r.status === 'fulfilled').length - console.log(`${tag} complete: ${ok}/${results.length} succeeded`) + const absent: string[] = [] + + // The warm runs first: + // - ISR cache manifest and snapshot for the new SHA + // - Cache parsed items for the pages to purge and re-render + const warmResults = await timings.time('warm', () => + settleInBatches(pathsToWarm, REVALIDATE_CONCURRENCY, (path) => + $fetch(path, { baseURL, method: 'GET', headers: selfCall }).catch((error) => { + console.error(`${tag} ✗ ${path}`, error?.statusCode ?? error?.message ?? error) + throw error + }) + ) + ) + + const purgeResults = await timings.time('purge', () => + settleInBatches([...pathsToPurge], REVALIDATE_CONCURRENCY, (path) => + $fetch(path, { baseURL, method: 'GET', headers: purgeHeaders }).catch((error) => { + // Content with no page of its own (e.g. a partial) has nothing cached to purge. + if (error?.statusCode === 404) { + absent.push(path) + return + } + console.error(`${tag} ✗ ${path}`, error?.statusCode ?? error?.message ?? error) + throw error + }) + ) + ) + + const warmed = warmResults.filter((r) => r.status === 'fulfilled').length + const failed = [...warmResults, ...purgeResults].filter((r) => r.status === 'rejected').length + const purged = purgeResults.filter((r) => r.status === 'fulfilled').length - absent.length + console.log( + `${tag} complete: ${warmed} warmed, ${purged} purged, ${absent.length} absent, ` + + `${failed} failed | ${timings.format()} | total=${timings.since()}ms` + ) + logAbsent(tag, absent) })() ) return { ok: true, requestId, + deliveryId, navChanged, - manifest: { - added: addedPaths, - removed: removedPaths, - metaChanged: metaChangedPaths, - }, + manifest: { upserted: changes.upserted, removed: changes.removed }, + routes: routesBreakdown(byReason), } }) + +/** One log line per purged path, grouped by reason. */ +function logBreakdown(tag: string, byReason: Map>): void { + for (const reason of REASON_ORDER) { + const paths = byReason.get(reason) + if (!paths?.size) continue + + const sorted = [...paths].sort() + for (const path of sorted.slice(0, MAX_LOGGED_PATHS_PER_REASON)) { + console.log(`${tag} ${reason}\t${path}`) + } + if (sorted.length > MAX_LOGGED_PATHS_PER_REASON) { + console.log(`${tag} ${reason}\t... (${sorted.length} total)`) + } + } +} + +/** One log line per absent path — expected to be empty. */ +function logAbsent(tag: string, absent: string[]): void { + for (const path of [...absent].sort()) { + console.log(`${tag} absent\t${path}`) + } +} + +/** Route counts by reason. */ +function routesBreakdown(byReason: Map>): { total: number } & Partial> { + const counts: Partial> = {} + for (const [reason, paths] of byReason) counts[reason] = paths.size + + const total = Object.values(counts).reduce((sum, count) => sum + (count ?? 0), 0) + return { total, ...counts } +} + +/** `Promise.allSettled` over `items`, at most `size` in flight. */ +async function settleInBatches( + items: T[], + size: number, + fn: (item: T) => Promise +): Promise[]> { + const results: PromiseSettledResult[] = [] + for (let i = 0; i < items.length; i += size) { + // Not `.map(fn)` — `map` passes the index, which lands in the callee's optional parameter. + results.push(...(await Promise.allSettled(items.slice(i, i + size).map((item) => fn(item))))) + } + return results +} diff --git a/server/routes/raw/[...slug].md.get.ts b/server/routes/raw/[...slug].md.get.ts index a871320..ef71111 100644 --- a/server/routes/raw/[...slug].md.get.ts +++ b/server/routes/raw/[...slug].md.get.ts @@ -1,4 +1,4 @@ -import { findFirstLeaf } from '../../../utils/first-leaf' +import { findFirstLeaf } from '../../../utils/navigation' export default defineEventHandler(async (event) => { const slug = getRouterParams(event)['slug.md'] diff --git a/server/utils/content.ts b/server/utils/content.ts index 862de45..2a80b1d 100644 --- a/server/utils/content.ts +++ b/server/utils/content.ts @@ -1,15 +1,8 @@ -import { defineContentPlugin, type CacheOptions, comarkContent } from 'comark-content'; +import { type CacheOptions, type ContentSource, DEFAULT_CONTENT_NAME } from 'comark-content' import fs from 'comark-content/sources/fs' import github from 'comark-content/sources/github' -import rangi from 'comark/plugins/rangi' -import security from 'comark/plugins/security' -import emoji from 'comark/plugins/emoji' -import toc from 'comark/plugins/toc' -import mermaid from 'comark/plugins/mermaid' -import yaml from 'comark-content/plugins/yaml' -import tracingOtel from 'comark-content/plugins/tracing/otel' -import { contentTracer } from './tracer.ts' -import { geistTheme } from '../../utils/geist-theme.ts' +import { withSnapshot } from 'comark-content/sources/snapshot' +import { createRuntimeContentInstance } from '../../utils/content.ts' /** * The instance this layer builds, derived from the factory rather than written @@ -27,48 +20,19 @@ export type DocsContent = Awaited> // assignment lands after the await, so two requests on a cold instance would each build a CMS. let content: Promise | undefined -// Bump CONTENT_PARSER_VERSION in `cache.ts` when these plugins or their options change cached output. -const comarkPlugins = [ - mermaid({ theme: 'zinc-light', themeDark: 'zinc-dark' }), - rangi({ theme: geistTheme }), - toc({ depth: 3 }), - emoji(), - security({ - blockedTags: ['script', 'iframe', 'embed', 'form', 'base', 'meta', 'link', 'style'], - allowDataImages: false, - }), -] - -// Bound to THIS instance so a preview content instance serves its own version's sections, not production's. -const searchSectionsPlugin = defineContentPlugin(() => ({ - name: 'search-sections', - setup(ctx) { - ctx.addServeHandler('search-sections', async () => Response.json(await buildSearchSections(ctx as unknown as DocsContent))) - }, -}))() - /** - * Create a new content instance reading content at `ref` (a commit SHA or branch). `remote` forces the - * GitHub source, `cache` overrides comark's (in-memory by default), `watch` is dev file watching. + * Create a new content instance reading content at `ref` (a commit SHA or branch). + * - `remote` forces the GitHub source + * - `cache` overrides comark's (in-memory by default) + * - `basePath` is the base path for the content instance + * - `watch` is dev file watching */ export async function createSourceContent( ref: string, opts: { remote?: boolean; cache?: CacheOptions; basePath?: string; watch?: boolean } = {} ) { - // A no-op unless the consumer shadows it from their own `server/utils/`. Re-typed as the layer's own - // options: a consumer's hook is declared against the wide `ContentOptions`, and letting that widen the - // argument would erase the source and plugin types `comarkContent` infers from the literal. - const tracer = contentTracer() - const instance = comarkContent({ - markdown: { - plugins: comarkPlugins, - }, + const instance = createRuntimeContentInstance({ source: contentSource(ref, { remote: opts.remote }), - plugins: [ - yaml(), // enable .navigation.yml to be detected - searchSectionsPlugin, - tracer && tracingOtel({ tracer }), - ], cache: opts.cache, basePath: opts.basePath, }) @@ -140,7 +104,7 @@ export async function getProdContent(): Promise { return content } -function contentSource(ref: string, opts: { remote?: boolean } = {}) { +function contentSource(ref: string, opts: { remote?: boolean } = {}): ContentSource { const { docs } = useRuntimeConfig() if (import.meta.dev) { @@ -149,7 +113,7 @@ function contentSource(ref: string, opts: { remote?: boolean } = {}) { return fs(docs.contentPath) } - return github({ + const source = github({ repo: githubRepo(), branch: ref, path: docs.contentDir, @@ -157,6 +121,11 @@ function contentSource(ref: string, opts: { remote?: boolean } = {}) { // `ref` is an immutable commit SHA => we can cache hard. ttl: 60 * 60 * 24, }) + + // The snapshot shipped during build by `modules/snapshot/`. + return withSnapshot(source, () => + useStorage('assets:comark-content').get(`${ref}/${DEFAULT_CONTENT_NAME}/snapshot.json`) + ) } /** Per-instance registry of preview CMS instances, keyed by `::`. */ diff --git a/server/utils/github.ts b/server/utils/github.ts index eb995a5..684e009 100644 --- a/server/utils/github.ts +++ b/server/utils/github.ts @@ -1,5 +1,6 @@ import { createHash, timingSafeEqual } from 'node:crypto' import { createStorage } from 'unstorage' +import { fetchLastContentCommit } from '../../utils/github' export interface GitHubCommit { added?: string[] @@ -13,6 +14,7 @@ export interface GitHubPushPayload { before?: string commits?: GitHubCommit[] head_commit?: GitHubCommit & { id?: string } + repository?: { full_name?: string } } /** Constant-time string comparison. */ @@ -79,19 +81,15 @@ export async function resolveContentSha( if (cached) return cached } - const token = githubToken() - let commits: Array<{ sha: string }> + // Shared with the build-time seed, which walks the built commit instead of a branch — see + // `fetchLastContentCommit()`. One query, so the two cannot drift apart. + let sha: string | undefined try { - commits = await $fetch>(`https://api.github.com/repos/${githubRepo()}/commits`, { - headers: { - Accept: 'application/vnd.github+json', - ...(token ? { Authorization: `Bearer ${token}` } : {}), - }, - query: { - sha: branch, - path: normalizeContentDir(contentDir), - per_page: 1, - }, + sha = await fetchLastContentCommit({ + repo: githubRepo(), + path: contentDir, + ref: branch, + token: githubToken(), }) } catch (error: unknown) { // Only a definitive 404 is cacheable; a 5xx, rate-limit 403 or network blip stays retryable. @@ -104,7 +102,6 @@ export async function resolveContentSha( throw error } - const sha = commits[0]?.sha if (!sha) { if (opts.cacheMisses) await refStorage.setItem(key, UNRESOLVED) throw createError({ statusCode: 404, statusMessage: `Content not found at ref: ${branch}` }) diff --git a/server/utils/local.ts b/server/utils/local.ts index 8d7ae1b..5e92c9b 100644 --- a/server/utils/local.ts +++ b/server/utils/local.ts @@ -4,7 +4,7 @@ import type { Source } from 'comark-content' import type { PageCommit } from './github' const exec = promisify(execFile) -/** Root of the git repository holding the content (resolved at build time by modules/config.ts). */ +/** Root of the git repository holding the content (resolved at build time by modules/config/). */ function repoRoot(): string { return useRuntimeConfig().docs.repoRoot } diff --git a/server/utils/paths.ts b/server/utils/paths.ts index ae8f710..4988fa2 100644 --- a/server/utils/paths.ts +++ b/server/utils/paths.ts @@ -1,55 +1,4 @@ -/** Repo-relative content prefix (e.g. `docs/content/`), derived at build time by modules/config.ts. */ +/** Repo-relative content prefix (e.g. `docs/content/`), derived at build time by modules/config/. */ export function contentPrefix(): string { return `${useRuntimeConfig().docs.contentDir.replace(/\/$/, '')}/` } - -/** Whether a GitHub repo path is a content markdown file. */ -export function isContentMd(path: string): boolean { - return path.startsWith(contentPrefix()) && path.toLowerCase().endsWith('.md') -} - -/** Whether a GitHub repo path is a navigation config file (`.navigation.yml` / `.json`). */ -export function isNavConfig(path: string): boolean { - return path.startsWith(contentPrefix()) && /\.navigation\.(?:ya?ml|json)$/i.test(path) -} - -/** - * Parse a content repo path into route segments (`1.getting-started/2.intro.md` → - * `['getting-started', 'intro']`). `isIndex` covers both `index.md` and `index/index.md`. - */ -export function slugFromPath(path: string): { isIndex: boolean; segments: string[] } | null { - const prefix = contentPrefix() - if (!path.startsWith(prefix) || !path.toLowerCase().endsWith('.md')) return null - - const relative = path.slice(prefix.length, -3) - const segments = relative.split('/').map((s) => s.replace(/^\d+\./, '')) - const last = segments[segments.length - 1] - const isIndex = last === 'index' - if (isIndex) segments.pop() - return { isIndex, segments } -} - -/** Frontend page route (e.g. `1.getting-started/2.intro.md` → `/getting-started/intro`, root → `/`). */ -export function pageUrlForPath(path: string): string | null { - const result = slugFromPath(path) - if (!result) return null - const { isIndex, segments } = result - if (isIndex && segments.length === 0) return '/' - return `/${segments.join('/')}` -} - -/** Raw markdown route — the only per-file route that stays cached, as `/api/pages` is served live. */ -export function rawUrlForPath(path: string): string | null { - const result = slugFromPath(path) - if (!result) return null - - const { isIndex, segments } = result - if (isIndex && segments.length === 0) return '/raw/index.md' - return `/raw/${segments.join('/')}.md` -} - -/** Nuxt payload route for a frontend page route */ -export function payloadUrlForRoute(route: string, buildId?: string): string { - const path = `${route === '/' ? '' : route}/_payload.json` - return buildId ? `${path}?${buildId}` : path -} diff --git a/server/utils/timing.ts b/server/utils/timing.ts new file mode 100644 index 0000000..17a4cce --- /dev/null +++ b/server/utils/timing.ts @@ -0,0 +1,29 @@ +/** Named phase timings for one revalidate webhook run. */ +export interface Timings { + /** Time a sync or async `fn` under `label`; records its duration and returns its result. */ + time(label: string, fn: () => T | Promise): Promise + /** `label=123ms label2=45ms`, in recorded order — for one log line. */ + format(): string + /** ms since this recorder was created — spans the sync response and the background `waitUntil` phase. */ + since(): number +} + +export function createTimings(): Timings { + const start = performance.now() + const entries: { label: string; ms: number }[] = [] + + async function time(label: string, fn: () => T | Promise): Promise { + const phaseStart = performance.now() + try { + return await fn() + } finally { + entries.push({ label, ms: Math.round(performance.now() - phaseStart) }) + } + } + + function format(): string { + return entries.map(({ label, ms }) => `${label}=${ms}ms`).join(' ') + } + + return { time, format, since: () => Math.round(performance.now() - start) } +} diff --git a/server/utils/webhook.ts b/server/utils/webhook.ts new file mode 100644 index 0000000..dd8c7b0 --- /dev/null +++ b/server/utils/webhook.ts @@ -0,0 +1,117 @@ +import { DEFAULT_CONTENT_NAME, type ContentListFile } from 'comark-content' +import type { GitHubCommit } from './github' +import { hashManifestItem } from './json' + +/** How a push changed the content source, already filtered to `contentDir`. */ +export interface ContentChanges { + /** Manifest keys (`default/`) of files added or modified. */ + upserted: string[] + /** Manifest keys of files removed — only the previous manifest can resolve their paths. */ + removed: string[] + /** A `.navigation.*` file changed, so the tree changed regardless of which pages did. */ + navTouched: boolean +} + +/** Files the content source can actually serve — matches the parsers installed in `content.ts`. */ +const CONTENT_EXTENSIONS = ['.md', '.yml', '.yaml', '.json'] + +/** The content instance's name (see `createSourceContent()` in `content.ts`) — unnamed, so `default`. */ +const SOURCE_NAME = DEFAULT_CONTENT_NAME + +/** + * A push's changed content files, named by their manifest key (`default/`) — the + * reverse of `meta.key`, so a diff against `manifest.items` doesn't need to re-derive file → URL + * mappings that comark already owns. + */ +export function changesForPush(contentDir: string, commits: GitHubCommit[]): ContentChanges { + const upserted = new Set() + const removed = new Set() + let navTouched = false + + const consider = (file: string, into: Set) => { + const key = manifestKeyFor(file, contentDir) + if (!key) return + + if (isNavConfigFile(file)) navTouched = true + else into.add(key) + } + + for (const commit of commits) { + for (const file of commit.added ?? []) consider(file, upserted) + for (const file of commit.modified ?? []) consider(file, upserted) + for (const file of commit.removed ?? []) consider(file, removed) + } + + // A path removed and re-added in the same push is an upsert, not a removal. + for (const key of upserted) removed.delete(key) + + return { upserted: [...upserted], removed: [...removed], navTouched } +} + +/** Repo-relative path → its key in the manifest, or `null` when it can't be a content file. */ +function manifestKeyFor(file: string, contentDir: string): string | null { + const dir = contentDir.replace(/^\/+|\/+$/g, '') + const prefix = dir ? `${dir}/` : '' + + if (prefix && !file.startsWith(prefix)) return null + if (!CONTENT_EXTENSIONS.some((ext) => file.toLowerCase().endsWith(ext))) return null + + return `${SOURCE_NAME}/${file.slice(prefix.length)}` +} + +/** Directory configuration (`.navigation.yml`), which contributes to the tree rather than a page. */ +function isNavConfigFile(file: string): boolean { + return /\.navigation\.(?:ya?ml|json)$/i.test(file) +} + +/** + * The payload URL a client-side navigation fetches for `path` + */ +export function payloadUrlForPage(path: string, buildId?: string): string { + const base = path === '/' ? '/_payload.json' : `${path.replace(/\/$/, '')}/_payload.json` + return buildId ? `${base}?_b=${buildId}` : base +} + +/** `default/` (a manifest key) → page path, the reverse of what the path-keyed manifest gives. */ +export function indexByFileKey(items: Record): Map { + const index = new Map() + for (const item of Object.values(items)) index.set(item.meta.key, item.path) + return index +} + +/** + * Which pages a push changed, and whether the tree itself moved. + */ +export function diffContent( + changes: ContentChanges, + before: Record, + after: Record +): { pagePaths: string[]; navChanged: boolean } { + const pagePaths = new Set() + + const afterByKey = indexByFileKey(after) + const beforeByKey = indexByFileKey(before) + + for (const key of changes.upserted) { + const path = afterByKey.get(key) + if (path) pagePaths.add(path) + } + for (const key of changes.removed) { + const path = beforeByKey.get(key) + if (path) pagePaths.add(path) + } + + const beforeKeys = Object.keys(before) + const afterKeys = Object.keys(after) + const navChanged = + beforeKeys.length !== afterKeys.length || + afterKeys.some((key) => !before[key]) || + // Listing fields (title, description, icon, `navigation`…) are what the tree renders from. + afterKeys.some((key) => before[key] && !sameListing(before[key]!, after[key]!)) + + return { pagePaths: [...pagePaths], navChanged } +} + +function sameListing(a: ContentListFile, b: ContentListFile): boolean { + return a.path === b.path && hashManifestItem(a) === hashManifestItem(b) +} diff --git a/test/content-contract.test.ts b/test/content-contract.test.ts index 20de79d..291a78e 100644 --- a/test/content-contract.test.ts +++ b/test/content-contract.test.ts @@ -5,12 +5,18 @@ * which is how comark-content#77's `metaOnly` -> `partial` rename silently degraded * the webhook's body warm-up. This exercises the surface the layer depends on. */ +import { mkdtemp, readFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' import { fileURLToPath } from 'node:url' import { describe, expect, it } from 'vitest' -import { comarkContent, defineContentPlugin } from 'comark-content' +import { comarkContent, DEFAULT_CONTENT_NAME, readArtifact, writeSnapshots } from 'comark-content' import fsSource from 'comark-content/sources/fs' import githubSource from 'comark-content/sources/github' -import { createContentClient, defineContentClientPlugin } from 'comark-content/client' +import snapshot, { withSnapshot } from 'comark-content/sources/snapshot' +import { createContentClient } from 'comark-content/client' +import sqliteWasm from 'comark-content/database/sqlite-wasm' +import sqliteFullTextSearch from 'comark-content/plugins/sqlite-full-text-search' import memoryDriver from 'unstorage/drivers/memory' const fixture = fileURLToPath(new URL('./fixtures/content-contract', import.meta.url)) @@ -36,7 +42,16 @@ function createFixtureContent() { describe('comark-content contract', () => { it('exposes every entrypoint the layer imports', () => { - for (const entry of [comarkContent, defineContentPlugin, fsSource, githubSource, createContentClient, defineContentClientPlugin]) { + for (const entry of [ + comarkContent, + readArtifact, + fsSource, + githubSource, + createContentClient, + // Browser-only at runtime, but the subpaths resolve under node — enough to catch a rename. + sqliteWasm, + sqliteFullTextSearch, + ]) { expect(typeof entry).toBe('function') } }) @@ -71,24 +86,100 @@ describe('comark-content contract', () => { expect(cached!.nodes.length).toBeGreaterThan(0) }) - it('dispatches plugin serve handlers through content.handler', async () => { - // Mirrors the `search-sections` plugin in server/utils/content.ts. - const ping = defineContentPlugin(() => ({ - name: 'ping', - setup(ctx) { - ctx.addServeHandler('ping', async () => Response.json({ ok: true })) - }, - })) - - const content = comarkContent({ - source: fsSource(fixture), - cache: { driver: memoryDriver() }, - plugins: [ping()], + it('serves the manifest and snapshot artifacts through content.handler', async () => { + const content = createFixtureContent() + await content.init(full) + + // The exact paths the search worker fetches and `modules/config/` declares ISR rules for. + for (const path of ['manifest.json', `snapshot/${DEFAULT_CONTENT_NAME}.json`]) { + const response = await content.handler(new Request(`http://localhost/api/content/${path}`)) + expect(response.status, path).toBe(200) + const artifact = await response.json() + expect(Object.keys(artifact), path).toContain('checksum') + expect(Object.keys(await readArtifact(artifact)).length, path).toBeGreaterThan(0) + } + }) + + it('hydrates a sourceless instance from those artifacts', async () => { + const server = createFixtureContent() + await server.init(full) + + const fetchArtifact = async (path: string) => + await (await server.handler(new Request(`http://localhost/api/content/${path}`))).json() + + // The search feature is this round-trip, so a break here is a silently empty search index. + // `snapshot()`'s first argument is the full-body tier; the second (optional) manifest tier + // lets a bare `init()` skip downloading bodies until a document is actually requested. + const client = comarkContent({ + source: snapshot( + () => fetchArtifact(`snapshot/${DEFAULT_CONTENT_NAME}.json`), + () => fetchArtifact('manifest.json') + ), }) + await client.init() - const response = await content.handler(new Request('http://localhost/api/content/ping')) + expect(Object.keys((await client.manifest()).items)).toEqual(['/']) - expect(response.status).toBe(200) - expect(await response.json()).toEqual({ ok: true }) + // Bodies have to arrive parsed: the client has no source to read a document from. + const doc = await client.get('/') + expect(doc?.data?.title).toBe('Contract fixture') + expect(doc?.nodes?.length).toBeGreaterThan(0) + }) + + describe('build-time seed', () => { + /** + * `modules/snapshot/` writes the seed with `writeSnapshots()`, and + * `server/utils/content.ts` reads it back through a Nitro server asset. Two things here are + * layout, not behaviour, and both are silent when wrong: the per-instance subdirectory, and + * the fact that a server asset hands back JSON *text*. + */ + async function writeSeed() { + const dir = await mkdtemp(join(tmpdir(), 'comark-seed-')) + await writeSnapshots(createFixtureContent(), { dir }) + // One directory per instance, named after it — ours is unnamed, so `default`. + const read = (file: string) => readFile(join(dir, DEFAULT_CONTENT_NAME, file), 'utf8') + return { snapshot: () => read('snapshot.json'), manifest: () => read('manifest.json') } + } + + it('hydrates a withSnapshot instance from the seed without reading the source', async () => { + const seed = await writeSeed() + + // A source that throws on any read: hydrating from the seed must not touch it. This is the + // cold start being bought — in production the reads it stands in for are GitHub API calls. + const unreachable = { + ...fsSource(fixture), + keys: () => { + throw new Error('the origin was walked') + }, + } + + const content = comarkContent({ + source: withSnapshot(unreachable, seed.snapshot, seed.manifest), + cache: { driver: memoryDriver() }, + }) + await content.init(full) + + expect(Object.keys((await content.manifest()).items)).toEqual(['/']) + const doc = await content.get('/') + expect(doc?.data?.title).toBe('Contract fixture') + expect(doc?.nodes?.length).toBeGreaterThan(0) + }) + + it('falls back to the source when no seed is stored', async () => { + // What every ref other than the build commit gets: loaders return `null`, so the origin is + // the only provider. A seed that cannot prove it belongs to this ref must never be used. + const content = comarkContent({ + source: withSnapshot( + fsSource(fixture), + () => null, + () => null + ), + cache: { driver: memoryDriver() }, + }) + await content.init(full) + + expect(Object.keys((await content.manifest()).items)).toEqual(['/']) + expect((await content.get('/'))?.data?.title).toBe('Contract fixture') + }) }) }) diff --git a/test/geist-theme.test.ts b/test/geist.test.ts similarity index 99% rename from test/geist-theme.test.ts rename to test/geist.test.ts index a24a6e6..bcb48ab 100644 --- a/test/geist-theme.test.ts +++ b/test/geist.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest' import { parseMarkdown } from 'comark' import rangi from 'comark/plugins/rangi' -import { geistDark, geistLight, geistTheme } from '../utils/geist-theme' +import { geistDark, geistLight, geistTheme } from '../utils/geist' describe('Geist syntax theme', () => { it('uses the live Geist light syntax roles', () => { diff --git a/test/git.test.ts b/test/git.test.ts index 54f9356..c73ac78 100644 --- a/test/git.test.ts +++ b/test/git.test.ts @@ -1,6 +1,9 @@ +import { execFileSync } from 'node:child_process' +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import { parseGitRemote } from '../utils/git' -import { inferSiteURL } from '../utils/meta' +import { getLastCommit, getTreeSha, hasParent, parseGitRemote } from '../utils/git' describe('parseGitRemote', () => { it('parses SSH remotes', () => { @@ -31,53 +34,76 @@ describe('parseGitRemote', () => { }) }) -describe('inferSiteURL', () => { - const keys = [ - 'NUXT_PUBLIC_SITE_URL', - 'NUXT_SITE_URL', - 'VERCEL_PROJECT_PRODUCTION_URL', - 'VERCEL_BRANCH_URL', - 'VERCEL_URL', - 'URL', - 'CI_PAGES_URL', - 'CF_PAGES_URL', - ] - let saved: Record - - // `Reflect.deleteProperty` rather than `delete process.env[key]`: same effect, - // without tripping `no-dynamic-delete`. - const unset = (key: string) => Reflect.deleteProperty(process.env, key) - - beforeEach(() => { - saved = Object.fromEntries(keys.map((key) => [key, process.env[key]])) - for (const key of keys) unset(key) + +describe('commit and tree helpers', () => { + let repo: string + + const run = (...args: string[]) => execFileSync('git', args, { cwd: repo, stdio: 'ignore' }) + const write = async (file: string, body: string) => { + await mkdir(dirname(join(repo, file)), { recursive: true }) + await writeFile(join(repo, file), body, 'utf8') + } + + beforeEach(async () => { + repo = await mkdtemp(join(tmpdir(), 'comark-git-')) + run('init', '-q', '-b', 'main') + run('config', 'user.email', 'test@example.com') + run('config', 'user.name', 'Test') + + await write('content/index.md', '# one\n') + run('add', '-A') + run('commit', '-qm', 'add content') + + // A later commit that leaves `content/` untouched, so HEAD is not the last content commit. + await write('src/app.ts', 'export const a = 1\n') + run('add', '-A') + run('commit', '-qm', 'add code') }) - afterEach(() => { - for (const [key, value] of Object.entries(saved)) { - if (value === undefined) unset(key) - else process.env[key] = value - } + afterEach(async () => { + await rm(repo, { recursive: true, force: true }) }) - it('returns undefined when nothing is set', () => { - expect(inferSiteURL()).toBeUndefined() + it('finds the last commit touching a directory, not HEAD', () => { + const head = execFileSync('git', ['rev-parse', 'HEAD'], { cwd: repo, encoding: 'utf8' }).trim() + const last = getLastCommit(repo, 'content') + + expect(last).toMatch(/^[0-9a-f]{40}$/) + expect(last).not.toBe(head) }) - it('adds https to a bare Vercel host', () => { - process.env.VERCEL_URL = 'my-app-abc123.vercel.app' - expect(inferSiteURL()).toBe('https://my-app-abc123.vercel.app') + it('returns the same tree for a ref whose content matches HEAD', () => { + // The whole safety property of the build-time seed: the commit it is labelled with has to hold + // the content that was parsed. Here the code commit did not touch `content/`, so both agree. + const last = getLastCommit(repo, 'content')! + expect(getTreeSha(repo, last, 'content')).toBe(getTreeSha(repo, 'HEAD', 'content')) }) - it('prefers the explicit override over the platform value', () => { - process.env.VERCEL_URL = 'my-app-abc123.vercel.app' - process.env.NUXT_PUBLIC_SITE_URL = 'https://docs.example.com' - expect(inferSiteURL()).toBe('https://docs.example.com') + it('returns a different tree once the content changes', async () => { + const before = getTreeSha(repo, 'HEAD', 'content')! + + await write('content/index.md', '# two\n') + run('add', '-A') + run('commit', '-qm', 'edit content') + + expect(getTreeSha(repo, 'HEAD', 'content')).not.toBe(before) + // A stale label is what the seed must never be written under. + expect(getTreeSha(repo, 'HEAD~1', 'content')).toBe(before) }) - it('prefers the production URL over the per-branch one', () => { - process.env.VERCEL_BRANCH_URL = 'branch.vercel.app' - process.env.VERCEL_PROJECT_PRODUCTION_URL = 'docs.comark.dev' - expect(inferSiteURL()).toBe('https://docs.comark.dev') + it('returns undefined for a ref or path outside the checkout', () => { + expect(getTreeSha(repo, 'HEAD', 'nope')).toBeUndefined() + expect(getTreeSha(repo, 'a'.repeat(40), 'content')).toBeUndefined() + expect(getLastCommit(repo, 'nope')).toBeUndefined() + }) + + it('reports a missing parent at the root commit', () => { + const root = execFileSync('git', ['rev-list', '--max-parents=0', 'HEAD'], { + cwd: repo, + encoding: 'utf8', + }).trim() + + expect(hasParent(repo, 'HEAD')).toBe(true) + expect(hasParent(repo, root)).toBe(false) }) }) diff --git a/test/github.test.ts b/test/github.test.ts index b39103d..24c4904 100644 --- a/test/github.test.ts +++ b/test/github.test.ts @@ -1,6 +1,9 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { resolveContentSha } from '../server/utils/github' +/** One commits-query response. `resolveContentSha` reads only `sha`. */ +const commits = (sha: string) => new Response(JSON.stringify([{ sha }]), { status: 200 }) + afterEach(() => { vi.unstubAllEnvs() vi.unstubAllGlobals() @@ -8,21 +11,23 @@ afterEach(() => { describe('resolveContentSha', () => { it('resolves the latest commit touching the configured content directory', async () => { - const fetch = vi.fn().mockResolvedValue([{ sha: 'content-sha' }]) - vi.stubGlobal('$fetch', fetch) + const fetch = vi.fn().mockResolvedValue(commits('content-sha')) + vi.stubGlobal('fetch', fetch) await expect(resolveContentSha('feat/docs', '/docs/content/')).resolves.toBe('content-sha') - expect(fetch).toHaveBeenCalledWith( - 'https://api.github.com/repos/comarkdown/comark-docs/commits', - expect.objectContaining({ - query: { sha: 'feat/docs', path: 'docs/content', per_page: 1 }, - }) - ) + + const requested = new URL(String(fetch.mock.calls[0]![0])) + expect(requested.pathname).toBe('/repos/comarkdown/comark-docs/commits') + expect(Object.fromEntries(requested.searchParams)).toEqual({ + sha: 'feat/docs', + path: 'docs/content', + per_page: '1', + }) }) it('caches each branch and content directory independently', async () => { - const fetch = vi.fn().mockResolvedValueOnce([{ sha: 'docs-sha' }]).mockResolvedValueOnce([{ sha: 'api-sha' }]) - vi.stubGlobal('$fetch', fetch) + const fetch = vi.fn().mockResolvedValueOnce(commits('docs-sha')).mockResolvedValueOnce(commits('api-sha')) + vi.stubGlobal('fetch', fetch) await expect(resolveContentSha('test/cache-key', 'docs/content')).resolves.toBe('docs-sha') await expect(resolveContentSha('test/cache-key', 'docs/content')).resolves.toBe('docs-sha') @@ -31,8 +36,8 @@ describe('resolveContentSha', () => { }) it('can refresh a cached content revision for the push webhook', async () => { - const fetch = vi.fn().mockResolvedValueOnce([{ sha: 'before' }]).mockResolvedValueOnce([{ sha: 'after' }]) - vi.stubGlobal('$fetch', fetch) + const fetch = vi.fn().mockResolvedValueOnce(commits('before')).mockResolvedValueOnce(commits('after')) + vi.stubGlobal('fetch', fetch) await expect(resolveContentSha('test/refresh', 'docs/content')).resolves.toBe('before') await expect(resolveContentSha('test/refresh', 'docs/content')).resolves.toBe('before') diff --git a/test/paths.test.ts b/test/paths.test.ts index e8c96bb..36f36e9 100644 --- a/test/paths.test.ts +++ b/test/paths.test.ts @@ -1,14 +1,6 @@ import { afterEach, describe, expect, it } from 'vitest' import { resetRuntimeConfig, setRuntimeConfig } from './setup' -import { - contentPrefix, - isContentMd, - isNavConfig, - pageUrlForPath, - payloadUrlForRoute, - rawUrlForPath, - slugFromPath, -} from '../server/utils/paths' +import { contentPrefix } from '../server/utils/paths' afterEach(resetRuntimeConfig) @@ -19,79 +11,3 @@ describe('contentPrefix', () => { expect(contentPrefix()).toBe('docs/content/') }) }) - -describe('isContentMd', () => { - it('matches markdown under the content dir only', () => { - expect(isContentMd('content/index.md')).toBe(true) - expect(isContentMd('content/1.guide/2.intro.MD')).toBe(true) - expect(isContentMd('content/.navigation.yml')).toBe(false) - expect(isContentMd('README.md')).toBe(false) - expect(isContentMd('other/content/x.md')).toBe(false) - }) - - it('follows a nested content dir', () => { - setRuntimeConfig({ contentDir: 'docs/content' }) - expect(isContentMd('docs/content/x.md')).toBe(true) - expect(isContentMd('content/x.md')).toBe(false) - }) -}) - -describe('isNavConfig', () => { - it('matches the yml/yaml/json navigation files', () => { - expect(isNavConfig('content/1.guide/.navigation.yml')).toBe(true) - expect(isNavConfig('content/.navigation.yaml')).toBe(true) - expect(isNavConfig('content/.navigation.json')).toBe(true) - expect(isNavConfig('content/navigation.yml')).toBe(false) - expect(isNavConfig('content/x.md')).toBe(false) - }) -}) - -describe('slugFromPath', () => { - it('strips numeric ordering prefixes at every level', () => { - expect(slugFromPath('content/1.getting-started/2.intro.md')).toEqual({ - isIndex: false, - segments: ['getting-started', 'intro'], - }) - }) - - it('treats index files as their parent', () => { - expect(slugFromPath('content/index.md')).toEqual({ isIndex: true, segments: [] }) - expect(slugFromPath('content/1.guide/index.md')).toEqual({ isIndex: true, segments: ['guide'] }) - }) - - it('returns null for anything outside the content dir', () => { - expect(slugFromPath('README.md')).toBeNull() - expect(slugFromPath('content/.navigation.yml')).toBeNull() - }) -}) - -describe('pageUrlForPath', () => { - it('maps content files to page routes', () => { - expect(pageUrlForPath('content/index.md')).toBe('/') - expect(pageUrlForPath('content/1.guide/index.md')).toBe('/guide') - expect(pageUrlForPath('content/1.guide/2.intro.md')).toBe('/guide/intro') - expect(pageUrlForPath('content/x.yml')).toBeNull() - }) -}) - -describe('rawUrlForPath', () => { - it('maps content files to their raw markdown mirror', () => { - expect(rawUrlForPath('content/index.md')).toBe('/raw/index.md') - expect(rawUrlForPath('content/1.guide/2.intro.md')).toBe('/raw/guide/intro.md') - expect(rawUrlForPath('content/1.guide/index.md')).toBe('/raw/guide.md') - expect(rawUrlForPath('README.md')).toBeNull() - }) -}) - -describe('payloadUrlForRoute', () => { - it('builds the payload URL the browser actually requests', () => { - expect(payloadUrlForRoute('/')).toBe('/_payload.json') - expect(payloadUrlForRoute('/guide/intro')).toBe('/guide/intro/_payload.json') - }) - - it('appends the build id when there is one', () => { - // The webhook has to purge the exact keyed URL, not the bare path. - expect(payloadUrlForRoute('/guide', 'abc123')).toBe('/guide/_payload.json?abc123') - expect(payloadUrlForRoute('/', 'abc123')).toBe('/_payload.json?abc123') - }) -}) diff --git a/test/setup.ts b/test/setup.ts index ba5873c..d5a4cde 100644 --- a/test/setup.ts +++ b/test/setup.ts @@ -1,7 +1,7 @@ // Nitro auto-imports, provided by hand: modules under `server/` are written against Nitro's globals, so // importing one directly in a test leaves those names undefined. Declaring the few the tests touch here beats // pulling in the whole Nuxt/Nitro harness for a handful of pure functions. `useRuntimeConfig` returns the shape -// `modules/config.ts` seeds. +// `modules/config/` seeds. import memoryDriver from 'unstorage/drivers/memory' export interface TestRuntimeConfig { diff --git a/test/webhook.test.ts b/test/webhook.test.ts new file mode 100644 index 0000000..258fd39 --- /dev/null +++ b/test/webhook.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, it } from 'vitest' +import type { ContentListFile } from 'comark-content' +import type { GitHubCommit } from '../server/utils/github' +import { changesForPush, diffContent, indexByFileKey, payloadUrlForPage } from '../server/utils/webhook' +import { rawUrlForPage } from '../server/utils/markdown' + +const commit = (partial: GitHubCommit): GitHubCommit => partial + +describe('changesForPush', () => { + it('classifies added/modified/removed content files, keyed by their manifest key', () => { + const commits = [ + commit({ + added: ['content/1.guide/2.intro.md'], + modified: ['content/index.md'], + removed: ['content/old.md'], + }), + ] + expect(changesForPush('content', commits)).toEqual({ + upserted: ['default/1.guide/2.intro.md', 'default/index.md'], + removed: ['default/old.md'], + navTouched: false, + }) + }) + + it('ignores files outside the content dir', () => { + expect(changesForPush('content', [commit({ modified: ['README.md', 'other/content/x.md'] })])).toEqual({ + upserted: [], + removed: [], + navTouched: false, + }) + }) + + it('follows a nested content dir', () => { + expect(changesForPush('docs/content', [commit({ modified: ['docs/content/x.md'] })])).toEqual({ + upserted: ['default/x.md'], + removed: [], + navTouched: false, + }) + }) + + it('covers every parser extension, not just markdown', () => { + const commits = [commit({ added: ['content/data.yml', 'content/data.yaml', 'content/data.json'] })] + expect(changesForPush('content', commits).upserted).toEqual([ + 'default/data.yml', + 'default/data.yaml', + 'default/data.json', + ]) + }) + + it('flags a navigation config file instead of collecting it', () => { + const commits = [commit({ modified: ['content/1.guide/.navigation.yml'] })] + expect(changesForPush('content', commits)).toEqual({ upserted: [], removed: [], navTouched: true }) + }) + + it('treats a path removed and re-added in the same push as an upsert', () => { + const commits = [commit({ added: ['content/index.md'], removed: ['content/index.md'] })] + expect(changesForPush('content', commits)).toEqual({ + upserted: ['default/index.md'], + removed: [], + navTouched: false, + }) + }) +}) + +describe('payloadUrlForPage', () => { + it('matches the `_b` query param Nuxt requests (`nuxt/dist/app/composables/payload.js`)', () => { + expect(payloadUrlForPage('/')).toBe('/_payload.json') + expect(payloadUrlForPage('/guide/intro')).toBe('/guide/intro/_payload.json') + expect(payloadUrlForPage('/guide', 'abc123')).toBe('/guide/_payload.json?_b=abc123') + expect(payloadUrlForPage('/', 'abc123')).toBe('/_payload.json?_b=abc123') + }) +}) + +describe('rawUrlForPage', () => { + it('is the exact inverse of pagePathFromRawSlug', () => { + expect(rawUrlForPage('/')).toBe('/raw/index.md') + expect(rawUrlForPage('/guide/intro')).toBe('/raw/guide/intro.md') + }) +}) + +describe('indexByFileKey', () => { + it('maps a manifest key back to its page path', () => { + const items: Record = { + '/guide/intro': { path: '/guide/intro', data: {}, meta: { key: 'content/1.guide/2.intro.md' } } as never, + } + expect(indexByFileKey(items).get('content/1.guide/2.intro.md')).toBe('/guide/intro') + }) +}) + +describe('diffContent', () => { + const file = (path: string, key: string, data: Record = {}): ContentListFile => + ({ path, data, meta: { key } }) as never + + it('resolves upserted/removed manifest keys to page paths', () => { + const before = { '/old': file('/old', 'content/old.md') } + const after = { '/guide/intro': file('/guide/intro', 'content/1.guide/2.intro.md') } + const changes = { upserted: ['content/1.guide/2.intro.md'], removed: ['content/old.md'], navTouched: false } + expect(diffContent(changes, before, after).pagePaths.sort()).toEqual(['/guide/intro', '/old']) + }) + + it('flags navChanged when a page is added or removed', () => { + const before = { '/a': file('/a', 'content/a.md') } + const after = { '/a': file('/a', 'content/a.md'), '/b': file('/b', 'content/b.md') } + expect(diffContent({ upserted: [], removed: [], navTouched: false }, before, after).navChanged).toBe(true) + }) + + it('flags navChanged when listing data changes, even with the same page set', () => { + const before = { '/a': file('/a', 'content/a.md', { title: 'A' }) } + const after = { '/a': file('/a', 'content/a.md', { title: 'B' }) } + expect(diffContent({ upserted: [], removed: [], navTouched: false }, before, after).navChanged).toBe(true) + }) + + it('does not flag navChanged when nothing listing-relevant moved', () => { + const before = { '/a': file('/a', 'content/a.md', { title: 'A' }) } + const after = { '/a': file('/a', 'content/a.md', { title: 'A' }) } + expect(diffContent({ upserted: ['content/a.md'], removed: [], navTouched: false }, before, after).navChanged).toBe( + false + ) + }) +}) diff --git a/utils/content.ts b/utils/content.ts new file mode 100644 index 0000000..57dd34a --- /dev/null +++ b/utils/content.ts @@ -0,0 +1,60 @@ +import type { Tracer } from '@opentelemetry/api' +import { type ContentOptions, comarkContent } from 'comark-content' +import markdown from 'comark-content/plugins/markdown' +import yaml from 'comark-content/plugins/yaml' +import tracingOtel from 'comark-content/plugins/tracing/otel' +import rangi from 'comark/plugins/rangi' +import security from 'comark/plugins/security' +import emoji from 'comark/plugins/emoji' +import toc from 'comark/plugins/toc' +import mermaid from 'comark/plugins/mermaid' +import { geistTheme } from './geist.ts' +import { contentTracer } from '../server/utils/tracer.ts' + +/** Frontmatter kept in the manifest, so `list()` and `navigation()` render without reading bodies. */ +const LISTING_FIELDS = ['title', 'description', 'navigation', 'icon', 'layout'] + +// Bump CONTENT_PARSER_VERSION in `server/utils/cache.ts` when these plugins or their options change cached output. +const comarkPlugins = [ + mermaid({ theme: 'zinc-light', themeDark: 'zinc-dark' }), + rangi({ theme: geistTheme }), + toc({ depth: 3 }), + emoji(), + security({ + blockedTags: ['script', 'iframe', 'embed', 'form', 'base', 'meta', 'link', 'style'], + allowDataImages: false, + }), +] + +/** + * The parser, in one place for: + * - The build-time seed + * - The runtime instance + */ +function create(options: Pick, tracer?: Tracer) { + return comarkContent({ + source: options.source, + plugins: [ + markdown({ + comark: { plugins: comarkPlugins }, + listingFields: LISTING_FIELDS, + }), + yaml({ listingFields: LISTING_FIELDS }), + tracer && tracingOtel({ tracer }), + ], + cache: options.cache, + basePath: options.basePath, + }) +} + +/** An instance serving requests: traced, and cached per content SHA. */ +export function createRuntimeContentInstance(options: Pick) { + return create(options, contentTracer()) +} + +/** + * The throwaway instance the build-time seed is parsed with (`modules/snapshot/`). + */ +export function createBuildContentInstance(options: Pick) { + return create(options) +} diff --git a/utils/geist-theme.ts b/utils/geist.ts similarity index 100% rename from utils/geist-theme.ts rename to utils/geist.ts diff --git a/utils/git.ts b/utils/git.ts index ad1a7ba..e323fa6 100644 --- a/utils/git.ts +++ b/utils/git.ts @@ -1,4 +1,4 @@ -import { execSync } from 'node:child_process' +import { execFileSync } from 'node:child_process' export interface GitInfo { name: string @@ -6,11 +6,10 @@ export interface GitInfo { url: string } -function git(command: string, cwd: string): string | undefined { +/** Run git with an argv array — no shell, so paths with spaces need no quoting. */ +function git(args: string[], cwd: string): string | undefined { try { - return execSync(`git ${command}`, { cwd, stdio: ['ignore', 'pipe', 'ignore'] }) - .toString() - .trim() + return execFileSync('git', args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim() } catch { return undefined } @@ -27,19 +26,19 @@ export function getGitBranch(cwd: string): string { if (envName && envName !== 'HEAD') return envName - const branch = git('rev-parse --abbrev-ref HEAD', cwd) + const branch = git(['rev-parse', '--abbrev-ref', 'HEAD'], cwd) return branch && branch !== 'HEAD' ? branch : 'main' } /** Absolute path of the git repository root containing `cwd`, if any. */ export function getGitRoot(cwd: string): string | undefined { - return git('rev-parse --show-toplevel', cwd) + return git(['rev-parse', '--show-toplevel'], cwd) } /** * Owner/name/url from a git remote URL, in both forms `git remote get-url` emits (`git@host:owner/name.git`, * `https://host/owner/name(.git)`). Split out from `getLocalGitInfo` so the regex is testable without a - * checkout — every inferred default in `modules/config.ts` (site name, edit links, webhook repo) flows from it. + * checkout — every inferred default in `modules/config/` (site name, edit links, webhook repo) flows from it. */ export function parseGitRemote(remote: string): GitInfo | undefined { const match = remote.trim().match(/^(?:git@|https?:\/\/)([^/:]+)[/:]([^/]+)\/(.+?)(?:\.git)?$/) @@ -51,7 +50,7 @@ export function parseGitRemote(remote: string): GitInfo | undefined { /** Owner/name/url parsed from the `origin` remote of the local checkout. */ export function getLocalGitInfo(cwd: string): GitInfo | undefined { - const remote = git('remote get-url origin', cwd) + const remote = git(['remote', 'get-url', 'origin'], cwd) return remote ? parseGitRemote(remote) : undefined } @@ -74,3 +73,32 @@ export function getGitEnv(): GitInfo | undefined { return { name, owner, url: `https://${provider || 'github'}.com/${owner}/${name}` } } + +/** + * The last commit touching `dir`, or `undefined`. + * + * Unverified on purpose. CI clones shallowly, and when the last commit touching `dir` predates the + * fetched window git answers with the shallow boundary commit rather than nothing — at depth 1, + * that is HEAD for every path. Confirm the answer with {@link getTreeSha} before trusting it to + * name a commit's content. + */ +export function getLastCommit(cwd: string, dir: string): string | undefined { + const sha = git(['log', '-1', '--format=%H', '--', dir], cwd) + return sha && /^[0-9a-f]{40}$/.test(sha) ? sha : undefined +} + +/** Tree object id of `:`, or `undefined` when the ref or the path is not in this checkout. */ +export function getTreeSha(cwd: string, ref: string, dir: string): string | undefined { + return git(['rev-parse', `${ref}:${dir}`], cwd) +} + +/** Whether `ref` has a parent in this checkout. `false` at a shallow-clone boundary. */ +export function hasParent(cwd: string, ref: string): boolean { + return Boolean(git(['rev-parse', '--verify', `${ref}^`], cwd)) +} + +/** The commit checked out here, falling back to the CI-provided one. */ +export function headCommit(cwd: string): string | undefined { + const sha = git(['rev-parse', 'HEAD'], cwd) || process.env.VERCEL_GIT_COMMIT_SHA + return sha && /^[0-9a-f]{40}$/.test(sha) ? sha : undefined +} diff --git a/utils/github.ts b/utils/github.ts new file mode 100644 index 0000000..24cfec0 --- /dev/null +++ b/utils/github.ts @@ -0,0 +1,36 @@ +export interface LastContentCommitOptions { + /** `owner/name` of the content repository. */ + repo: string + /** Content directory; leading and trailing slashes are trimmed. */ + path: string + /** Branch or commit to walk history from. */ + ref: string + token?: string +} + +/** + * The last commit reachable from `ref` that touched `path`. + */ +export async function fetchLastContentCommit(opts: LastContentCommitOptions): Promise { + const query = new URLSearchParams({ + sha: opts.ref, + path: opts.path.replace(/^\/+|\/+$/g, ''), + per_page: '1', + }) + + const response = await fetch(`https://api.github.com/repos/${opts.repo}/commits?${query}`, { + headers: { + Accept: 'application/vnd.github+json', + ...(opts.token ? { Authorization: `Bearer ${opts.token}` } : {}), + }, + }) + + if (!response.ok) { + throw Object.assign(new Error(`GitHub commits query failed with ${response.status}`), { + statusCode: response.status, + }) + } + + const commits = (await response.json()) as Array<{ sha?: string }> + return commits[0]?.sha +} diff --git a/utils/meta.ts b/utils/meta.ts deleted file mode 100644 index 505f57f..0000000 --- a/utils/meta.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { readFile } from 'node:fs/promises' -import { resolve } from 'pathe' -import { withHttps } from 'ufo' - -/** Infer the public site URL from the deployment platform env. */ -export function inferSiteURL(): string | undefined { - // https://github.com/unjs/std-env/issues/59 - const url = - process.env.NUXT_PUBLIC_SITE_URL || - process.env.NUXT_SITE_URL || - process.env.VERCEL_PROJECT_PRODUCTION_URL || - process.env.VERCEL_BRANCH_URL || - process.env.VERCEL_URL || - process.env.URL || // Netlify - process.env.CI_PAGES_URL || // GitLab Pages - process.env.CF_PAGES_URL // Cloudflare Pages - - return url ? withHttps(url) : undefined -} - -export async function getPackageJsonMetadata(dir: string): Promise<{ name?: string; description?: string }> { - try { - const parsed = JSON.parse(await readFile(resolve(dir, 'package.json'), 'utf-8')) - return { name: parsed.name, description: parsed.description } - } catch { - return {} - } -} diff --git a/utils/first-leaf.ts b/utils/navigation.ts similarity index 100% rename from utils/first-leaf.ts rename to utils/navigation.ts