diff --git a/.changeset/experimental-incremental.md b/.changeset/experimental-incremental.md new file mode 100644 index 000000000..9309da3de --- /dev/null +++ b/.changeset/experimental-incremental.md @@ -0,0 +1,7 @@ +--- +'@doc-kit/cli': minor +'@doc-kit/core': minor +'@doc-kit/generator-react': minor +--- + +Add `doc-kit generate --experimental-incremental` diff --git a/docs/cli.md b/docs/cli.md index 499765e4e..2cd4baef2 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -40,6 +40,7 @@ Runs the generators and writes their output. Requires a `target` and an links). - `-p, --threads ` {number} Worker threads to use (minimum 1). - `--chunk-size ` {number} Items per worker thread (minimum 1). +- `--experimental-incremental` {boolean} Rebuild against the existing output. **Experimental.** ## `doc-kit preview` diff --git a/docs/configuration.md b/docs/configuration.md index a0d750dd9..36dcdb95f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -137,6 +137,9 @@ Top-level, alongside `target` and `global`: CPU count. Each worker's heap is limited to 512MB; to raise the limit, run Node.js with `--max-old-space-size` (directly or through `NODE_OPTIONS`). - `chunkSize` {number} Items processed per worker thread. **Default:** `10`. +- `experimentalIncremental` {boolean} Build against the existing output + instead of from scratch. **Default:** `false`. **Experimental:** the option + and the manifest format are subject to change. ## Generator options @@ -262,17 +265,18 @@ precedence): CLI options map to configuration properties: -| CLI Option | Config Property | Example | -| ---------------------- | ------------------ | ------------------------- | -| `--input ` | `global.input` | `--input src/` | -| `--output ` | `global.output` | `--output dist/` | -| `--ignore ` | `global.ignore[]` | `--ignore test/` | -| `--minify` | `global.minify` | `--minify` | -| `--git-ref ` | `global.ref` | `--git-ref v20.0.0` | -| `--version ` | `global.version` | `--version 20.0.0` | -| `--changelog ` | `global.changelog` | `--changelog https://...` | -| `--index ` | `global.index` | `--index file://...` | -| `--type-map ` | `metadata.typeMap` | `--type-map file://...` | -| `--target ` | `target` | `--target json` | -| `--threads ` | `threads` | `--threads 4` | -| `--chunk-size ` | `chunkSize` | `--chunk-size 10` | +| CLI Option | Config Property | Example | +| ---------------------------- | ------------------------- | ---------------------------- | +| `--input ` | `global.input` | `--input src/` | +| `--output ` | `global.output` | `--output dist/` | +| `--ignore ` | `global.ignore[]` | `--ignore test/` | +| `--minify` | `global.minify` | `--minify` | +| `--git-ref ` | `global.ref` | `--git-ref v20.0.0` | +| `--version ` | `global.version` | `--version 20.0.0` | +| `--changelog ` | `global.changelog` | `--changelog https://...` | +| `--index ` | `global.index` | `--index file://...` | +| `--type-map ` | `metadata.typeMap` | `--type-map file://...` | +| `--target ` | `target` | `--target json` | +| `--threads ` | `threads` | `--threads 4` | +| `--chunk-size ` | `chunkSize` | `--chunk-size 10` | +| `--experimental-incremental` | `experimentalIncremental` | `--experimental-incremental` | diff --git a/docs/creating-generators.md b/docs/creating-generators.md index c2912d335..fbfe5016d 100644 --- a/docs/creating-generators.md +++ b/docs/creating-generators.md @@ -149,6 +149,13 @@ function transformToMyFormat(entries, version) { } ``` +`generate` is called with the input, a worker for +[parallel processing](#parallel-processing-with-workers), and the generator +context (`GeneratorContext`): what the orchestrator provides besides the +input. Its one property so far is `incremental`, the generator's section of +the manifest of an [incremental build](#incremental-builds); when the build +is not incremental, its methods are all no-ops. + ### Step 5: Make the Generator Loadable Generators are loaded dynamically by import specifier. Anything that resolves @@ -567,6 +574,57 @@ export async function generate(input, worker) { } ``` +### Incremental builds + +An incremental build makes again only the items whose output the previous +build did not make from the same key, and keeps the manifest of what every +item produced at `/.doc-kit/manifest.json`. The `incremental` +property of the [generator context](#step-4-implement-the-generator-logic) is +the generator's section of it: + +```mjs displayName="generate.mjs" +import { writeFile } from '@doc-kit/core/utils/file.mjs'; + +export async function generate(input, worker, { incremental }) { + // `stale` are the items to make again; the rest keep their files. `files` + // names what an item makes, be it one file or many. + const { stale } = incremental.partition(input, { + files: item => `${item.api}.myformat`, + }); + + for (const item of stale) { + await writeFile(join(config.output, `${item.api}.myformat`), render(item)); + } + + // Removes the previous build's files this one did not record + await incremental.prune(); +} +``` + +An item's key is its JSON, hashed with the generator's configuration, so a +configuration change makes everything again. The generator's metadata can +refine that under `incremental`: + +- `key(item)` is what an item's output is made from instead: a string, bytes, + or an array of them. +- `omit` lists the configuration keys that do not shape the output, such as + paths and functions. `input`, `output`, `ignore` and `pathsToCopy` are + always left out. +- `salt` lists further parts of every key, such as a format version. + +`partition(items, { key, files })` takes a key function in place of the +declared one, and `salt(compute)` adds parts computed at run time, such as +the hash of a built component library; `compute` only runs in an incremental +build, and before any key is derived. Files that belong to no item, such as +assets, or whose names are only known once made, are recorded by hand: +`record(item, ...files)`, with a label in place of the item for the former. + +Without `experimentalIncremental` (or an `output`) every item is stale and +nothing is recorded, so the generator needs no branches of its own. Whether +or not a generator uses its section, writing through core's `writeFile` (or +`writeJSON`) leaves a file alone during an incremental build when it already +holds the same bytes. + ### Copying Assets ```mjs displayName="generate.mjs" diff --git a/packages/cli/bin/commands/generate.mjs b/packages/cli/bin/commands/generate.mjs index 4122f6fb3..3e7237e8b 100644 --- a/packages/cli/bin/commands/generate.mjs +++ b/packages/cli/bin/commands/generate.mjs @@ -25,6 +25,7 @@ const { runGenerators } = createGenerator(); * @property {string} index * @property {boolean} minify * @property {string} typeMap + * @property {boolean} experimentalIncremental */ /** @@ -53,6 +54,10 @@ export const generateOptions = [ new Option('--index ', 'index.md URL or path'), new Option('--minify', 'Minify?'), new Option('--type-map ', 'Type map URL or path'), + new Option( + '--experimental-incremental', + 'Rebuild only what changed against the existing output (experimental)' + ), ]; const generate = new Command('generate').description('Generate API docs'); diff --git a/packages/core/package.json b/packages/core/package.json index bc3b6922e..af24b4e96 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -34,6 +34,7 @@ }, "imports": { "#generators/*": "./src/generators/*", + "#incremental/*": "./src/incremental/*", "#logger/*": "./src/logger/*", "#parsers/*": [ "./src/parsers/*", diff --git a/packages/core/src/__tests__/generators.test.mjs b/packages/core/src/__tests__/generators.test.mjs index d3be80da3..433c5a7c1 100644 --- a/packages/core/src/__tests__/generators.test.mjs +++ b/packages/core/src/__tests__/generators.test.mjs @@ -1,6 +1,13 @@ import assert from 'node:assert/strict'; +import { existsSync } from 'node:fs'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { describe, it, mock, beforeEach } from 'node:test'; +import { DISABLED_SECTION } from '../incremental/index.mjs'; +import { loadManifest, manifestPath } from '../incremental/manifest.mjs'; + // Tracks how many times each synthetic generator actually executed so we can // assert that a shared dependency runs exactly once even after its result is // evicted from the cache once its last consumer has read it. @@ -99,6 +106,25 @@ const syntheticGenerators = { return { all: input }; }, }, + // A generator recording an output through the incremental section it is + // handed, and one failing after it + 'gen-record': { + name: 'gen-record', + dependsOn: 'metadata', + generate: async (input, worker, { incremental }) => { + record('gen-record'); + incremental.record('recorded', 'recorded.txt'); + return { disabled: incremental === DISABLED_SECTION }; + }, + }, + 'gen-fail': { + name: 'gen-fail', + dependsOn: 'metadata', + generate: async () => { + record('gen-fail'); + throw new Error('boom'); + }, + }, }; mock.module('../generators/loader.mjs', { @@ -245,4 +271,65 @@ describe('createGenerator orchestration', () => { { name: 'gen-a', markdown, ran: ['ast', 'metadata'] }, ]); }); + + describe('incremental building', () => { + const createOutput = async context => { + const output = await mkdtemp(join(tmpdir(), 'doc-kit-orchestrator-')); + context.after(() => rm(output, { recursive: true, force: true })); + + return output; + }; + + it('hands each generator its section and saves the manifest once every target is done', async context => { + const output = await createOutput(context); + const { runGenerators } = createGenerator(); + + const results = await runGenerators({ + target: ['gen-record'], + threads: 1, + experimentalIncremental: true, + 'gen-record': { output }, + }); + + assert.deepStrictEqual(results, [{ disabled: false }]); + + const { manifest } = await loadManifest(output); + assert.deepStrictEqual( + Object.values(manifest.generators['gen-record'].outputs), + [['recorded.txt']] + ); + }); + + it('writes no manifest when a generator fails', async context => { + const output = await createOutput(context); + const { runGenerators } = createGenerator(); + + await assert.rejects( + runGenerators({ + target: ['gen-record', 'gen-fail'], + threads: 1, + experimentalIncremental: true, + 'gen-record': { output }, + 'gen-fail': { output }, + }), + /boom/ + ); + + assert.equal(existsSync(manifestPath(output)), false); + }); + + it('hands out disabled sections and writes nothing without the flag', async context => { + const output = await createOutput(context); + const { runGenerators } = createGenerator(); + + const results = await runGenerators({ + target: ['gen-record'], + threads: 1, + 'gen-record': { output }, + }); + + assert.deepStrictEqual(results, [{ disabled: true }]); + assert.equal(existsSync(manifestPath(output)), false); + }); + }); }); diff --git a/packages/core/src/generators.mjs b/packages/core/src/generators.mjs index b9959f4ae..6e6f21927 100644 --- a/packages/core/src/generators.mjs +++ b/packages/core/src/generators.mjs @@ -6,6 +6,7 @@ import { resolveGeneratorSpecifier, } from './generators/loader.mjs'; import { resolvePipeline } from './generators/pipeline.mjs'; +import { createIncrementalStore } from './incremental/index.mjs'; import logger from './logger/index.mjs'; import createWorkerPool from './threading/index.mjs'; import createParallelWorker from './threading/parallel.mjs'; @@ -27,6 +28,9 @@ const createGenerator = () => { /** @type {import('piscina').Piscina} */ let pool; + /** @type {import('./incremental/types').IncrementalStore} */ + let incremental; + /** * Gets the collected input from a dependency generator. * @@ -89,7 +93,12 @@ const createGenerator = () => { // Returned rather than awaited, so this function doesn't hold on to the // input while the generator runs, and the generator can let it go - return generate(dependencyInput, await worker); + return generate(dependencyInput, await worker, { + incremental: await incremental.section( + generator, + configuration[name] + ), + }); })().then(result => { // For streaming generators, "Completed" is logged when the cache // finishes collecting, not here when the generator returns @@ -136,6 +145,10 @@ const createGenerator = () => { // Create worker pool pool = createWorkerPool(threads); + incremental = createIncrementalStore({ + enabled: configuration.experimentalIncremental === true, + }); + // Schedule all generators for (const specifier of targets) { scheduleGenerator(specifier, generators, inputOf, configuration); @@ -147,6 +160,8 @@ const createGenerator = () => { targets.map(specifier => cache.consume(specifier)) ); + await incremental.save(); + await pool.destroy(); return results; diff --git a/packages/core/src/generators/json/__tests__/generate.test.mjs b/packages/core/src/generators/json/__tests__/generate.test.mjs index 935e747aa..cfa273e66 100644 --- a/packages/core/src/generators/json/__tests__/generate.test.mjs +++ b/packages/core/src/generators/json/__tests__/generate.test.mjs @@ -1,6 +1,8 @@ import assert from 'node:assert/strict'; -import { readFile } from 'node:fs/promises'; -import { basename } from 'node:path'; +import { existsSync } from 'node:fs'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { basename, join } from 'node:path'; import { describe, it } from 'node:test'; import { fileURLToPath } from 'node:url'; @@ -8,11 +10,15 @@ import Ajv from 'ajv'; import { loadGenerator } from '#generators/loader.mjs'; import { parseApiDoc } from '#generators/metadata/utils/parse.mjs'; +import { createIncrementalStore } from '#incremental/index.mjs'; +import { setConfig } from '#utils/configuration/index.mjs'; import { findFiles } from '#utils/glob.mjs'; import { loadMarkdownPlugins } from '#utils/markdown/plugins.mjs'; import { getProcessor } from '#utils/markdown/processor.mjs'; import { QUERIES } from '#utils/queries/index.mjs'; +import { generate, processChunk } from '../generate.mjs'; +import json from '../index.mjs'; import schema from '../schema.json' with { type: 'json' }; import { buildDocument } from '../utils/document.mjs'; @@ -41,21 +47,30 @@ const dependencies = { const validate = new Ajv({ allErrors: true, strict: true }).compile(schema); /** - * Runs a fixture through the metadata stage and builds its document. + * The metadata entries of a fixture, as the `metadata` generator yields them. */ -const buildFixture = async name => { +const parseFixture = async (name, transform = source => source) => { const content = await readFile(new URL(name, fixtures), 'utf-8'); - // The AST stage links the stability prefix; mirror that - const source = content.replace( - QUERIES.stabilityIndexPrefix, - match => `[${match}](documentation.html#stability-index)` + return parseApiDoc( + { + path: `/${basename(name, '.md')}`, + tree: getProcessor('ast').parse(transform(content)), + }, + typeMap ); +}; - const path = `/${basename(name, '.md')}`; - const entries = parseApiDoc( - { path, tree: getProcessor('ast').parse(source) }, - typeMap +/** + * Runs a fixture through the metadata stage and builds its document. + */ +const buildFixture = async name => { + // The AST stage links the stability prefix; mirror that + const entries = await parseFixture(name, source => + source.replace( + QUERIES.stabilityIndexPrefix, + match => `[${match}](documentation.html#stability-index)` + ) ); return buildDocument(entries[0], entries, dependencies); @@ -284,4 +299,62 @@ describe('json', () => { ); }); }); + + describe('incremental building', () => { + /** + * Runs an incremental build of the fixtures into the output, through a + * worker that builds on this thread and notes which modules it was given. + */ + const build = async (names, output) => { + const built = []; + + const worker = { + async *stream(items, dependencies) { + built.push(...items.map(({ head }) => head.api)); + + yield await processChunk(items, [...items.keys()], dependencies); + }, + }; + + const store = createIncrementalStore({ enabled: true }); + const incremental = await store.section(json, { output }); + const entries = ( + await Promise.all(names.map(name => parseFixture(name))) + ).flat(); + + const documents = []; + + for await (const chunk of generate(entries, worker, { incremental })) { + documents.push(...chunk); + } + + await store.save(); + + return { documents: documents.map(({ id }) => id).toSorted(), built }; + }; + + it('builds only the changed modules, yields every document, and removes those of dropped modules', async context => { + const output = await mkdtemp(join(tmpdir(), 'doc-kit-json-')); + context.after(() => rm(output, { recursive: true, force: true })); + + await setConfig({ target: ['json'], output }); + + const first = await build(['module.md', 'misc.md'], output); + + assert.deepEqual(first, { + documents: ['misc', 'module'], + built: ['module', 'misc'], + }); + + // Nothing changed: the documents are read back, not built + const second = await build(['module.md', 'misc.md'], output); + + assert.deepEqual(second, { documents: ['misc', 'module'], built: [] }); + + const third = await build(['module.md'], output); + + assert.deepEqual(third, { documents: ['module'], built: [] }); + assert.equal(existsSync(join(output, 'misc.json')), false); + }); + }); }); diff --git a/packages/core/src/generators/json/generate.mjs b/packages/core/src/generators/json/generate.mjs index 6ce545eb8..18defc80d 100644 --- a/packages/core/src/generators/json/generate.mjs +++ b/packages/core/src/generators/json/generate.mjs @@ -4,7 +4,7 @@ import { join } from 'node:path'; import getConfig from '#utils/configuration/index.mjs'; import { GITHUB_BLOB_URL, populate } from '#utils/configuration/templates.mjs'; -import { withExt, writeJSON } from '#utils/file.mjs'; +import { readJSON, withExt, writeJSON } from '#utils/file.mjs'; import { groupNodesByModule } from '#utils/generators.mjs'; import { buildDocument } from './utils/document.mjs'; @@ -28,7 +28,7 @@ export async function processChunk(slicedInput, itemIndices, dependencies) { * * @type {import('./types').Generator['generate']} */ -export async function* generate(input, worker) { +export async function* generate(input, worker, { incremental }) { const config = getConfig('json'); /** @type {import('./types').Dependencies} */ @@ -47,7 +47,18 @@ export async function* generate(input, worker) { entries: nodes, })); - for await (const chunk of worker.stream(modules, dependencies)) { + const { fresh, stale } = incremental.partition(modules, { + /** + * + */ + files: ({ head }) => withExt(head.path, 'json'), + }); + + for (const { head } of fresh) { + yield [await readJSON(join(config.output, withExt(head.path, 'json')))]; + } + + for await (const chunk of worker.stream(stale, dependencies)) { if (config.output) { await Promise.all( chunk.map(document => @@ -62,4 +73,6 @@ export async function* generate(input, worker) { yield chunk; } + + await incremental.prune(); } diff --git a/packages/core/src/generators/json/index.mjs b/packages/core/src/generators/json/index.mjs index 129869796..ab3a879d6 100644 --- a/packages/core/src/generators/json/index.mjs +++ b/packages/core/src/generators/json/index.mjs @@ -1,6 +1,6 @@ 'use strict'; -import { SCHEMA_URL } from './constants.mjs'; +import { SCHEMA_URL, SCHEMA_VERSION } from './constants.mjs'; import { generate, processChunk } from './generate.mjs'; /** @@ -22,6 +22,8 @@ export default { hasParallelProcessor: true, + incremental: { salt: [SCHEMA_VERSION] }, + markdown: { remarkPlugins: [ '#plugins/type-annotations/remark.mjs', diff --git a/packages/core/src/generators/types.d.ts b/packages/core/src/generators/types.d.ts index 8bee0399e..696837b3d 100644 --- a/packages/core/src/generators/types.d.ts +++ b/packages/core/src/generators/types.d.ts @@ -41,9 +41,15 @@ declare global { */ export type Generate = ( input: I, - worker: ParallelWorker + worker: ParallelWorker, + context: GeneratorContext ) => O extends AsyncGenerator ? O : Promise; + // What the orchestrator passes to `generate` as its third argument + export type GeneratorContext = { + incremental: import('../incremental/types').IncrementalSection; + }; + /** * Type for the optional processChunk function of a generator * @template I - Input type @@ -70,6 +76,11 @@ declare global { hasParallelProcessor?: boolean; + /** + * What an incremental build needs to know about this generator's outputs + */ + incremental?: import('../incremental/types').IncrementalOptions; + /** * The unified plugins this generator processes Markdown with, listed as * in the `markdown` option, with paths relative to its module. Each list diff --git a/packages/core/src/incremental/__tests__/hash.test.mjs b/packages/core/src/incremental/__tests__/hash.test.mjs new file mode 100644 index 000000000..8ae56dbfd --- /dev/null +++ b/packages/core/src/incremental/__tests__/hash.test.mjs @@ -0,0 +1,38 @@ +'use strict'; + +import assert from 'node:assert/strict'; +import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; + +import { hash, hashDirectory } from '../hash.mjs'; + +describe('hash', () => { + it('terminates each part, so splitting differently gives another hash', () => { + assert.notEqual(hash('ab', 'c'), hash('a', 'bc')); + assert.equal(hash('ab', 'c'), hash('ab', 'c')); + }); +}); + +describe('hashDirectory', () => { + it('depends on every path and byte, not on read order', async context => { + const directory = await mkdtemp(join(tmpdir(), 'doc-kit-hash-')); + context.after(() => rm(directory, { recursive: true, force: true })); + + await mkdir(join(directory, 'assets')); + await writeFile(join(directory, 'library.mjs'), 'export {}'); + await writeFile(join(directory, 'assets/chunk.mjs'), 'a'); + + const first = await hashDirectory(directory); + + assert.equal(await hashDirectory(directory), first); + + await writeFile(join(directory, 'assets/chunk.mjs'), 'b'); + assert.notEqual(await hashDirectory(directory), first); + + await writeFile(join(directory, 'assets/chunk.mjs'), 'a'); + await writeFile(join(directory, 'assets/other.mjs'), ''); + assert.notEqual(await hashDirectory(directory), first); + }); +}); diff --git a/packages/core/src/incremental/__tests__/index.test.mjs b/packages/core/src/incremental/__tests__/index.test.mjs new file mode 100644 index 000000000..95ad462be --- /dev/null +++ b/packages/core/src/incremental/__tests__/index.test.mjs @@ -0,0 +1,206 @@ +'use strict'; + +import assert from 'node:assert/strict'; +import { existsSync } from 'node:fs'; +import { mkdtemp, readdir, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; + +import { writeFile } from '#utils/file.mjs'; + +import { SCHEMA } from '../constants.mjs'; +import { createIncrementalStore, DISABLED_SECTION } from '../index.mjs'; +import { loadManifest, manifestPath, saveManifest } from '../manifest.mjs'; + +const createOutput = async context => { + const output = await mkdtemp(join(tmpdir(), 'doc-kit-incremental-')); + context.after(() => rm(output, { recursive: true, force: true })); + + return output; +}; + +const html = { name: 'html' }; + +/** + * A section of a fresh store over the output, and the store to save it with. + */ +const open = async (output, config = {}) => { + const store = createIncrementalStore({ enabled: true }); + + return { store, section: await store.section(html, { output, ...config }) }; +}; + +const files = item => `${item.id}.html`; + +/** + * Writes the items' files. + */ +const make = async (output, items) => { + for (const item of items) { + await writeFile(join(output, files(item)), item.id); + } +}; + +describe('createIncrementalStore', () => { + it('hands out the no-op section when disabled or without an output', async context => { + const output = await createOutput(context); + const disabled = createIncrementalStore({ enabled: false }); + + assert.equal(await disabled.section(html, { output }), DISABLED_SECTION); + assert.equal( + await createIncrementalStore({ enabled: true }).section(html, {}), + DISABLED_SECTION + ); + + await disabled.save(); + assert.equal(existsSync(manifestPath(output)), false); + }); + + it('partitions items by whether their key made files that are still there', async context => { + const output = await createOutput(context); + const a = { id: 'a' }; + const b = { id: 'b' }; + + const first = await open(output); + + assert.deepEqual(first.section.partition([a, b], { files }).stale, [a, b]); + await make(output, [a]); + await first.store.save(); + + const { outputs } = (await loadManifest(output)).manifest.generators.html; + + // `b`'s file was recorded but never written, so it is not fresh + assert.deepEqual(Object.values(outputs).toSorted(), [ + ['a.html'], + ['b.html'], + ]); + assert.match(Object.keys(outputs)[0], /^[0-9a-f]{32}$/); + + const second = await open(output); + + assert.deepEqual(second.section.partition([{ id: 'a' }, b]), { + fresh: [{ id: 'a' }], + stale: [b], + }); + + // The same item keyed differently, or with its file gone, is stale + const third = await open(output); + + assert.deepEqual( + third.section.partition([a], { key: () => 'other' }).stale, + [a] + ); + await rm(join(output, 'a.html')); + assert.deepEqual((await open(output)).section.partition([a]).stale, [a]); + }); + + it('salts the keys with the configuration minus the omitted keys, the declared parts and the computed ones', async context => { + const output = await createOutput(context); + const a = { id: 'a' }; + + const first = await open(output, { title: 'A', input: 'x' }); + await first.section.salt(() => ['run']); + first.section.partition([a], { files }); + await make(output, [a]); + await first.store.save(); + + const same = await open(output, { title: 'A', input: 'y' }); + await same.section.salt(() => ['run']); + assert.deepEqual(same.section.partition([a]).fresh, [a]); + + const otherConfig = await open(output, { title: 'B', input: 'x' }); + await otherConfig.section.salt(() => ['run']); + assert.deepEqual(otherConfig.section.partition([a]).stale, [a]); + + const otherRun = await open(output, { title: 'A', input: 'x' }); + await otherRun.section.salt(() => ['other']); + assert.deepEqual(otherRun.section.partition([a]).stale, [a]); + + await assert.rejects(otherRun.section.salt(), /salted after/); + await DISABLED_SECTION.salt(() => assert.fail('computed')); + }); + + it('prunes the files the previous build recorded and this one did not', async context => { + const output = await createOutput(context); + + for (const file of ['kept.html', 'stale.html', 'api/fs/old.html']) { + await writeFile(join(output, file), file); + } + + await writeFile(join(output, 'mine.txt'), 'not doc-kit’s'); + + await saveManifest(output, { + schema: SCHEMA, + generators: { + html: { + outputs: { + k: ['kept.html', 'stale.html'], + o: ['api/fs/old.html', 'missing.html'], + }, + }, + }, + }); + + const { store, section } = await open(output); + + section.record('label', 'kept.html'); + await section.prune(); + + assert.equal(existsSync(join(output, 'kept.html')), true); + assert.equal(existsSync(join(output, 'mine.txt')), true); + assert.equal(existsSync(join(output, 'stale.html')), false); + // The directory the removal emptied went too + assert.equal(existsSync(join(output, 'api')), false); + + await store.save(); + + assert.deepEqual( + Object.values( + (await loadManifest(output)).manifest.generators.html.outputs + ), + [['kept.html']] + ); + }); + + it('carries over the sections of generators not in the run, one manifest per output', async context => { + const output = await createOutput(context); + const other = await createOutput(context); + + await saveManifest(output, { + schema: SCHEMA, + generators: { + json: { outputs: { j: ['a.json'] } }, + html: { outputs: { h: ['a.html'] } }, + }, + }); + + const store = createIncrementalStore({ enabled: true }); + + (await store.section(html, { output })).record('x', 'b.html'); + (await store.section({ name: 'sitemap' }, { output })).record('x', 's.xml'); + (await store.section(html, { output: other })).record('x', 'c.html'); + + assert.equal( + await store.section(html, { output }), + await store.section(html, { output }) + ); + + await store.save(); + + const { generators } = (await loadManifest(output)).manifest; + + assert.deepEqual(generators.json, { outputs: { j: ['a.json'] } }); + assert.deepEqual(Object.values(generators.html.outputs), [['b.html']]); + assert.deepEqual(Object.values(generators.sitemap.outputs), [['s.xml']]); + assert.deepEqual( + Object.values( + (await loadManifest(other)).manifest.generators.html.outputs + ), + [['c.html']] + ); + assert.deepEqual(await readdir(join(output, '.doc-kit')), [ + 'manifest.json', + ]); + }); +}); diff --git a/packages/core/src/incremental/__tests__/manifest.test.mjs b/packages/core/src/incremental/__tests__/manifest.test.mjs new file mode 100644 index 000000000..a1af881af --- /dev/null +++ b/packages/core/src/incremental/__tests__/manifest.test.mjs @@ -0,0 +1,72 @@ +'use strict'; + +import assert from 'node:assert/strict'; +import { mkdtemp, readdir, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { describe, it } from 'node:test'; + +import { writeFile } from '#utils/file.mjs'; + +import { SCHEMA } from '../constants.mjs'; +import { loadManifest, manifestPath, saveManifest } from '../manifest.mjs'; + +const createOutput = async context => { + const output = await mkdtemp(join(tmpdir(), 'doc-kit-manifest-')); + context.after(() => rm(output, { recursive: true, force: true })); + + return output; +}; + +const writeManifest = (output, content) => + writeFile(manifestPath(output), content); + +describe('loadManifest', () => { + it('treats a missing manifest as empty, saying so', async context => { + const output = await createOutput(context); + const { manifest, reason } = await loadManifest(output); + + assert.deepEqual(manifest, { schema: SCHEMA, generators: {} }); + assert.match(reason, /no manifest/); + }); + + it('treats invalid JSON as empty', async context => { + const output = await createOutput(context); + await writeManifest(output, '{ not json'); + + const { manifest, reason } = await loadManifest(output); + + assert.deepEqual(manifest.generators, {}); + assert.match(reason, /not valid JSON/); + }); + + it('treats another schema as empty', async context => { + const output = await createOutput(context); + await writeManifest(output, JSON.stringify({ schema: 0, generators: {} })); + + const { manifest, reason } = await loadManifest(output); + + assert.deepEqual(manifest.generators, {}); + assert.match(reason, /schema 0/); + }); + + it('reads back what was saved, with no temporary file left', async context => { + const output = await createOutput(context); + const saved = { + schema: SCHEMA, + generators: { + html: { outputs: { k: ['a.html'] } }, + }, + }; + + await saveManifest(output, saved); + + const { manifest, reason } = await loadManifest(output); + + assert.equal(reason, undefined); + assert.deepEqual(manifest, saved); + assert.deepEqual(await readdir(dirname(manifestPath(output))), [ + 'manifest.json', + ]); + }); +}); diff --git a/packages/core/src/incremental/constants.mjs b/packages/core/src/incremental/constants.mjs new file mode 100644 index 000000000..048b43067 --- /dev/null +++ b/packages/core/src/incremental/constants.mjs @@ -0,0 +1,13 @@ +'use strict'; + +// Bump whenever the shape of what is stored changes +export const SCHEMA = 1; + +export const MANIFEST_DIRECTORY = '.doc-kit'; +export const MANIFEST_FILE = 'manifest.json'; + +// Hex characters of a sha256 digest kept: 128 bits +export const KEY_LENGTH = 32; + +// Global configuration keys holding locations, left out of every salt +export const SALT_CONFIG_OMIT = ['input', 'ignore', 'output', 'pathsToCopy']; diff --git a/packages/core/src/incremental/hash.mjs b/packages/core/src/incremental/hash.mjs new file mode 100644 index 000000000..1b5a62c01 --- /dev/null +++ b/packages/core/src/incremental/hash.mjs @@ -0,0 +1,42 @@ +'use strict'; + +import { createHash } from 'node:crypto'; +import { readFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import { findFiles } from '#utils/glob.mjs'; + +import { KEY_LENGTH } from './constants.mjs'; + +/** + * @param {...(string | Uint8Array)} parts + * @returns {string} The hex digest, {@link KEY_LENGTH} characters long + */ +export const hash = (...parts) => { + const hasher = createHash('sha256'); + + for (const part of parts) { + hasher.update(part); + hasher.update('\0'); + } + + return hasher.digest('hex').slice(0, KEY_LENGTH); +}; + +/** + * Hashes every file under a directory, by relative path and content + * + * @param {string} directory + * @returns {Promise} + */ +export const hashDirectory = async directory => { + const files = findFiles('**/*', { cwd: directory }).sort(); + + const parts = []; + + for (const file of files) { + parts.push(file, await readFile(join(directory, file))); + } + + return hash(...parts); +}; diff --git a/packages/core/src/incremental/index.mjs b/packages/core/src/incremental/index.mjs new file mode 100644 index 000000000..fa36a2d8f --- /dev/null +++ b/packages/core/src/incremental/index.mjs @@ -0,0 +1,308 @@ +'use strict'; + +import { existsSync } from 'node:fs'; +import { rm, rmdir } from 'node:fs/promises'; +import { join, posix, resolve } from 'node:path'; + +import logger from '#logger/index.mjs'; +import { enforceArray } from '#utils/array.mjs'; +import { omitKeys } from '#utils/misc.mjs'; + +import { SALT_CONFIG_OMIT, SCHEMA } from './constants.mjs'; +import { hash } from './hash.mjs'; +import { loadManifest, saveManifest } from './manifest.mjs'; + +export { hashDirectory } from './hash.mjs'; + +const incrementalLogger = logger.child('incremental'); + +/** + * Normalizes an output path + * + * @param {string} path + * @returns {string} + */ +const normalizePath = path => + posix.normalize(path.replaceAll('\\', '/')).replace(/^\/+/, ''); + +/** + * The section a generator gets when incremental building is off + * + * @type {import('./types').IncrementalSection} + */ +export const DISABLED_SECTION = Object.freeze({ + /** @returns {void} */ + record: () => {}, + /** @returns {Promise} */ + salt: async () => {}, + /** @returns {{ fresh: [], stale: Array }} */ + partition: items => ({ fresh: [], stale: items }), + /** @returns {Promise} */ + prune: async () => {}, +}); + +/** + * Creates one generator's view of a manifest + * + * @param {import('./types').IncrementalGenerator} generator + * @param {Record} config - The generator's configuration + * @param {string} output - The absolute output directory + * @param {import('./types').ManifestSection | undefined} previous + * @returns {import('./types').IncrementalSection & { toJSON(): import('./types').ManifestSection }} + */ +const createSection = (generator, config, output, previous) => { + const { name, incremental: options = {} } = generator; + const previousOutputs = previous?.outputs ?? {}; + + /** @type {Map>} Files by the key of the item they belong to */ + const outputs = new Map(); + + /** @type {WeakMap} The keys `partition` derived */ + const keys = new WeakMap(); + + let salt; + let upToDate = 0; + + /** + * @param {Array} [parts] - Computed at run time + * @returns {string} + */ + const computeSalt = (parts = []) => + (salt = hash( + JSON.stringify( + omitKeys(config, [...SALT_CONFIG_OMIT, ...(options.omit ?? [])]) + ), + ...(options.salt ?? []), + ...parts + )); + + /** + * @param {unknown} item + * @param {((item: unknown) => import('./types').KeyParts) | undefined} key + * @returns {string} + */ + const keyOf = (item, key = options.key) => { + const cacheable = typeof item === 'object' && item !== null; + + if (cacheable && keys.has(item)) { + return keys.get(item); + } + + salt ??= computeSalt(); + + const derived = hash( + salt, + ...enforceArray(key ? key(item) : JSON.stringify(item)) + ); + + if (cacheable) { + keys.set(item, derived); + } + + return derived; + }; + + /** + * @param {unknown} item - What the files were made from, or a label + * @param {...string} files - Relative to the output directory + */ + const record = (item, ...files) => { + const key = keyOf(item); + const recorded = outputs.get(key) ?? new Set(); + + for (const file of files) { + recorded.add(normalizePath(file)); + } + + outputs.set(key, recorded); + }; + + return { + record, + + /** + * Adds run-time parts to the salt; call before any key is derived + * + * @param {() => Array | Promise>} [compute] + */ + async salt(compute) { + if (salt !== undefined) { + throw new Error(`"${name}" salted after its keys were derived`); + } + + computeSalt(await compute?.()); + }, + + /** + * Splits the items by whether the previous build made their files from + * the same key, carrying the files of those over and recording the + * others' from `files` + * + * @template T + * @param {Array} items + * @param {object} [options] + * @param {(item: T) => import('./types').KeyParts} [options.key] + * @param {(item: T) => string | Array} [options.files] + * @returns {{ fresh: Array, stale: Array }} + */ + partition(items, { key, files } = {}) { + const fresh = []; + const stale = []; + + for (const item of items) { + const previous = previousOutputs[keyOf(item, key)] ?? []; + + if ( + previous.length > 0 && + previous.every(file => existsSync(join(output, file))) + ) { + record(item, ...previous); + fresh.push(item); + } else { + if (files) { + record(item, ...enforceArray(files(item))); + } + + stale.push(item); + } + } + + upToDate += fresh.length; + + return { fresh, stale }; + }, + + /** + * Deletes the previous build's files this build did not record + */ + async prune() { + const recorded = new Set( + [...outputs.values()].flatMap(files => [...files]) + ); + const removed = Object.values(previousOutputs) + .flat() + .filter(file => !recorded.has(file)); + + await Promise.all( + removed.map(file => rm(join(output, file), { force: true })) + ); + + // Directories the removals emptied go too + for (const file of removed) { + for ( + let directory = posix.dirname(file); + directory !== '.'; + directory = posix.dirname(directory) + ) { + try { + await rmdir(join(output, directory)); + } catch { + break; + } + } + } + + incrementalLogger.info( + `Kept ${upToDate} of ${outputs.size} entries, removed ${removed.length} files`, + { generator: name } + ); + }, + + /** + * @returns {import('./types').ManifestSection} + */ + toJSON() { + return { + outputs: Object.fromEntries( + [...outputs.keys()] + .sort() + .map(key => [key, [...outputs.get(key)].sort()]) + ), + }; + }, + }; +}; + +/** + * Creates the store of a run's manifests, one per output directory + * + * @param {{ enabled: boolean }} options + * @returns {import('./types').IncrementalStore} + */ +export const createIncrementalStore = ({ enabled }) => { + /** + * @type {Map> }>>} + */ + const directories = new Map(); + + /** + * @param {string} output - The absolute output directory + */ + const open = output => { + if (!directories.has(output)) { + directories.set( + output, + loadManifest(output).then(({ manifest, reason }) => { + if (reason) { + incrementalLogger.info(`${reason}; building everything`); + } + + return { previous: manifest, sections: new Map() }; + }) + ); + } + + return directories.get(output); + }; + + return { + /** + * @param {import('./types').IncrementalGenerator} generator + * @param {Record | undefined} config - The generator's configuration + * @returns {Promise} + */ + async section(generator, config) { + if (!enabled || !config?.output) { + return DISABLED_SECTION; + } + + const { name } = generator; + const directory = resolve(config.output); + const { previous, sections } = await open(directory); + + if (!sections.has(name)) { + sections.set( + name, + createSection(generator, config, directory, previous.generators[name]) + ); + } + + return sections.get(name); + }, + + /** + * Writes every manifest + */ + async save() { + if (!enabled) { + return; + } + + for (const [directory, state] of directories) { + const { previous, sections } = await state; + const generators = { ...previous.generators }; + + for (const [name, section] of sections) { + generators[name] = section.toJSON(); + } + + await saveManifest(directory, { schema: SCHEMA, generators }); + + incrementalLogger.debug(`Saved manifest`, { + output: directory, + generators: Object.keys(generators), + }); + } + }, + }; +}; diff --git a/packages/core/src/incremental/manifest.mjs b/packages/core/src/incremental/manifest.mjs new file mode 100644 index 000000000..1a34ff57f --- /dev/null +++ b/packages/core/src/incremental/manifest.mjs @@ -0,0 +1,65 @@ +'use strict'; + +import { rename } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; + +import { readJSON, writeJSON } from '#utils/file.mjs'; + +import { MANIFEST_DIRECTORY, MANIFEST_FILE, SCHEMA } from './constants.mjs'; + +/** + * @param {string} output + * @returns {string} + */ +export const manifestPath = output => + join(resolve(output), MANIFEST_DIRECTORY, MANIFEST_FILE); + +/** + * Reads a manifest; an unusable one is empty, with the `reason` + * + * @param {string} output + * @returns {Promise<{ manifest: import('./types').Manifest, reason?: string }>} + */ +export const loadManifest = async output => { + const path = manifestPath(output); + const empty = { schema: SCHEMA, generators: {} }; + + let manifest; + + try { + manifest = await readJSON(path); + } catch (error) { + if (error.code === 'ENOENT') { + return { manifest: empty, reason: `no manifest at ${path}` }; + } + + if (error instanceof SyntaxError) { + return { manifest: empty, reason: `${path} is not valid JSON` }; + } + + throw error; + } + + if (manifest.schema !== SCHEMA) { + return { + manifest: empty, + reason: `${path} has schema ${manifest.schema}, expected ${SCHEMA}`, + }; + } + + return { manifest }; +}; + +/** + * Writes a manifest atomically + * + * @param {string} output + * @param {import('./types').Manifest} manifest + */ +export const saveManifest = async (output, manifest) => { + const path = manifestPath(output); + const temporary = `${path}.tmp`; + + await writeJSON(temporary, manifest); + await rename(temporary, path); +}; diff --git a/packages/core/src/incremental/types.d.ts b/packages/core/src/incremental/types.d.ts new file mode 100644 index 000000000..181bafa28 --- /dev/null +++ b/packages/core/src/incremental/types.d.ts @@ -0,0 +1,59 @@ +export type ManifestSection = { + // The files each item's key produced, relative to the output directory + // (POSIX, no leading slash) + outputs: Record>; +}; + +// The manifest at `/.doc-kit/manifest.json` +export type Manifest = { + schema: number; + generators: Record; +}; + +export type KeyParts = string | Uint8Array | Array; + +// A generator's view of the manifest for one run +export type IncrementalSection = { + // What an item (or a label, for files that belong to no item) produced; + // what is not recorded is pruned + record(item: unknown, ...files: Array): void; + // Adds run-time parts to the salt; computed only when enabled, and only + // before any key is derived + salt(compute?: () => KeyParts[] | Promise): Promise; + // Splits the items by whether the previous build made their files from the + // same key, and records the stale ones' from `files`; `key` overrides the + // declared one + partition( + items: Array, + options?: { + key?: (item: T) => KeyParts; + files?: (item: T) => string | Array; + } + ): { fresh: Array; stale: Array }; + prune(): Promise; +}; + +// What a generator declares for its incremental builds +export type IncrementalOptions = { + // Configuration keys that do not shape the output, left out of the salt + omit?: Array; + // Parts of the salt besides the configuration + salt?: Array; + // What an item's output is made from, hashed with the salt; the item's JSON + // by default + key?: (item: I) => KeyParts; +}; + +export type IncrementalGenerator = Pick< + GeneratorMetadata, + 'name' | 'incremental' +>; + +// A run's manifests, one per output directory +export type IncrementalStore = { + section( + generator: IncrementalGenerator, + config: Record | undefined + ): Promise; + save(): Promise; +}; diff --git a/packages/core/src/threading/parallel.mjs b/packages/core/src/threading/parallel.mjs index 136bac69f..fc193dcc4 100644 --- a/packages/core/src/threading/parallel.mjs +++ b/packages/core/src/threading/parallel.mjs @@ -52,6 +52,7 @@ const createTask = ( // Only pass the needed configuration to this generator configuration: { [generatorName]: configuration[generatorName], + experimentalIncremental: configuration.experimentalIncremental, }, }; }; diff --git a/packages/core/src/utils/__tests__/file.test.mjs b/packages/core/src/utils/__tests__/file.test.mjs index 09bb54f1a..f83719099 100644 --- a/packages/core/src/utils/__tests__/file.test.mjs +++ b/packages/core/src/utils/__tests__/file.test.mjs @@ -1,9 +1,14 @@ 'use strict'; import assert from 'node:assert/strict'; +import { mkdtemp, rm, stat } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { describe, it } from 'node:test'; +import { setTimeout } from 'node:timers/promises'; -import { withExt } from '../file.mjs'; +import { setConfig } from '../configuration/index.mjs'; +import { withExt, writeFile } from '../file.mjs'; describe('withExt', () => { it('should replace an existing extension', () => { @@ -24,3 +29,32 @@ describe('withExt', () => { assert.strictEqual(withExt('file.test.mjs', 'js'), 'file.test.js'); }); }); + +describe('writeFile', () => { + const createFile = async context => { + const directory = await mkdtemp(join(tmpdir(), 'doc-kit-file-')); + context.after(() => rm(directory, { recursive: true, force: true })); + + return join(directory, 'nested', 'file.txt'); + }; + + it('leaves identical bytes alone in an incremental build only', async context => { + const file = await createFile(context); + await setConfig({ experimentalIncremental: true }); + context.after(() => setConfig({})); + + await writeFile(file, 'content'); + const { mtimeMs } = await stat(file); + + // Let the clock move, so an unchanged mtime means no write happened + await setTimeout(20); + + await writeFile(file, Buffer.from('content')); + assert.equal((await stat(file)).mtimeMs, mtimeMs); + + await setConfig({}); + await setTimeout(20); + await writeFile(file, '"content"'); + assert.notEqual((await stat(file)).mtimeMs, mtimeMs); + }); +}); diff --git a/packages/core/src/utils/configuration/__tests__/index.test.mjs b/packages/core/src/utils/configuration/__tests__/index.test.mjs index 140c0f350..d4cfbf33a 100644 --- a/packages/core/src/utils/configuration/__tests__/index.test.mjs +++ b/packages/core/src/utils/configuration/__tests__/index.test.mjs @@ -212,6 +212,7 @@ describe('config.mjs', () => { target: 'json', threads: 4, chunkSize: 5, + experimentalIncremental: true, }; const config = createConfigFromCLIOptions(options); @@ -231,6 +232,7 @@ describe('config.mjs', () => { target: 'json', threads: 4, chunkSize: 5, + experimentalIncremental: true, }); }); diff --git a/packages/core/src/utils/configuration/index.mjs b/packages/core/src/utils/configuration/index.mjs index 9690e0bea..1d66d1a1e 100644 --- a/packages/core/src/utils/configuration/index.mjs +++ b/packages/core/src/utils/configuration/index.mjs @@ -82,6 +82,7 @@ export const getDefaultConfig = (generators, config) => ? 1 : Math.min(cpus().length, DEFAULT_MAX_THREADS), chunkSize: DEFAULT_CHUNK_SIZE, + experimentalIncremental: false, }) ); @@ -229,6 +230,7 @@ export const createConfigFromCLIOptions = options => ({ target: options.target, threads: options.threads, chunkSize: options.chunkSize, + experimentalIncremental: options.experimentalIncremental, }); /** diff --git a/packages/core/src/utils/configuration/types.d.ts b/packages/core/src/utils/configuration/types.d.ts index 2437302bc..7c58d21b6 100644 --- a/packages/core/src/utils/configuration/types.d.ts +++ b/packages/core/src/utils/configuration/types.d.ts @@ -16,6 +16,9 @@ export type Configuration = { // Number of items to process per worker thread chunkSize: number; + + // Enable incremental building? + experimentalIncremental: boolean; } & { [K in keyof AllGenerators]: GlobalConfiguration & AllGenerators[K]['defaultConfiguration']; diff --git a/packages/core/src/utils/file.mjs b/packages/core/src/utils/file.mjs index 314d39743..3b2a4da73 100644 --- a/packages/core/src/utils/file.mjs +++ b/packages/core/src/utils/file.mjs @@ -1,6 +1,8 @@ import fs from 'node:fs/promises'; import { dirname } from 'node:path'; +import getConfig from '#utils/configuration/index.mjs'; + /** * Returns the input string with the `ext` extension, replacing any pre-existing extension * @param {string} str @@ -10,14 +12,38 @@ export const withExt = (str, ext) => `${str.replace(/\.[0-9a-z]+$/i, '')}${ext ? `.${ext}` : ''}`; /** - * Writes a file, recursively + * @param {string} file + * @param {string | Uint8Array} data + */ +const holds = async (file, data) => { + try { + return (await fs.readFile(file)).equals(Buffer.from(data)); + } catch { + return false; + } +}; + +/** + * Writes a file, recursively. An incremental build leaves a file holding the + * same bytes alone. * * @type {typeof fs.writeFile} */ -export const writeFile = (file, ...args) => - fs - .mkdir(dirname(file), { recursive: true }) - .then(() => fs.writeFile(file, ...args)); +export const writeFile = async (file, data, ...args) => { + if (getConfig()?.experimentalIncremental && (await holds(file, data))) { + return; + } + + await fs.mkdir(dirname(file), { recursive: true }); + await fs.writeFile(file, data, ...args); +}; + +/** + * @param {string} file + * @returns {Promise} + */ +export const readJSON = async file => + JSON.parse(await fs.readFile(file, 'utf-8')); /** * Writes a value as JSON diff --git a/packages/react/src/html/README.md b/packages/react/src/html/README.md index caa8610f6..736361015 100644 --- a/packages/react/src/html/README.md +++ b/packages/react/src/html/README.md @@ -215,9 +215,12 @@ generator's `constants.mjs`), so that the page and the library share one Preact. `buildClient` receives `{ entry, virtualImports, config }`. The client `entry` is a single program shared by every page. It must be bundled into `config.output` and the call must return -`{ scripts, preloads, stylesheets, fonts }`: paths relative to the output root -of the module scripts to load, the chunks they statically import (rendered as -`modulepreload` hints), the stylesheets, and optionally the fonts to preload. +`{ scripts, preloads, stylesheets, fonts, emitted }`: paths relative to the +output root of the module scripts to load, the chunks they statically import +(rendered as `modulepreload` hints), the stylesheets, optionally the fonts to +preload, and optionally every file the build wrote, for an incremental build +to remove the ones a later build does not write again (without it, only the +files every page loads are tracked). The generator renders those into every page, resolved against the page's location. Name every file after its content (a hash), so hosts can cache them indefinitely (see [Publishing](../../../docs/publishing.md#cache-the-assets)). @@ -513,6 +516,23 @@ The populated page is the final HTML: it is minified when `minify` is set and written as is. Put `${assets}` in the ``, or the page loads no script and no stylesheet. +## Incremental builds + +With [`experimentalIncremental`](../../../../docs/configuration.md#execution-options) +on, a page is rendered again only when its key changed. The key is the page +itself: its JSX, metadata, table of contents and reading time. `all.html` is +keyed on the module pages it is made of. + +Every key also carries what every page depends on: the built component +library, hashed by its bytes (it bundles every component and the +`#theme/config` module, so a change to a component, a dependency or the page +set changes it), the client assets, the template, and the `html` +configuration minus its paths and functions. + +The files the client build wrote are recorded too, so a later build removes +the ones it no longer emits; see `emitted` under +[bundler adapters](#bundler-adapters). + ## Client-side navigation The site moves between its own pages client-side, through the diff --git a/packages/react/src/html/__tests__/generate.test.mjs b/packages/react/src/html/__tests__/generate.test.mjs index a2198925b..62256abfa 100644 --- a/packages/react/src/html/__tests__/generate.test.mjs +++ b/packages/react/src/html/__tests__/generate.test.mjs @@ -1,11 +1,25 @@ import assert from 'node:assert/strict'; -import { mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { existsSync } from 'node:fs'; +import { + mkdtemp, + readdir, + readFile, + rm, + stat, + writeFile, +} from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, it } from 'node:test'; +import { setTimeout } from 'node:timers/promises'; import { pathToFileURL } from 'node:url'; import { loadGenerator } from '@doc-kit/core/generators/loader.mjs'; +import { + createIncrementalStore, + DISABLED_SECTION, +} from '@doc-kit/core/incremental/index.mjs'; +import { loadManifest } from '@doc-kit/core/incremental/manifest.mjs'; import { setConfig } from '@doc-kit/core/utils/configuration/index.mjs'; import { loadMarkdownPlugins } from '@doc-kit/core/utils/markdown/plugins.mjs'; import { jsx, toJs } from 'estree-util-to-js'; @@ -16,12 +30,30 @@ import { generate as chunk } from '../../section-pages/generate.mjs'; import { compile, createViteBundler } from '../bundlers/vite.mjs'; import { FONTS } from '../constants.mjs'; import { generate } from '../generate.mjs'; +import html from '../index.mjs'; +import { pageFileName } from '../utils/processing.mjs'; +import { processChunk } from '../utils/render.mjs'; // Pages are rendered with the pipeline of `jsx-ast` await loadMarkdownPlugins( await loadGenerator(import.meta.resolve('../../jsx-ast/index.mjs')) ); +/** + * Generates the site on this thread, as the orchestrator does without the + * incremental option. + */ +const generateSite = pages => + generate(pages, undefined, { incremental: DISABLED_SECTION }); + +/** + * Builds the pages of module entries, one page per module. + */ +const buildPages = async entries => + Promise.all(entries.map(entry => buildContent([entry], entry))).then( + contents => contents.map(toPage) + ); + /** * Converts a page's JSX AST into the `{ data, headings, readingTime, content }` * shape `html` consumes, mirroring the conversion the jsx-ast worker performs. @@ -94,7 +126,7 @@ describe('web generate', () => { buildContent(notFoundPage.entries, notFoundPage.head), ]); - await generate(contents.map(toPage)); + await generateSite(contents.map(toPage)); const [fsHTML, notFoundHTML] = await Promise.all([ readFile(join(output, 'api/fs.html'), 'utf8'), @@ -137,11 +169,7 @@ describe('web generate', () => { createEntry('index', 'Index'), ]; - await generate( - await Promise.all( - entries.map(entry => buildContent([entry], entry)) - ).then(contents => contents.map(toPage)) - ); + await generateSite(await buildPages(entries)); const html = await readFile(join(output, 'all.html'), 'utf8'); @@ -201,7 +229,7 @@ describe('web generate', () => { [...pages.values()].map(group => buildContent(group, group[0])) ); - await generate(contents.map(toPage)); + await generateSite(contents.map(toPage)); const [fsHTML, readFileHTML] = await Promise.all([ readFile(join(output, 'fs.html'), 'utf8'), @@ -247,7 +275,7 @@ describe('web generate', () => { }; const fs = createEntry('fs', 'File system'); - await generate([toPage(await buildContent([fs], fs))]); + await generateSite([toPage(await buildContent([fs], fs))]); const html = await readFile(join(output, 'fs.html'), 'utf8'); assert.match(html, /Custom project docs/); @@ -265,7 +293,7 @@ describe('web generate', () => { const notFoundPage = buildNotFoundPage(); const content = await buildContent(notFoundPage.entries, notFoundPage.head); - await generate([toPage(content)]); + await generateSite([toPage(content)]); const html = await readFile(join(output, '404.html'), 'utf8'); assert.match(html, /src="https:\/\/example\.com\/docs\/assets\//); @@ -288,7 +316,7 @@ describe('web generate', () => { }); const fs = createEntry('fs', 'File system'); - await generate([toPage(await buildContent([fs], fs))]); + await generateSite([toPage(await buildContent([fs], fs))]); const assets = await readdir(join(output, 'assets')); const client = assets.find(file => /^client-.*\.js$/.test(file)); @@ -361,7 +389,7 @@ describe('web generate', () => { }; const fs = createEntry('fs', 'File system'); - await generate([toPage(await buildContent([fs], fs))]); + await generateSite([toPage(await buildContent([fs], fs))]); const html = await readFile(join(output, 'fs.html'), 'utf8'); assert.match(html, /
fs:1<\/article>/); @@ -382,4 +410,219 @@ describe('web generate', () => { ); assert.deepStrictEqual(calls, ['server', 'client', 'compile']); }); + + describe('incremental', () => { + /** + * Runs an incremental build of the pages into the output, through a + * worker that renders on this thread and notes which pages it rendered. + * + * @returns {Promise<{ rendered: Array, manifest: import('@doc-kit/core/incremental/types').Manifest }>} + */ + const build = async (pages, output) => { + const rendered = []; + + const worker = { + async *stream(tasks, extra) { + rendered.push(...tasks.map(task => pageFileName(task.data))); + + yield await processChunk(tasks, [...tasks.keys()], extra); + }, + }; + + const store = createIncrementalStore({ enabled: true }); + const incremental = await store.section(html, { output }); + + await generate(pages, worker, { incremental }); + await store.save(); + + // Let the clock move, so an unchanged mtime means no write happened + await setTimeout(20); + + return { rendered, manifest: (await loadManifest(output)).manifest }; + }; + + /** + * The modification times of the given files, by file. + */ + const mtimes = async (output, files) => + Object.fromEntries( + await Promise.all( + files.map(async file => [ + file, + (await stat(join(output, file))).mtimeMs, + ]) + ) + ); + + it('renders nothing the second time and leaves every file alone', async context => { + const { config, output } = await createTestConfiguration(context); + config.html.generateAllPage = true; + + const pages = await buildPages([ + createEntry('fs', 'File system'), + createEntry('zlib', 'Zlib'), + ]); + + const first = await build(pages, output); + + assert.deepEqual(first.rendered.toSorted(), [ + 'all.html', + 'fs.html', + 'zlib.html', + ]); + + const files = Object.values( + first.manifest.generators.html.outputs + ).flat(); + const pageFiles = files.filter(file => !file.startsWith('assets/')); + const before = await mtimes(output, pageFiles); + + const second = await build(pages, output); + + // The client is bundled again (into the same files); the pages are not + assert.deepEqual(second.rendered, []); + assert.deepEqual(await mtimes(output, pageFiles), before); + + // Building the same library twice gives the same bytes: the salt holds + assert.deepEqual(second.manifest, first.manifest); + + // The pages and the client's files are all recorded + assert.ok(files.includes('fs.html')); + assert.ok(files.some(file => file.startsWith('assets/'))); + }); + + it('renders an edited page again, and all.html with it, but nothing else', async context => { + const { config, output } = await createTestConfiguration(context); + config.html.generateAllPage = true; + + const fs = createEntry('fs', 'File system'); + const zlib = createEntry('zlib', 'Zlib'); + + await build(await buildPages([fs, zlib]), output); + const before = await mtimes(output, ['fs.html', 'zlib.html', 'all.html']); + + // A change after the first paragraph: the description every page's + // documentation index shows stays the same, so the library does too + zlib.content.children.push({ + type: 'paragraph', + children: [{ type: 'text', value: 'More about zlib' }], + }); + + const { rendered } = await build(await buildPages([fs, zlib]), output); + const after = await mtimes(output, ['fs.html', 'zlib.html', 'all.html']); + + assert.deepEqual(rendered, ['all.html', 'zlib.html']); + assert.equal(after['fs.html'], before['fs.html']); + assert.notEqual(after['zlib.html'], before['zlib.html']); + assert.notEqual(after['all.html'], before['all.html']); + assert.match( + await readFile(join(output, 'all.html'), 'utf8'), + /More about zlib/ + ); + }); + + it('renders every page again when the page set changes', async context => { + const { config, output } = await createTestConfiguration(context); + config.html.generateAllPage = false; + + const fs = createEntry('fs', 'File system'); + const zlib = createEntry('zlib', 'Zlib'); + + await build(await buildPages([fs, zlib]), output); + + // A new module is on every page's sidebar + const { rendered } = await build( + await buildPages([fs, zlib, createEntry('net', 'Net')]), + output + ); + + assert.deepEqual(rendered.toSorted(), [ + 'fs.html', + 'net.html', + 'zlib.html', + ]); + }); + + it('removes the pages of a dropped module, and nothing else', async context => { + const { config, output } = await createTestConfiguration(context, [ + 'section-pages', + ]); + config.html.generateAllPage = false; + config['section-pages'].maxDepth = 2; + + const fs = [ + createEntry('fs', 'File system'), + createEntry('fs', '`fs.readFile()`', { + depth: 2, + slug: 'fsreadfile', + type: 'method', + name: 'readFile', + }), + createEntry('fs', '`fs.watch()`', { + depth: 2, + slug: 'fswatch', + type: 'method', + name: 'watch', + }), + ]; + const zlib = createEntry('zlib', 'Zlib'); + + const toPages = async entries => { + const groups = Map.groupBy(await chunk(entries), entry => entry.api); + + return Promise.all( + [...groups.values()].map(group => buildContent(group, group[0])) + ).then(contents => contents.map(toPage)); + }; + + await build(await toPages([...fs, zlib]), output); + + assert.ok(existsSync(join(output, 'fs/readFile.html'))); + await writeFile(join(output, 'mine.txt'), 'not doc-kit’s'); + + const { rendered, manifest } = await build(await toPages([zlib]), output); + + // The sidebar lost a module, so the remaining page is rendered again + assert.deepEqual(rendered, ['zlib.html']); + assert.equal(existsSync(join(output, 'fs.html')), false); + assert.equal(existsSync(join(output, 'fs')), false); + assert.ok(existsSync(join(output, 'zlib.html'))); + assert.ok(existsSync(join(output, 'mine.txt'))); + assert.deepEqual( + Object.values(manifest.generators.html.outputs) + .flat() + .filter(file => !file.startsWith('assets/')), + ['zlib.html'] + ); + }); + + it('removes the client files a previous build wrote and this one did not', async context => { + const { config, output } = await createTestConfiguration(context); + config.html.generateAllPage = false; + + const pages = await buildPages([createEntry('fs', 'File system')]); + + await build(pages, output); + + const stylesheets = async () => + (await readdir(join(output, 'assets'))).filter(file => + file.endsWith('.css') + ); + + const [before] = await stylesheets(); + + const stylesheet = join(output, '..', `doc-kit-test-${Date.now()}.css`); + context.after(() => rm(stylesheet, { force: true })); + await writeFile(stylesheet, '.extra { color: red }'); + config.html.stylesheets = [stylesheet]; + + const { rendered } = await build(pages, output); + + // The assets changed, so the page did, and the old stylesheet is gone + assert.deepEqual(rendered, ['fs.html']); + const after = await stylesheets(); + assert.equal(after.length, 1); + assert.notEqual(after[0], before); + }); + }); }); diff --git a/packages/react/src/html/bundlers/vite.mjs b/packages/react/src/html/bundlers/vite.mjs index 95a913abf..352d49c08 100644 --- a/packages/react/src/html/bundlers/vite.mjs +++ b/packages/react/src/html/bundlers/vite.mjs @@ -1,7 +1,8 @@ -import { readFile, rm, rmdir } from 'node:fs/promises'; +import { rm, rmdir } from 'node:fs/promises'; import { dirname, join, posix, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; +import { readJSON } from '@doc-kit/core/utils/file.mjs'; import { build as viteBuild, defaultClientConditions, @@ -387,7 +388,7 @@ export const buildClient = async ({ const outDir = resolve(config.output); - await viteBuild( + const result = await viteBuild( createViteConfig({ sources, input: { [CLIENT_NAME]: CLIENT_ENTRY_ID }, @@ -407,11 +408,15 @@ export const buildClient = async ({ typeof configuredManifest === 'string' ? configuredManifest : MANIFEST_NAME ); - const manifest = JSON.parse(await readFile(manifestPath, 'utf8')); + const manifest = await readJSON(manifestPath); + + let emitted = result.output.map(({ fileName }) => fileName); if (!configuredManifest) { await rm(manifestPath); await rmdir(dirname(manifestPath)).catch(() => {}); + + emitted = emitted.filter(file => file !== MANIFEST_NAME); } const entries = Object.values(manifest); @@ -434,6 +439,7 @@ export const buildClient = async ({ preloads: collectImports(manifest, chunk), stylesheets: [...new Set(stylesheets)], fonts, + emitted, }; }; diff --git a/packages/react/src/html/generate.mjs b/packages/react/src/html/generate.mjs index 83f7ba23e..db4fd9f5f 100644 --- a/packages/react/src/html/generate.mjs +++ b/packages/react/src/html/generate.mjs @@ -5,13 +5,16 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { pathToFileURL } from 'node:url'; +import { hashDirectory } from '@doc-kit/core/incremental/index.mjs'; import logger from '@doc-kit/core/logger/index.mjs'; import getConfig from '@doc-kit/core/utils/configuration/index.mjs'; +import { omitKeys } from '@doc-kit/core/utils/misc.mjs'; import { resolveBundler } from './bundlers/index.mjs'; import { buildAllPage } from './utils/all.mjs'; import { copyStaticAssets } from './utils/copying.mjs'; import createProgramBuilder, { moduleFileName } from './utils/generate.mjs'; +import { pageFileName } from './utils/processing.mjs'; import { createPageWriter } from './utils/render.mjs'; const htmlLogger = logger.child('html'); @@ -34,7 +37,7 @@ const htmlLogger = logger.child('html'); * * @type {import('./types').Generator['generate']} */ -export async function generate(input, worker) { +export async function generate(input, worker, { incremental }) { const config = getConfig('html'); const template = await readFile(config.templatePath, 'utf-8'); @@ -48,12 +51,16 @@ export async function generate(input, worker) { pages: [...input], outDir, config, + template, + incremental, }); // The pages' programs are compiled, so rendering doesn't need their code input = undefined; await createPageWriter(worker)(tasks, { template, assets }); + + await incremental.prune(); } finally { await rm(outDir, { recursive: true, force: true }); } @@ -62,8 +69,8 @@ export async function generate(input, worker) { } /** - * Bundles the component library and the client assets, and compiles every - * page's program into `outDir`, for the worker pool to render. + * Bundles the component library and the client assets, and compiles the stale + * pages' programs into `outDir`, for the worker pool to render. * * Everything it sets up for that, the bundler included, is let go once it * returns: with one thread, the pages are rendered on this thread next. @@ -72,14 +79,23 @@ export async function generate(input, worker) { * @param {Array} options.pages - Every page, in render order * @param {string} options.outDir - Where the library and the programs are written * @param {import('./types').ResolvedWebConfiguration} options.config + * @param {string} options.template + * @param {import('@doc-kit/core/incremental/types').IncrementalSection} options.incremental * @returns {Promise<{ assets: import('./types').ClientAssets, tasks: Array }>} */ -const buildPrograms = async ({ pages, outDir, config }) => { +const buildPrograms = async ({ + pages, + outDir, + config, + template, + incremental, +}) => { const all = config.generateAllPage ? buildAllPage(pages) : undefined; + const allPages = all ? [...pages, all] : pages; // Every page's metadata, in render order — the sidebar, the index and the // cross links need the whole set. - const datas = [...pages, ...(all ? [all] : [])].map(({ data }) => data); + const datas = allPages.map(({ data }) => data); // Loaded here rather than with the generator, so the threads rendering // pages, which load this module too, never load what only this needs @@ -106,6 +122,28 @@ const buildPrograms = async ({ pages, outDir, config }) => { htmlLogger.debug('Built the client assets', assets); + // Everything a page depends on besides itself. The library's bytes cover + // every component, dependency and the page set it embeds; only the library + // is in `outDir` yet + await incremental.salt(async () => [ + await hashDirectory(outDir), + JSON.stringify(assets), + template, + ]); + + const stale = partitionPages({ pages: allPages, all, incremental }); + + // A bundler not saying what it wrote only has the files every page loads + // tracked + const emitted = assets.emitted ?? [ + ...assets.scripts, + ...assets.preloads, + ...assets.stylesheets, + ...(assets.fonts ?? []), + ]; + + incremental.record('client', ...emitted); + const modulesDir = join(outDir, 'pages'); await mkdir(modulesDir); @@ -136,25 +174,80 @@ const buildPrograms = async ({ pages, outDir, config }) => { }; }; + // The composed page imports the module pages' programs, so they are compiled + // whenever it is rendered + const allIsStale = all !== undefined && stale.has(all); + const parts = new Set(allIsStale ? all.parts : []); + + const toCompile = pages.filter( + page => stale.has(page) || parts.has(page.data.api) + ); const tasks = []; - for (const page of pages) { - tasks.push(await compile(page)); + for (const page of toCompile) { + const task = await compile(page); + + if (stale.has(page)) { + tasks.push(task); + } } - // The composed page imports the other pages' compiled programs, which - // exist from here on, so it is rendered by the worker pool alongside them. + // The composed page is rendered by the worker pool alongside the others. // Rendering it on this thread instead would hold the whole site in it and // block the pool from shutting down its idle workers until it is done. // It goes first: it takes by far the longest, so the other pages are // rendered while it is. - if (all) { - const allTask = await compile(all); - - tasks.unshift(allTask); + if (allIsStale) { + tasks.unshift(await compile(all)); } - htmlLogger.debug(`Compiled ${tasks.length} page programs`); + htmlLogger.debug( + `Compiled ${toCompile.length + (allIsStale ? 1 : 0)} page programs, ` + + `${tasks.length} to render` + ); return { assets, tasks }; }; + +/** + * A page's metadata + * + * @param {import('./types').Page} page + * @returns {string} + */ +const pageJSON = ({ data, headings, readingTime }) => + JSON.stringify( + { data: omitKeys(data, ['content']), headings, readingTime }, + (key, value) => + key === 'position' || key === 'yamlPosition' ? undefined : value + ); + +/** + * Picks the pages to render + * + * @param {object} options + * @param {Array} options.pages + * @param {import('./types').ComposedPage | undefined} options.all + * @param {import('@doc-kit/core/incremental/types').IncrementalSection} options.incremental + * @returns {Set} + */ +const partitionPages = ({ pages, all, incremental }) => { + const byApi = new Map(pages.map(page => [page.data.api, page])); + + /** + * @param {import('./types').Page} page + * @returns {Array} + */ + const keyParts = page => + page === all + ? [pageJSON(all), ...all.parts.flatMap(api => keyParts(byApi.get(api)))] + : [pageJSON(page), page.content]; + + const { stale } = incremental.partition(pages, { + key: keyParts, + /** @param {import('./types').Page} page */ + files: page => pageFileName(page.data), + }); + + return new Set(stale); +}; diff --git a/packages/react/src/html/index.mjs b/packages/react/src/html/index.mjs index 607592a39..7ecd1241a 100644 --- a/packages/react/src/html/index.mjs +++ b/packages/react/src/html/index.mjs @@ -92,6 +92,18 @@ export default { generate, + incremental: { + // Keys holding functions or paths; what they point at is hashed by content + omit: [ + 'bundler', + 'templatePath', + 'imports', + 'virtualImports', + 'stylesheets', + 'components', + ], + }, + // Rendering, templating, minifying and writing the pages scales with the // page count, so it is farmed out to the worker pool. hasParallelProcessor: true, diff --git a/packages/react/src/html/types.d.ts b/packages/react/src/html/types.d.ts index 5bfb72510..7772999f5 100644 --- a/packages/react/src/html/types.d.ts +++ b/packages/react/src/html/types.d.ts @@ -47,6 +47,12 @@ export type PageTask = Pick & { moduleURL: string; }; +// What the workers render every page with. +export type RenderOptions = { + template: string; + assets: ClientAssets; +}; + // The client assets every page loads, as paths relative to the output root. export type ClientAssets = { // Module scripts, in load order. @@ -57,6 +63,9 @@ export type ClientAssets = { stylesheets: Array; // Fonts to preload, so text does not wait on the stylesheet to request them. fonts?: Array; + // Every file the build wrote, for an incremental build to prune; without it, + // only the files above are tracked. + emitted?: Array; }; export type ServerBundleOptions = { diff --git a/packages/react/src/html/utils/render.mjs b/packages/react/src/html/utils/render.mjs index 4758e9c62..57d34dce6 100644 --- a/packages/react/src/html/utils/render.mjs +++ b/packages/react/src/html/utils/render.mjs @@ -1,10 +1,10 @@ 'use strict'; -import { mkdir, writeFile } from 'node:fs/promises'; -import { dirname, join } from 'node:path'; +import { join } from 'node:path'; import logger from '@doc-kit/core/logger/index.mjs'; import getConfig from '@doc-kit/core/utils/configuration/index.mjs'; +import { writeFile } from '@doc-kit/core/utils/file.mjs'; import { omitKeys } from '@doc-kit/core/utils/misc.mjs'; import { pageFileName, populatePage } from './processing.mjs'; @@ -25,7 +25,7 @@ const renderLogger = logger.child('html'); * * @param {Array} tasks * @param {Array} indices - The tasks to process - * @param {{ template: string, assets: import('../types').ClientAssets }} extra + * @param {import('../types').RenderOptions} extra * @returns {Promise>} The written file names */ export const processChunk = async (tasks, indices, { template, assets }) => { @@ -64,7 +64,6 @@ export const processChunk = async (tasks, indices, { template, assets }) => { const fileName = pageFileName(data); const path = join(config.output, fileName); - await mkdir(dirname(path), { recursive: true }); await writeFile(path, html); written.push(fileName); @@ -80,7 +79,7 @@ export const processChunk = async (tasks, indices, { template, assets }) => { * calling thread instead. * * @param {ParallelWorker} [worker] - * @returns {(tasks: Array, extra: { template: string, assets: import('../types').ClientAssets }) => Promise} + * @returns {(tasks: Array, extra: import('../types').RenderOptions) => Promise} */ export const createPageWriter = worker => async (tasks, extra) => { const chunks = worker diff --git a/scripts/comparators/files.mjs b/scripts/comparators/files.mjs index 7e7d1035c..ed240ce03 100644 --- a/scripts/comparators/files.mjs +++ b/scripts/comparators/files.mjs @@ -1,6 +1,7 @@ import { glob } from 'node:fs/promises'; import path from 'node:path'; +import { MANIFEST_DIRECTORY } from '../../packages/core/src/incremental/constants.mjs'; import { BENCHMARK_FILE, COMPARISON_FILE } from '../constants.mjs'; const METADATA_FILES = new Set([BENCHMARK_FILE, COMPARISON_FILE]); @@ -9,7 +10,8 @@ export const listOutputFiles = async directory => { const entries = glob('**/*', { cwd: directory, withFileTypes: true, - exclude: entry => METADATA_FILES.has(entry.name), + exclude: entry => + METADATA_FILES.has(entry.name) || entry.name === MANIFEST_DIRECTORY, }); return (await Array.fromAsync(entries))