feat: incremental building

.changeset/experimental-incremental.md

@@ -0,0 +1,7 @@
+---
+'@doc-kit/cli': minor
+'@doc-kit/core': minor
+'@doc-kit/generator-react': minor
+---
+
+Add `doc-kit generate --experimental-incremental`

docs/cli.md

@@ -40,6 +40,7 @@ Runs the generators and writes their output. Requires a `target` and an
   links).
 - `-p, --threads <n>` {number} Worker threads to use (minimum 1).
 - `--chunk-size <n>` {number} Items per worker thread (minimum 1).
+- `--experimental-incremental` {boolean} Rebuild against the existing output. **Experimental.**
 
 ## `doc-kit preview`
 

docs/configuration.md

@@ -137,6 +137,9 @@ Top-level, alongside `target` and `global`:
   CPU count. Each worker's heap is limited to 512MB; to raise the limit, run
   Node.js with `--max-old-space-size` (directly or through `NODE_OPTIONS`).
 - `chunkSize` {number} Items processed per worker thread. **Default:** `10`.
+- `experimentalIncremental` {boolean} Build against the existing output
+  instead of from scratch. **Default:** `false`. **Experimental:** the option
+  and the manifest format are subject to change.
 
 ## Generator options
 
@@ -262,17 +265,18 @@ precedence):
 
 CLI options map to configuration properties:
 
-| CLI Option             | Config Property    | Example                   |
-| ---------------------- | ------------------ | ------------------------- |
-| `--input <path>`       | `global.input`     | `--input src/`            |
-| `--output <path>`      | `global.output`    | `--output dist/`          |
-| `--ignore <pattern>`   | `global.ignore[]`  | `--ignore test/`          |
-| `--minify`             | `global.minify`    | `--minify`                |
-| `--git-ref <ref>`      | `global.ref`       | `--git-ref v20.0.0`       |
-| `--version <version>`  | `global.version`   | `--version 20.0.0`        |
-| `--changelog <url>`    | `global.changelog` | `--changelog https://...` |
-| `--index <url>`        | `global.index`     | `--index file://...`      |
-| `--type-map <map>`     | `metadata.typeMap` | `--type-map file://...`   |
-| `--target <generator>` | `target`           | `--target json`           |
-| `--threads <n>`        | `threads`          | `--threads 4`             |
-| `--chunk-size <n>`     | `chunkSize`        | `--chunk-size 10`         |
+| CLI Option                   | Config Property           | Example                      |
+| ---------------------------- | ------------------------- | ---------------------------- |
+| `--input <path>`             | `global.input`            | `--input src/`               |
+| `--output <path>`            | `global.output`           | `--output dist/`             |
+| `--ignore <pattern>`         | `global.ignore[]`         | `--ignore test/`             |
+| `--minify`                   | `global.minify`           | `--minify`                   |
+| `--git-ref <ref>`            | `global.ref`              | `--git-ref v20.0.0`          |
+| `--version <version>`        | `global.version`          | `--version 20.0.0`           |
+| `--changelog <url>`          | `global.changelog`        | `--changelog https://...`    |
+| `--index <url>`              | `global.index`            | `--index file://...`         |
+| `--type-map <map>`           | `metadata.typeMap`        | `--type-map file://...`      |
+| `--target <generator>`       | `target`                  | `--target json`              |
+| `--threads <n>`              | `threads`                 | `--threads 4`                |
+| `--chunk-size <n>`           | `chunkSize`               | `--chunk-size 10`            |
+| `--experimental-incremental` | `experimentalIncremental` | `--experimental-incremental` |

docs/creating-generators.md

@@ -149,6 +149,13 @@ function transformToMyFormat(entries, version) {
 }
 ```
 
+`generate` is called with the input, a worker for
+[parallel processing](#parallel-processing-with-workers), and the generator
+context (`GeneratorContext`): what the orchestrator provides besides the
+input. Its one property so far is `incremental`, the generator's section of
+the manifest of an [incremental build](#incremental-builds); when the build
+is not incremental, its methods are all no-ops.
+
 ### Step 5: Make the Generator Loadable
 
 Generators are loaded dynamically by import specifier. Anything that resolves
@@ -567,6 +574,57 @@ export async function generate(input, worker) {
 }
 ```
 
+### Incremental builds
+
+An incremental build makes again only the items whose output the previous
+build did not make from the same key, and keeps the manifest of what every
+item produced at `<output>/.doc-kit/manifest.json`. The `incremental`
+property of the [generator context](#step-4-implement-the-generator-logic) is
+the generator's section of it:
+
+```mjs displayName="generate.mjs"
+import { writeFile } from '@doc-kit/core/utils/file.mjs';
+
+export async function generate(input, worker, { incremental }) {
+  // `stale` are the items to make again; the rest keep their files. `files`
+  // names what an item makes, be it one file or many.
+  const { stale } = incremental.partition(input, {
+    files: item => `${item.api}.myformat`,
+  });
+
+  for (const item of stale) {
+    await writeFile(join(config.output, `${item.api}.myformat`), render(item));
+  }
+
+  // Removes the previous build's files this one did not record
+  await incremental.prune();
+}
+```
+
+An item's key is its JSON, hashed with the generator's configuration, so a
+configuration change makes everything again. The generator's metadata can
+refine that under `incremental`:
+
+- `key(item)` is what an item's output is made from instead: a string, bytes,
+  or an array of them.
+- `omit` lists the configuration keys that do not shape the output, such as
+  paths and functions. `input`, `output`, `ignore` and `pathsToCopy` are
+  always left out.
+- `salt` lists further parts of every key, such as a format version.
+
+`partition(items, { key, files })` takes a key function in place of the
+declared one, and `salt(compute)` adds parts computed at run time, such as
+the hash of a built component library; `compute` only runs in an incremental
+build, and before any key is derived. Files that belong to no item, such as
+assets, or whose names are only known once made, are recorded by hand:
+`record(item, ...files)`, with a label in place of the item for the former.
+
+Without `experimentalIncremental` (or an `output`) every item is stale and
+nothing is recorded, so the generator needs no branches of its own. Whether
+or not a generator uses its section, writing through core's `writeFile` (or
+`writeJSON`) leaves a file alone during an incremental build when it already
+holds the same bytes.
+
 ### Copying Assets
 
 ```mjs displayName="generate.mjs"

packages/cli/bin/commands/generate.mjs

@@ -25,6 +25,7 @@ const { runGenerators } = createGenerator();
  * @property {string} index
  * @property {boolean} minify
  * @property {string} typeMap
+ * @property {boolean} experimentalIncremental
  */
 
 /**
@@ -53,6 +54,10 @@ export const generateOptions = [
   new Option('--index <url>', 'index.md URL or path'),
   new Option('--minify', 'Minify?'),
   new Option('--type-map <url>', 'Type map URL or path'),
+  new Option(
+    '--experimental-incremental',
+    'Rebuild only what changed against the existing output (experimental)'
+  ),
 ];
 
 const generate = new Command('generate').description('Generate API docs');

packages/core/package.json

@@ -34,6 +34,7 @@
   },
   "imports": {
     "#generators/*": "./src/generators/*",
+    "#incremental/*": "./src/incremental/*",
     "#logger/*": "./src/logger/*",
     "#parsers/*": [
       "./src/parsers/*",

packages/core/src/__tests__/generators.test.mjs

@@ -1,6 +1,13 @@
 import assert from 'node:assert/strict';
+import { existsSync } from 'node:fs';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
 import { describe, it, mock, beforeEach } from 'node:test';
 
+import { DISABLED_SECTION } from '../incremental/index.mjs';
+import { loadManifest, manifestPath } from '../incremental/manifest.mjs';
+
 // Tracks how many times each synthetic generator actually executed so we can
 // assert that a shared dependency runs exactly once even after its result is
 // evicted from the cache once its last consumer has read it.
@@ -99,6 +106,25 @@ const syntheticGenerators = {
       return { all: input };
     },
   },
+  // A generator recording an output through the incremental section it is
+  // handed, and one failing after it
+  'gen-record': {
+    name: 'gen-record',
+    dependsOn: 'metadata',
+    generate: async (input, worker, { incremental }) => {
+      record('gen-record');
+      incremental.record('recorded', 'recorded.txt');
+      return { disabled: incremental === DISABLED_SECTION };
+    },
+  },
+  'gen-fail': {
+    name: 'gen-fail',
+    dependsOn: 'metadata',
+    generate: async () => {
+      record('gen-fail');
+      throw new Error('boom');
+    },
+  },
 };
 
 mock.module('../generators/loader.mjs', {
@@ -245,4 +271,65 @@ describe('createGenerator orchestration', () => {
       { name: 'gen-a', markdown, ran: ['ast', 'metadata'] },
     ]);
   });
+
+  describe('incremental building', () => {
+    const createOutput = async context => {
+      const output = await mkdtemp(join(tmpdir(), 'doc-kit-orchestrator-'));
+      context.after(() => rm(output, { recursive: true, force: true }));
+
+      return output;
+    };
+
+    it('hands each generator its section and saves the manifest once every target is done', async context => {
+      const output = await createOutput(context);
+      const { runGenerators } = createGenerator();
+
+      const results = await runGenerators({
+        target: ['gen-record'],
+        threads: 1,
+        experimentalIncremental: true,
+        'gen-record': { output },
+      });
+
+      assert.deepStrictEqual(results, [{ disabled: false }]);
+
+      const { manifest } = await loadManifest(output);
+      assert.deepStrictEqual(
+        Object.values(manifest.generators['gen-record'].outputs),
+        [['recorded.txt']]
+      );
+    });
+
+    it('writes no manifest when a generator fails', async context => {
+      const output = await createOutput(context);
+      const { runGenerators } = createGenerator();
+
+      await assert.rejects(
+        runGenerators({
+          target: ['gen-record', 'gen-fail'],
+          threads: 1,
+          experimentalIncremental: true,
+          'gen-record': { output },
+          'gen-fail': { output },
+        }),
+        /boom/
+      );
+
+      assert.equal(existsSync(manifestPath(output)), false);
+    });
+
+    it('hands out disabled sections and writes nothing without the flag', async context => {
+      const output = await createOutput(context);
+      const { runGenerators } = createGenerator();
+
+      const results = await runGenerators({
+        target: ['gen-record'],
+        threads: 1,
+        'gen-record': { output },
+      });
+
+      assert.deepStrictEqual(results, [{ disabled: true }]);
+      assert.equal(existsSync(manifestPath(output)), false);
+    });
+  });
 });

packages/core/src/generators.mjs

@@ -6,6 +6,7 @@ import {
   resolveGeneratorSpecifier,
 } from './generators/loader.mjs';
 import { resolvePipeline } from './generators/pipeline.mjs';
+import { createIncrementalStore } from './incremental/index.mjs';
 import logger from './logger/index.mjs';
 import createWorkerPool from './threading/index.mjs';
 import createParallelWorker from './threading/parallel.mjs';
@@ -27,6 +28,9 @@ const createGenerator = () => {
   /** @type {import('piscina').Piscina} */
   let pool;
 
+  /** @type {import('./incremental/types').IncrementalStore} */
+  let incremental;
+
   /**
    * Gets the collected input from a dependency generator.
    *
@@ -89,7 +93,12 @@ const createGenerator = () => {
 
         // Returned rather than awaited, so this function doesn't hold on to the
         // input while the generator runs, and the generator can let it go
-        return generate(dependencyInput, await worker);
+        return generate(dependencyInput, await worker, {
+          incremental: await incremental.section(
+            generator,
+            configuration[name]
+          ),
+        });
       })().then(result => {
         // For streaming generators, "Completed" is logged when the cache
         // finishes collecting, not here when the generator returns
@@ -136,6 +145,10 @@ const createGenerator = () => {
     // Create worker pool
     pool = createWorkerPool(threads);
 
+    incremental = createIncrementalStore({
+      enabled: configuration.experimentalIncremental === true,
+    });
+
     // Schedule all generators
     for (const specifier of targets) {
       scheduleGenerator(specifier, generators, inputOf, configuration);
@@ -147,6 +160,8 @@ const createGenerator = () => {
       targets.map(specifier => cache.consume(specifier))
     );
 
+    await incremental.save();
+
     await pool.destroy();
 
     return results;