internal(ci): Cut CircleCI critical path and skip tests on docs-only changes

.circleci/config.yml

@@ -21,19 +21,23 @@ executors:
       YARN_ENABLE_IMMUTABLE_INSTALLS: true
 
 commands:
-  halt-unless-esmodule-relevant-change:
-    # Reads the flag computed once in the `setup` job. Transported via cache
+  halt-unless-relevant-change:
+    # Reads a flag computed once in the `setup` job. Transported via cache
     # (not the workspace) so jobs can halt before paying attach_workspace.
+    parameters:
+      flag:
+        type: enum
+        enum: ["esmodule", "tests"]
     steps:
       - restore_cache:
           keys:
-            - esmodule-relevant-v1-{{ .Environment.CIRCLE_SHA1 }}
+            - ci-relevant-v2-{{ .Environment.CIRCLE_SHA1 }}
       - run:
-          name: Skip job if only non-esmodule paths changed
+          name: Skip job if only non-<< parameters.flag >> paths changed
           command: |
-            FLAG="$(cat ~/project/.ci-esmodule-relevant 2>/dev/null)" || FLAG=''
+            FLAG="$(cat ~/project/.ci-<< parameters.flag >>-relevant 2>/dev/null)" || FLAG=''
             if [ "${FLAG}" = "false" ]; then
-              echo "No relevant esmodule-related changes; halting this job."
+              echo "No << parameters.flag >>-relevant changes; halting this job."
               circleci-agent step halt
             elif [ "${FLAG}" = "true" ]; then
               echo "Relevant changes detected; continuing job."
@@ -48,16 +52,20 @@ jobs:
     steps:
       - checkout
       - run:
-          name: Detect esmodule-relevant changes
+          name: Detect relevant changes
           command: |
-            # Compare against the default branch (or its previous commit).
+            # Compare against the default branch.
             # Use CIRCLE_DEFAULT_BRANCH so this stays correct if the default branch changes.
             BASE_BRANCH="${CIRCLE_DEFAULT_BRANCH:-master}"
-            # Fail open: run the esmodule jobs unless we can prove irrelevance.
+            # Fail open: run jobs unless we can prove irrelevance.
             ESMODULE_RELEVANT=true
+            TESTS_RELEVANT=true
 
             if [ "${CIRCLE_BRANCH}" = "${BASE_BRANCH}" ]; then
-              DIFF_RANGE="HEAD~1...HEAD"
+              # A push can land several commits (rebase-merge), so HEAD~1 isn't
+              # the whole change; always run everything on the default branch.
+              echo "On ${BASE_BRANCH}; running all jobs."
+              DIFF_RANGE=""
             else
               # NOTE: never fetch with --depth here: a shallow fetch severs the
               # merge base, making the three-dot diff fail (and thus previously
@@ -68,25 +76,47 @@ jobs:
               DIFF_RANGE="origin/${BASE_BRANCH}...HEAD"
             fi
 
-            if CHANGED_FILES="$(git diff --name-only "${DIFF_RANGE}")"; then
+            # Grep a file, not a pipe: CircleCI runs bash with pipefail, so an
+            # early-exiting `grep -q` could SIGPIPE the writer and flip a check.
+            # (No here-strings either: CircleCI reads their angle brackets as parameter syntax.)
+            # --no-renames: a move out of packages/ must list the old path too.
+            if [ -z "${DIFF_RANGE}" ]; then
+              :
+            elif git -c core.quotePath=off diff --no-renames --name-only "${DIFF_RANGE}" > /tmp/ci-changed-files; then
               echo "Changed files:"
-              echo "${CHANGED_FILES}"
-              if echo "${CHANGED_FILES}" | grep -Eq \
-                '^(\.circleci/config\.yml|yarn\.lock|examples/todo-app/|examples/github-app/|examples/normalizr-relationships/|packages/)'; then
-                echo "Relevant changes detected."
-              else
+              cat /tmp/ci-changed-files
+              if ! grep -Eq \
+                '^(\.circleci/config\.yml|package\.json$|tsconfig[^/]*\.json$|babel\.config\.js$|\.yarnrc\.yml$|scripts/|yarn\.lock|examples/todo-app/|examples/github-app/|examples/normalizr-relationships/|packages/)' /tmp/ci-changed-files; then
                 echo "No relevant esmodule-related changes; downstream esmodule jobs will halt."
                 ESMODULE_RELEVANT=false
               fi
+              # Docs/website/tooling-only changes can't affect lint, typecheck or
+              # unit tests (Playground has unit tests, so it stays relevant).
+              if ! grep -Evq '^(website/|docs/|\.changeset/|\.cursor/|\.agents/|\.claude/|\.github/|[^/]+\.md$)' /tmp/ci-changed-files \
+                && ! grep -q '^website/src/components/Playground/' /tmp/ci-changed-files; then
+                echo "Only docs/website/tooling paths changed; test jobs will halt."
+                TESTS_RELEVANT=false
+              fi
             else
               echo "Could not diff ${DIFF_RANGE}; treating as relevant."
             fi
 
             echo "${ESMODULE_RELEVANT}" > ~/project/.ci-esmodule-relevant
+            echo "${TESTS_RELEVANT}" > ~/project/.ci-tests-relevant
+            # Later setup steps read these; the files are for downstream jobs.
+            echo "export ESMODULE_RELEVANT=${ESMODULE_RELEVANT} TESTS_RELEVANT=${TESTS_RELEVANT}" >> "${BASH_ENV}"
       - save_cache:
-          key: esmodule-relevant-v1-{{ .Environment.CIRCLE_SHA1 }}
+          key: ci-relevant-v2-{{ .Environment.CIRCLE_SHA1 }}
           paths:
             - .ci-esmodule-relevant
+            - .ci-tests-relevant
+      - run:
+          name: Halt setup if nothing downstream is relevant
+          command: |
+            if [ "${TESTS_RELEVANT}" = "false" ] && [ "${ESMODULE_RELEVANT}" = "false" ]; then
+              echo "Every downstream job will halt; skipping install and build."
+              circleci-agent step halt
+            fi
       - run:
           name: Add examples/* to yarn workspace
           command: |
@@ -115,9 +145,16 @@ jobs:
           key: v14-dependencies-{{ checksum "yarn.lock" }}-{{ checksum "examples/github-app/package.json" }}-{{ checksum "examples/todo-app/package.json" }}
       - run:
           # These are independent (babel/rollup compile from src, not tsc output),
-          # so run them concurrently to shorten the critical path.
+          # so run them concurrently to shorten the critical path. Legacy types
+          # (for the esmodule-types TS matrix) are built here rather than in a
+          # separate job to save a job hop on the critical path.
           name: Build types and test lib (parallel)
-          command: yarn run ci:build:setup
+          command: |
+            if [ "${ESMODULE_RELEVANT}" = "false" ]; then
+              yarn run ci:build:setup
+            else
+              yarn run ci:build:setup:esmodule
+            fi
       - persist_to_workspace:
           root: ~/
           paths:
@@ -146,6 +183,8 @@ jobs:
     executor: node
     resource_class: medium
     steps:
+      - halt-unless-relevant-change:
+          flag: tests
       - attach_workspace:
           at: ~/
       - run:
@@ -155,6 +194,8 @@ jobs:
   typecheck:
     executor: node
     steps:
+      - halt-unless-relevant-change:
+          flag: tests
       - attach_workspace:
           at: ~/
       - run:
@@ -168,6 +209,8 @@ jobs:
     executor: node
     resource_class: large
     steps:
+      - halt-unless-relevant-change:
+          flag: tests
       - attach_workspace:
           at: ~/
       - run:
@@ -221,6 +264,8 @@ jobs:
     docker:
       - image: cimg/node:<< parameters.node-version >>
     steps:
+      - halt-unless-relevant-change:
+          flag: tests
       - attach_workspace:
           at: ~/
       - run:
@@ -238,29 +283,14 @@ jobs:
           command: |
             ANANSI_JEST_TYPECHECK=false yarn test --ci --maxWorkers=2 --selectProjects Node
 
-  setup-esmodule-types:
-    executor: node
-    resource_class: large
-    steps:
-      - halt-unless-esmodule-relevant-change
-      - attach_workspace:
-          at: ~/
-      - run:
-          name: Build Legacy Types
-          command: yarn run ci:build:legacy-types
-      - persist_to_workspace:
-          root: ~/
-          paths:
-            # explicitly list so we can ignore some directories that are not needed
-            - project/packages/*/ts*
-
   esmodule-types:
     parameters:
       typescript-version:
         type: string
     executor: node
     steps:
-      - halt-unless-esmodule-relevant-change
+      - halt-unless-relevant-change:
+          flag: esmodule
       - attach_workspace:
           at: ~/
       - run:
@@ -300,7 +330,8 @@ jobs:
   esmodule-types-latest:
     executor: node
     steps:
-      - halt-unless-esmodule-relevant-change
+      - halt-unless-relevant-change:
+          flag: esmodule
       - attach_workspace:
           at: ~/
       - run:
@@ -323,7 +354,8 @@ jobs:
   validate-esmodule-browser-build:
     executor: node
     steps:
-      - halt-unless-esmodule-relevant-change
+      - halt-unless-relevant-change:
+          flag: esmodule
       - attach_workspace:
           at: ~/
       - run: yarn run ci:build:esmodule
@@ -365,9 +397,6 @@ workflows:
       - esmodule-types-latest:
           requires:
             - setup
-      - setup-esmodule-types:
-          requires:
-            - setup
       - esmodule-types:
           matrix:
             parameters:
@@ -378,4 +407,4 @@ workflows:
               # 4.7 (but its broken so we do 4.8) lets you apply a generic type to a function type to see its return value
               typescript-version: ["4.0", "4.1", "4.3", "4.8", "5.3"]
           requires:
-            - setup-esmodule-types
+            - setup

.cursor/rules/ci-config.mdc

@@ -9,7 +9,10 @@ alwaysApply: false
 ## CircleCI (`.circleci/config.yml`)
 
 - Jest `--maxWorkers` is pinned per job to the `resource_class` vCPU count (large = 4, medium = 2) because docker containers report the host's CPUs via `os.cpus()`. Exception: the ReactNative `unit_tests` run is deliberately uncapped — its suites are fake-timer-wait dominated and capping workers flakes 5s test timeouts.
-- The esmodule jobs halt based on a `.ci-esmodule-relevant` flag computed once in `setup` and transported via `save_cache`/`restore_cache` (keyed on `CIRCLE_SHA1`) so jobs can halt before paying `attach_workspace`. The flag's path regex in `setup` must cover every path the esmodule jobs build (currently `examples/todo-app`, `examples/github-app`, `examples/normalizr-relationships`, `packages/`, `yarn.lock`, the config itself). Missing/unreadable flag fails open (jobs run).
+- Jobs halt via the `halt-unless-relevant-change` command based on flags computed once in `setup` (`.ci-esmodule-relevant`, `.ci-tests-relevant`) and transported via `save_cache`/`restore_cache` (keyed on `CIRCLE_SHA1`) so jobs can halt before paying `attach_workspace`. Missing/unreadable flags fail open (jobs run). On the default branch both flags are always true (a push may carry several commits). The diff uses `--no-renames` so moving a file out of a relevant dir still counts.
+  - `esmodule`: the path regex must cover every path the esmodule jobs build (currently `examples/todo-app`, `examples/github-app`, `examples/normalizr-relationships`, `packages/`, root `package.json`, root `tsconfig*.json`, `babel.config.js`, `.yarnrc.yml`, `scripts/`, `yarn.lock`, the config itself).
+  - `tests` (lint, typecheck, unit_tests, node_matrix): false only when every changed path is docs/website/tooling (`website/`, `docs/`, `.changeset/`, `.cursor/`, `.agents/`, `.claude/`, `.github/`, root `*.md`), except `website/src/components/Playground/` (has unit tests). When both flags are false, `setup` halts before install.
+- Legacy TS types (`ci:build:legacy-types`, consumed by `esmodule-types`) build inside `setup` (`ci:build:setup:esmodule`) only when the esmodule flag is set; there is no separate job, to keep a job hop off the critical path. In CI it builds only endpoint/normalizr/rest outputs for TS >= 4.0 (`LEGACY_MIN_TS=4.0`), since the oldest TS in the `esmodule-types` matrix is 4.0; release builds (`build:types`) still emit every version. `scripts/build-legacy-types.sh` builds each TS version concurrently.
 - Never `git fetch --depth` the base branch in the relevance check: a shallow fetch severs the merge base and the three-dot diff fails.
 - Changing root `package.json` `workspaces` requires updating the `setup` job's workspace trimming step.
 

package.json

@@ -24,8 +24,10 @@
     "build:types": "yarn build:copy:ambient && tsc --build --builders 1 && yarn workspaces foreach -Wpti --no-private run build:legacy-types",
     "ci:build": "yarn ci:build:types && yarn workspaces foreach -Wptiv --no-private run build:lib",
     "ci:build:setup": "run-p ci:build:types ci:build-test-lib",
+    "ci:build:setup:esmodule": "run-p ci:build:types-and-legacy ci:build-test-lib",
+    "ci:build:types-and-legacy": "yarn ci:build:types && yarn ci:build:legacy-types",
     "ci:build:types": "yarn build:copy:ambient && tsc --build --builders 1",
-    "ci:build:legacy-types": "yarn workspaces foreach -WptivR -j 10 --from @data-client/react --from @data-client/rest --from @data-client/graphql run build:legacy-types",
+    "ci:build:legacy-types": "LEGACY_MIN_TS=4.0 yarn workspaces foreach -Wpiv --include @data-client/endpoint --include @data-client/normalizr --include @data-client/rest run build:legacy-types",
     "ci:build-test-lib": "yarn workspace @data-client/core run build:lib && yarn workspace @data-client/test run build:lib && yarn workspace @data-client/test run build:bundle",
     "ci:build:esmodule": "yarn workspaces foreach -WptivR --from @data-client/react --from @data-client/rest --from @data-client/graphql run build:lib && yarn workspace @data-client/normalizr run build:js:node && yarn workspace @data-client/endpoint run build:js:node",
     "ci:build:bundlesize": "yarn workspaces foreach -Wptiv --no-private run build:lib && yarn workspace test-bundlesize run build:sizecompare",
@@ -47,6 +49,7 @@
     "g:tsc": "cd $INIT_CWD && tsc",
     "g:legacy-types": "cd $INIT_CWD && ../../scripts/build-legacy-types.sh",
     "g:runs": "cd $INIT_CWD && run-s",
+    "g:runp": "cd $INIT_CWD && run-p",
     "g:copy": "cd $INIT_CWD && copyfiles",
     "g:lint": "cd $INIT_CWD && eslint"
   },

packages/rest/package.json

@@ -123,7 +123,9 @@
     "build:js:browser": "BROWSERSLIST_ENV=legacy yarn g:rollup",
     "build:bundle": "yarn g:runs build:js:\\* && echo '{\"type\":\"commonjs\"}' > dist/package.json",
     "build:clean": "yarn g:clean ts4.0 ts4.1 index.d.ts next.d.ts",
-    "build:legacy-types": "yarn g:downtypes lib ts4.0 --to=4.0 && yarn g:downtypes lib ts4.1 --to=4.1 && yarn g:copy --up 1 ./src-4.1-types/**/*.d.ts ./ts4.0/ && yarn g:copy --up 1 ./src-4.1-types/**/*.d.ts ./ts4.1 && yarn g:copy --up 1 ./src-4.0-types/**/*.d.ts ./ts4.0",
+    "build:legacy-types": "yarn g:runp build:legacy-types:4.0 build:legacy-types:4.1",
+    "build:legacy-types:4.0": "yarn g:downtypes lib ts4.0 --to=4.0 && yarn g:copy --up 1 ./src-4.1-types/**/*.d.ts ./ts4.0/ && yarn g:copy --up 1 ./src-4.0-types/**/*.d.ts ./ts4.0",
+    "build:legacy-types:4.1": "yarn g:downtypes lib ts4.1 --to=4.1 && yarn g:copy --up 1 ./src-4.1-types/**/*.d.ts ./ts4.1",
     "build": "run build:lib && run build:legacy:lib && run build:bundle",
     "dev": "NODE_ENV=production BROWSERSLIST_ENV='2020' POLYFILL_TARGETS='chrome>88,safari>14' yarn g:babel --out-dir lib -w",
     "prepare": "run build:lib",

scripts/build-legacy-types.sh

@@ -1,27 +1,61 @@
 #!/bin/bash
+set -e
 
 # Check if at least one directory is provided
 if [ $# -eq 0 ]; then
     echo "Usage: $0 <version1> [version2] ..."
     exit 1
 fi
 
-destinations=("$@")
-# Loop through all provided arguments
+# Called directly (not via `yarn g:*`) to skip a yarn boot per step.
+downlevel_dts="$(dirname "$0")/../node_modules/.bin/downlevel-dts"
+
+# Copies only the .d.ts files under src dir $1 into $2 (ts* dirs are published).
+copy_types() {
+    [ -d "$1" ] || return 0
+    (cd "$1" && find . -name '*.d.ts') | while IFS= read -r file
+    do
+        mkdir -p "$2/$(dirname "$file")"
+        cp "$1/$file" "$2/$file"
+    done
+}
+
+# Custom types for a version also apply to every version listed after it,
+# so each output dir gets (in order): earlier versions' custom types, the
+# downleveled lib, then its own custom types. Each output dir only depends on
+# lib and src-*-types, so versions build concurrently.
+build_version() {
+    local version="$1"
+    shift
+    mkdir -p "./ts$version"
+    for earlier in "$@"
+    do
+        copy_types "./src-$earlier-types" "./ts$version"
+    done
+    "$downlevel_dts" lib "ts$version" --to="$version"
+    copy_types "./src-$version-types" "./ts$version"
+}
+
+# LEGACY_MIN_TS skips outputs no consumer reads (CI's oldest tested TS).
+below_min() {
+    [ -n "$LEGACY_MIN_TS" ] && [ "$1" != "$LEGACY_MIN_TS" ] \
+        && [ "$(printf '%s\n' "$1" "$LEGACY_MIN_TS" | sort -V | head -1)" = "$1" ]
+}
+
+pids=()
+earlier=()
 for version in "$@"
 do
-   yarn g:downtypes lib "ts$version" --to="$version"
-    # Check if custom type file directory exists
-    if [ -d "./src-$version-types" ]; then
-        for dest in "${destinations[@]}"
-        do
-            yarn g:copy --up 1 "./src-$version-types/**/*.d.ts" "./ts$dest/"
-            echo "Copied ./src-$version-types to ./ts$dest/"
-        done
-    else
-        echo "Custom types for $version not found."
+    if ! below_min "$version"; then
+        build_version "$version" "${earlier[@]}" &
+        pids+=($!)
     fi
-    # this is how you pop the first element off
-    unset destinations[0]
-    destinations=("${destinations[@]}")
-done
+    earlier+=("$version")
+done
+
+status=0
+for pid in "${pids[@]}"
+do
+    wait "$pid" || status=1
+done
+exit $status