diff --git a/.circleci/config.yml b/.circleci/config.yml index 165e9d3ea814..06221312b2f3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,19 +21,23 @@ executors: YARN_ENABLE_IMMUTABLE_INSTALLS: true commands: - halt-unless-esmodule-relevant-change: - # Reads the flag computed once in the `setup` job. Transported via cache + halt-unless-relevant-change: + # Reads a flag computed once in the `setup` job. Transported via cache # (not the workspace) so jobs can halt before paying attach_workspace. + parameters: + flag: + type: enum + enum: ["esmodule", "tests"] steps: - restore_cache: keys: - - esmodule-relevant-v1-{{ .Environment.CIRCLE_SHA1 }} + - ci-relevant-v2-{{ .Environment.CIRCLE_SHA1 }} - run: - name: Skip job if only non-esmodule paths changed + name: Skip job if only non-<< parameters.flag >> paths changed command: | - FLAG="$(cat ~/project/.ci-esmodule-relevant 2>/dev/null)" || FLAG='' + FLAG="$(cat ~/project/.ci-<< parameters.flag >>-relevant 2>/dev/null)" || FLAG='' if [ "${FLAG}" = "false" ]; then - echo "No relevant esmodule-related changes; halting this job." + echo "No << parameters.flag >>-relevant changes; halting this job." circleci-agent step halt elif [ "${FLAG}" = "true" ]; then echo "Relevant changes detected; continuing job." @@ -48,16 +52,20 @@ jobs: steps: - checkout - run: - name: Detect esmodule-relevant changes + name: Detect relevant changes command: | - # Compare against the default branch (or its previous commit). + # Compare against the default branch. # Use CIRCLE_DEFAULT_BRANCH so this stays correct if the default branch changes. BASE_BRANCH="${CIRCLE_DEFAULT_BRANCH:-master}" - # Fail open: run the esmodule jobs unless we can prove irrelevance. + # Fail open: run jobs unless we can prove irrelevance. ESMODULE_RELEVANT=true + TESTS_RELEVANT=true if [ "${CIRCLE_BRANCH}" = "${BASE_BRANCH}" ]; then - DIFF_RANGE="HEAD~1...HEAD" + # A push can land several commits (rebase-merge), so HEAD~1 isn't + # the whole change; always run everything on the default branch. + echo "On ${BASE_BRANCH}; running all jobs." + DIFF_RANGE="" else # NOTE: never fetch with --depth here: a shallow fetch severs the # merge base, making the three-dot diff fail (and thus previously @@ -68,25 +76,47 @@ jobs: DIFF_RANGE="origin/${BASE_BRANCH}...HEAD" fi - if CHANGED_FILES="$(git diff --name-only "${DIFF_RANGE}")"; then + # Grep a file, not a pipe: CircleCI runs bash with pipefail, so an + # early-exiting `grep -q` could SIGPIPE the writer and flip a check. + # (No here-strings either: CircleCI reads their angle brackets as parameter syntax.) + # --no-renames: a move out of packages/ must list the old path too. + if [ -z "${DIFF_RANGE}" ]; then + : + elif git -c core.quotePath=off diff --no-renames --name-only "${DIFF_RANGE}" > /tmp/ci-changed-files; then echo "Changed files:" - echo "${CHANGED_FILES}" - if echo "${CHANGED_FILES}" | grep -Eq \ - '^(\.circleci/config\.yml|yarn\.lock|examples/todo-app/|examples/github-app/|examples/normalizr-relationships/|packages/)'; then - echo "Relevant changes detected." - else + cat /tmp/ci-changed-files + if ! grep -Eq \ + '^(\.circleci/config\.yml|package\.json$|tsconfig[^/]*\.json$|babel\.config\.js$|\.yarnrc\.yml$|scripts/|yarn\.lock|examples/todo-app/|examples/github-app/|examples/normalizr-relationships/|packages/)' /tmp/ci-changed-files; then echo "No relevant esmodule-related changes; downstream esmodule jobs will halt." ESMODULE_RELEVANT=false fi + # Docs/website/tooling-only changes can't affect lint, typecheck or + # unit tests (Playground has unit tests, so it stays relevant). + if ! grep -Evq '^(website/|docs/|\.changeset/|\.cursor/|\.agents/|\.claude/|\.github/|[^/]+\.md$)' /tmp/ci-changed-files \ + && ! grep -q '^website/src/components/Playground/' /tmp/ci-changed-files; then + echo "Only docs/website/tooling paths changed; test jobs will halt." + TESTS_RELEVANT=false + fi else echo "Could not diff ${DIFF_RANGE}; treating as relevant." fi echo "${ESMODULE_RELEVANT}" > ~/project/.ci-esmodule-relevant + echo "${TESTS_RELEVANT}" > ~/project/.ci-tests-relevant + # Later setup steps read these; the files are for downstream jobs. + echo "export ESMODULE_RELEVANT=${ESMODULE_RELEVANT} TESTS_RELEVANT=${TESTS_RELEVANT}" >> "${BASH_ENV}" - save_cache: - key: esmodule-relevant-v1-{{ .Environment.CIRCLE_SHA1 }} + key: ci-relevant-v2-{{ .Environment.CIRCLE_SHA1 }} paths: - .ci-esmodule-relevant + - .ci-tests-relevant + - run: + name: Halt setup if nothing downstream is relevant + command: | + if [ "${TESTS_RELEVANT}" = "false" ] && [ "${ESMODULE_RELEVANT}" = "false" ]; then + echo "Every downstream job will halt; skipping install and build." + circleci-agent step halt + fi - run: name: Add examples/* to yarn workspace command: | @@ -115,9 +145,16 @@ jobs: key: v14-dependencies-{{ checksum "yarn.lock" }}-{{ checksum "examples/github-app/package.json" }}-{{ checksum "examples/todo-app/package.json" }} - run: # These are independent (babel/rollup compile from src, not tsc output), - # so run them concurrently to shorten the critical path. + # so run them concurrently to shorten the critical path. Legacy types + # (for the esmodule-types TS matrix) are built here rather than in a + # separate job to save a job hop on the critical path. name: Build types and test lib (parallel) - command: yarn run ci:build:setup + command: | + if [ "${ESMODULE_RELEVANT}" = "false" ]; then + yarn run ci:build:setup + else + yarn run ci:build:setup:esmodule + fi - persist_to_workspace: root: ~/ paths: @@ -146,6 +183,8 @@ jobs: executor: node resource_class: medium steps: + - halt-unless-relevant-change: + flag: tests - attach_workspace: at: ~/ - run: @@ -155,6 +194,8 @@ jobs: typecheck: executor: node steps: + - halt-unless-relevant-change: + flag: tests - attach_workspace: at: ~/ - run: @@ -168,6 +209,8 @@ jobs: executor: node resource_class: large steps: + - halt-unless-relevant-change: + flag: tests - attach_workspace: at: ~/ - run: @@ -221,6 +264,8 @@ jobs: docker: - image: cimg/node:<< parameters.node-version >> steps: + - halt-unless-relevant-change: + flag: tests - attach_workspace: at: ~/ - run: @@ -238,29 +283,14 @@ jobs: command: | ANANSI_JEST_TYPECHECK=false yarn test --ci --maxWorkers=2 --selectProjects Node - setup-esmodule-types: - executor: node - resource_class: large - steps: - - halt-unless-esmodule-relevant-change - - attach_workspace: - at: ~/ - - run: - name: Build Legacy Types - command: yarn run ci:build:legacy-types - - persist_to_workspace: - root: ~/ - paths: - # explicitly list so we can ignore some directories that are not needed - - project/packages/*/ts* - esmodule-types: parameters: typescript-version: type: string executor: node steps: - - halt-unless-esmodule-relevant-change + - halt-unless-relevant-change: + flag: esmodule - attach_workspace: at: ~/ - run: @@ -300,7 +330,8 @@ jobs: esmodule-types-latest: executor: node steps: - - halt-unless-esmodule-relevant-change + - halt-unless-relevant-change: + flag: esmodule - attach_workspace: at: ~/ - run: @@ -323,7 +354,8 @@ jobs: validate-esmodule-browser-build: executor: node steps: - - halt-unless-esmodule-relevant-change + - halt-unless-relevant-change: + flag: esmodule - attach_workspace: at: ~/ - run: yarn run ci:build:esmodule @@ -365,9 +397,6 @@ workflows: - esmodule-types-latest: requires: - setup - - setup-esmodule-types: - requires: - - setup - esmodule-types: matrix: parameters: @@ -378,4 +407,4 @@ workflows: # 4.7 (but its broken so we do 4.8) lets you apply a generic type to a function type to see its return value typescript-version: ["4.0", "4.1", "4.3", "4.8", "5.3"] requires: - - setup-esmodule-types + - setup diff --git a/.cursor/rules/ci-config.mdc b/.cursor/rules/ci-config.mdc index 12ab9e50a57f..a67967962284 100644 --- a/.cursor/rules/ci-config.mdc +++ b/.cursor/rules/ci-config.mdc @@ -9,7 +9,10 @@ alwaysApply: false ## CircleCI (`.circleci/config.yml`) - Jest `--maxWorkers` is pinned per job to the `resource_class` vCPU count (large = 4, medium = 2) because docker containers report the host's CPUs via `os.cpus()`. Exception: the ReactNative `unit_tests` run is deliberately uncapped — its suites are fake-timer-wait dominated and capping workers flakes 5s test timeouts. -- The esmodule jobs halt based on a `.ci-esmodule-relevant` flag computed once in `setup` and transported via `save_cache`/`restore_cache` (keyed on `CIRCLE_SHA1`) so jobs can halt before paying `attach_workspace`. The flag's path regex in `setup` must cover every path the esmodule jobs build (currently `examples/todo-app`, `examples/github-app`, `examples/normalizr-relationships`, `packages/`, `yarn.lock`, the config itself). Missing/unreadable flag fails open (jobs run). +- Jobs halt via the `halt-unless-relevant-change` command based on flags computed once in `setup` (`.ci-esmodule-relevant`, `.ci-tests-relevant`) and transported via `save_cache`/`restore_cache` (keyed on `CIRCLE_SHA1`) so jobs can halt before paying `attach_workspace`. Missing/unreadable flags fail open (jobs run). On the default branch both flags are always true (a push may carry several commits). The diff uses `--no-renames` so moving a file out of a relevant dir still counts. + - `esmodule`: the path regex must cover every path the esmodule jobs build (currently `examples/todo-app`, `examples/github-app`, `examples/normalizr-relationships`, `packages/`, root `package.json`, root `tsconfig*.json`, `babel.config.js`, `.yarnrc.yml`, `scripts/`, `yarn.lock`, the config itself). + - `tests` (lint, typecheck, unit_tests, node_matrix): false only when every changed path is docs/website/tooling (`website/`, `docs/`, `.changeset/`, `.cursor/`, `.agents/`, `.claude/`, `.github/`, root `*.md`), except `website/src/components/Playground/` (has unit tests). When both flags are false, `setup` halts before install. +- Legacy TS types (`ci:build:legacy-types`, consumed by `esmodule-types`) build inside `setup` (`ci:build:setup:esmodule`) only when the esmodule flag is set; there is no separate job, to keep a job hop off the critical path. In CI it builds only endpoint/normalizr/rest outputs for TS >= 4.0 (`LEGACY_MIN_TS=4.0`), since the oldest TS in the `esmodule-types` matrix is 4.0; release builds (`build:types`) still emit every version. `scripts/build-legacy-types.sh` builds each TS version concurrently. - Never `git fetch --depth` the base branch in the relevance check: a shallow fetch severs the merge base and the three-dot diff fails. - Changing root `package.json` `workspaces` requires updating the `setup` job's workspace trimming step. diff --git a/package.json b/package.json index 3d8a0bf65ebc..9fd4a1e05f08 100644 --- a/package.json +++ b/package.json @@ -24,8 +24,10 @@ "build:types": "yarn build:copy:ambient && tsc --build --builders 1 && yarn workspaces foreach -Wpti --no-private run build:legacy-types", "ci:build": "yarn ci:build:types && yarn workspaces foreach -Wptiv --no-private run build:lib", "ci:build:setup": "run-p ci:build:types ci:build-test-lib", + "ci:build:setup:esmodule": "run-p ci:build:types-and-legacy ci:build-test-lib", + "ci:build:types-and-legacy": "yarn ci:build:types && yarn ci:build:legacy-types", "ci:build:types": "yarn build:copy:ambient && tsc --build --builders 1", - "ci:build:legacy-types": "yarn workspaces foreach -WptivR -j 10 --from @data-client/react --from @data-client/rest --from @data-client/graphql run build:legacy-types", + "ci:build:legacy-types": "LEGACY_MIN_TS=4.0 yarn workspaces foreach -Wpiv --include @data-client/endpoint --include @data-client/normalizr --include @data-client/rest run build:legacy-types", "ci:build-test-lib": "yarn workspace @data-client/core run build:lib && yarn workspace @data-client/test run build:lib && yarn workspace @data-client/test run build:bundle", "ci:build:esmodule": "yarn workspaces foreach -WptivR --from @data-client/react --from @data-client/rest --from @data-client/graphql run build:lib && yarn workspace @data-client/normalizr run build:js:node && yarn workspace @data-client/endpoint run build:js:node", "ci:build:bundlesize": "yarn workspaces foreach -Wptiv --no-private run build:lib && yarn workspace test-bundlesize run build:sizecompare", @@ -47,6 +49,7 @@ "g:tsc": "cd $INIT_CWD && tsc", "g:legacy-types": "cd $INIT_CWD && ../../scripts/build-legacy-types.sh", "g:runs": "cd $INIT_CWD && run-s", + "g:runp": "cd $INIT_CWD && run-p", "g:copy": "cd $INIT_CWD && copyfiles", "g:lint": "cd $INIT_CWD && eslint" }, diff --git a/packages/rest/package.json b/packages/rest/package.json index 8752b71d9fe9..2d66e7ee8eb6 100644 --- a/packages/rest/package.json +++ b/packages/rest/package.json @@ -123,7 +123,9 @@ "build:js:browser": "BROWSERSLIST_ENV=legacy yarn g:rollup", "build:bundle": "yarn g:runs build:js:\\* && echo '{\"type\":\"commonjs\"}' > dist/package.json", "build:clean": "yarn g:clean ts4.0 ts4.1 index.d.ts next.d.ts", - "build:legacy-types": "yarn g:downtypes lib ts4.0 --to=4.0 && yarn g:downtypes lib ts4.1 --to=4.1 && yarn g:copy --up 1 ./src-4.1-types/**/*.d.ts ./ts4.0/ && yarn g:copy --up 1 ./src-4.1-types/**/*.d.ts ./ts4.1 && yarn g:copy --up 1 ./src-4.0-types/**/*.d.ts ./ts4.0", + "build:legacy-types": "yarn g:runp build:legacy-types:4.0 build:legacy-types:4.1", + "build:legacy-types:4.0": "yarn g:downtypes lib ts4.0 --to=4.0 && yarn g:copy --up 1 ./src-4.1-types/**/*.d.ts ./ts4.0/ && yarn g:copy --up 1 ./src-4.0-types/**/*.d.ts ./ts4.0", + "build:legacy-types:4.1": "yarn g:downtypes lib ts4.1 --to=4.1 && yarn g:copy --up 1 ./src-4.1-types/**/*.d.ts ./ts4.1", "build": "run build:lib && run build:legacy:lib && run build:bundle", "dev": "NODE_ENV=production BROWSERSLIST_ENV='2020' POLYFILL_TARGETS='chrome>88,safari>14' yarn g:babel --out-dir lib -w", "prepare": "run build:lib", diff --git a/scripts/build-legacy-types.sh b/scripts/build-legacy-types.sh index 8ab44bf68e95..9e5be7dab35c 100755 --- a/scripts/build-legacy-types.sh +++ b/scripts/build-legacy-types.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -e # Check if at least one directory is provided if [ $# -eq 0 ]; then @@ -6,22 +7,55 @@ if [ $# -eq 0 ]; then exit 1 fi -destinations=("$@") -# Loop through all provided arguments +# Called directly (not via `yarn g:*`) to skip a yarn boot per step. +downlevel_dts="$(dirname "$0")/../node_modules/.bin/downlevel-dts" + +# Copies only the .d.ts files under src dir $1 into $2 (ts* dirs are published). +copy_types() { + [ -d "$1" ] || return 0 + (cd "$1" && find . -name '*.d.ts') | while IFS= read -r file + do + mkdir -p "$2/$(dirname "$file")" + cp "$1/$file" "$2/$file" + done +} + +# Custom types for a version also apply to every version listed after it, +# so each output dir gets (in order): earlier versions' custom types, the +# downleveled lib, then its own custom types. Each output dir only depends on +# lib and src-*-types, so versions build concurrently. +build_version() { + local version="$1" + shift + mkdir -p "./ts$version" + for earlier in "$@" + do + copy_types "./src-$earlier-types" "./ts$version" + done + "$downlevel_dts" lib "ts$version" --to="$version" + copy_types "./src-$version-types" "./ts$version" +} + +# LEGACY_MIN_TS skips outputs no consumer reads (CI's oldest tested TS). +below_min() { + [ -n "$LEGACY_MIN_TS" ] && [ "$1" != "$LEGACY_MIN_TS" ] \ + && [ "$(printf '%s\n' "$1" "$LEGACY_MIN_TS" | sort -V | head -1)" = "$1" ] +} + +pids=() +earlier=() for version in "$@" do - yarn g:downtypes lib "ts$version" --to="$version" - # Check if custom type file directory exists - if [ -d "./src-$version-types" ]; then - for dest in "${destinations[@]}" - do - yarn g:copy --up 1 "./src-$version-types/**/*.d.ts" "./ts$dest/" - echo "Copied ./src-$version-types to ./ts$dest/" - done - else - echo "Custom types for $version not found." + if ! below_min "$version"; then + build_version "$version" "${earlier[@]}" & + pids+=($!) fi - # this is how you pop the first element off - unset destinations[0] - destinations=("${destinations[@]}") -done \ No newline at end of file + earlier+=("$version") +done + +status=0 +for pid in "${pids[@]}" +do + wait "$pid" || status=1 +done +exit $status