diff --git a/.bazelrc b/.bazelrc index e51de6907..6fddeec89 100644 --- a/.bazelrc +++ b/.bazelrc @@ -47,3 +47,21 @@ try-import %workspace%/fuzztest.bazelrc # Configure rules_rust to use the nightly channel toolchain by default. build --@rules_rust//rust/toolchain/channel:channel=nightly + +# CI cache configuration used by GitHub Actions workflows (--config=ci). +# Store only external repository archives and compilation action outputs in +# ~/.cache/bazel-cache rather than caching ~/.cache/bazel (Bazel's +# output_user_root), which also contains the full execroot, bazel-out working +# trees, .runfiles symlink forests, and test logs. +common:ci --repository_cache=~/.cache/bazel-cache/repo +build:ci --disk_cache=~/.cache/bazel-cache/disk +# Exclude linked binaries (CppLink) from the disk cache: statically linked +# debug/ASan/FuzzTest test binaries are tens to hundreds of MBs each and change +# frequently, whereas caching compilation, archive (.o / .a), and test result +# (TestRunner) actions keeps the cache compact while skipping unaffected tests. +build:ci --modify_execution_info=CppLink=+no-cache +# Automatically garbage-collect disk cache entries older than 7 days or +# exceeding 2 GB immediately when the Bazel server becomes idle after a command. +build:ci --experimental_disk_cache_gc_max_size=2G +build:ci --experimental_disk_cache_gc_max_age=7d +build:ci --experimental_disk_cache_gc_idle_delay=0 diff --git a/.github/workflows/bazel_test.yml b/.github/workflows/bazel_test.yml index 1f580ac72..c8a98c933 100644 --- a/.github/workflows/bazel_test.yml +++ b/.github/workflows/bazel_test.yml @@ -45,20 +45,38 @@ jobs: sudo apt-get update && sudo apt-get install -yq \ clang - name: Restore latest cache + # Skip restoring on scheduled nightly runs so the cache is rebuilt from + # scratch and stale artifacts from older commits/dependencies are + # evicted. + if: github.event_name != 'schedule' + id: cache-restore uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: - path: "~/.cache/bazel" + # Cache only the Bazel repository and disk caches (configured via + # --config=ci in .bazelrc) instead of ~/.cache/bazel + # (output_user_root), which contains the full execroot and linked + # test binaries. + path: "~/.cache/bazel-cache" key: bazel-cache-unit-${{ matrix.compilation_mode }} restore-keys: bazel-cache-unit-${{ matrix.compilation_mode }}- + - name: Configure Bazel CI cache + run: | + echo "common --config=ci" >> ~/.bazelrc - name: Run all tests with default --config run: | bazel test --build_tests_only --test_output=errors \ -c ${{ matrix.compilation_mode }} -- //... -//e2e_tests/... -//centipede/... - name: Save new cache based on main - if: github.ref == 'refs/heads/main' + # Save a new cache on nightly scheduled runs (or manual dispatch on + # main), or on a push to main when no previous cache exists for this + # key prefix. This avoids uploading duplicate immutable cache archives + # on every commit pushed to main. + if: >- + github.ref == 'refs/heads/main' && + (github.event_name != 'push' || steps.cache-restore.outputs.cache-matched-key == '') uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: - path: "~/.cache/bazel" + path: "~/.cache/bazel-cache" key: bazel-cache-unit-${{ matrix.compilation_mode }}-${{ github.run_id }} run_e2e_tests: name: Run e2e tests @@ -79,15 +97,23 @@ jobs: run: | sudo apt-get update && sudo apt-get install -yq \ clang - - name: Setup build configs for Bazel - run: | - EXTRA_CONFIGS=libfuzzer bazel run //bazel:setup_configs > fuzztest.bazelrc - name: Restore latest cache + # Skip restoring on scheduled nightly runs so the cache is rebuilt from + # scratch and stale artifacts from older commits/dependencies are + # evicted. + if: github.event_name != 'schedule' + id: cache-restore uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: - path: "~/.cache/bazel" + path: "~/.cache/bazel-cache" key: bazel-cache-e2e-${{ matrix.config }} restore-keys: bazel-cache-e2e-${{ matrix.config }}- + - name: Configure Bazel CI cache + run: | + echo "common --config=ci" >> ~/.bazelrc + - name: Setup build configs for Bazel + run: | + EXTRA_CONFIGS=libfuzzer bazel run //bazel:setup_configs > fuzztest.bazelrc - name: Run end-to-end tests without instrumentation if: matrix.config == 'uninstrumented' # Exclude corpus_database_test because it needs instrumentation. @@ -123,8 +149,14 @@ jobs: --platform_suffix=fuzztest-libfuzzer \ //e2e_tests:compatibility_mode_test - name: Save new cache based on main - if: github.ref == 'refs/heads/main' + # Save a new cache on nightly scheduled runs (or manual dispatch on + # main), or on a push to main when no previous cache exists for this + # key prefix. This avoids uploading duplicate immutable cache archives + # on every commit pushed to main. + if: >- + github.ref == 'refs/heads/main' && + (github.event_name != 'push' || steps.cache-restore.outputs.cache-matched-key == '') uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: - path: "~/.cache/bazel" + path: "~/.cache/bazel-cache" key: bazel-cache-e2e-${{ matrix.config }}-${{ github.run_id }} diff --git a/.github/workflows/bazel_test_centipede.yml b/.github/workflows/bazel_test_centipede.yml index 8e31fc09d..c77e6daf3 100644 --- a/.github/workflows/bazel_test_centipede.yml +++ b/.github/workflows/bazel_test_centipede.yml @@ -48,11 +48,23 @@ jobs: sudo apt-get update && sudo apt-get install -yq \ clang llvm libssl-dev - name: Restore latest cache + # Skip restoring on scheduled nightly runs so the cache is rebuilt from + # scratch and stale artifacts from older commits/dependencies are + # evicted. + if: github.event_name != 'schedule' + id: cache-restore uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: - path: "~/.cache/bazel" + # Cache only the Bazel repository and disk caches (configured via + # --config=ci in .bazelrc) instead of ~/.cache/bazel + # (output_user_root), which contains the full execroot and linked + # test binaries. + path: "~/.cache/bazel-cache" key: bazel-centipede-cache-${{ matrix.config }} restore-keys: bazel-centipede-cache-${{ matrix.config }}- + - name: Configure Bazel CI cache + run: | + echo "common --config=ci" >> ~/.bazelrc - name: Run unit tests if: matrix.config == 'default' run: | @@ -86,10 +98,16 @@ jobs: run: | bazel test --no//fuzztest:use_riegeli --test_output=errors --linkopt=-fsanitize=address --copt=-fsanitize=address --test_env=ASAN_OPTIONS=detect_leaks=0 --platform_suffix=asan --test_timeout=600 centipede/puzzles:all - name: Save new cache based on main - if: github.ref == 'refs/heads/main' + # Save a new cache on nightly scheduled runs (or manual dispatch on + # main), or on a push to main when no previous cache exists for this + # key prefix. This avoids uploading duplicate immutable cache archives + # on every commit pushed to main. + if: >- + github.ref == 'refs/heads/main' && + (github.event_name != 'push' || steps.cache-restore.outputs.cache-matched-key == '') uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: - path: "~/.cache/bazel" + path: "~/.cache/bazel-cache" key: bazel-centipede-cache-${{ matrix.config }}-${{ github.run_id }} run_tests_mac: name: Run Centipede tests (MacOS) @@ -101,11 +119,19 @@ jobs: - name: Checkout repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Restore latest cache + # Skip restoring on scheduled nightly runs so the cache is rebuilt from + # scratch and stale artifacts from older commits/dependencies are + # evicted. + if: github.event_name != 'schedule' + id: cache-restore uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: - path: "~/.cache/bazel" + path: "~/.cache/bazel-cache" key: bazel-centipede-cache-mac restore-keys: bazel-centipede-cache-mac- + - name: Configure Bazel CI cache + run: | + echo "common --config=ci" >> ~/.bazelrc - name: Add LLVM symbolizer to path run: | ln -s $(brew --prefix llvm@18)/bin/llvm-symbolizer /usr/local/bin @@ -114,25 +140,31 @@ jobs: run: | # Timed wait and rusage is inaccurate for unknown reasons in MacOS for a few testing # environments, thus disabling a few related non-critical tests. - bazel --output_user_root="${HOME}/.cache/bazel" test --local_test_jobs=1 --test_output=errors --no//fuzztest:use_riegeli -- centipede:all \ + bazel test --local_test_jobs=1 --test_output=errors --no//fuzztest:use_riegeli -- centipede:all \ -centipede:periodic_action_test -centipede:rusage_profiler_test -centipede:rusage_stats_test && \ - bazel --output_user_root="${HOME}/.cache/bazel" test --local_test_jobs=1 --test_output=errors --no//fuzztest:use_riegeli --test_filter='-*ValidateTimelapseSnapshots' -- centipede:rusage_profiler_test && \ - bazel --output_user_root="${HOME}/.cache/bazel" test --local_test_jobs=1 --test_output=errors --no//fuzztest:use_riegeli --test_filter='-RUsageMemoryTest.Accuracy' -- centipede:rusage_stats_test + bazel test --local_test_jobs=1 --test_output=errors --no//fuzztest:use_riegeli --test_filter='-*ValidateTimelapseSnapshots' -- centipede:rusage_profiler_test && \ + bazel test --local_test_jobs=1 --test_output=errors --no//fuzztest:use_riegeli --test_filter='-RUsageMemoryTest.Accuracy' -- centipede:rusage_stats_test - name: Run e2e tests without Riegeli if: ${{ !cancelled() }} run: | - bazel --output_user_root="${HOME}/.cache/bazel" test --test_output=errors --no//fuzztest:use_riegeli centipede/testing:instrumentation_test centipede/testing:runner_test centipede/testing:multi_dso_test + bazel test --test_output=errors --no//fuzztest:use_riegeli centipede/testing:instrumentation_test centipede/testing:runner_test centipede/testing:multi_dso_test - name: Run puzzles without Riegeli if: ${{ !cancelled() }} run: | - bazel --output_user_root="${HOME}/.cache/bazel" test --test_output=errors --no//fuzztest:use_riegeli --test_timeout=600 centipede/puzzles:all + bazel test --test_output=errors --no//fuzztest:use_riegeli --test_timeout=600 centipede/puzzles:all - name: Run puzzles without Riegeli with ASAN if: ${{ !cancelled() }} run: | - bazel --output_user_root="${HOME}/.cache/bazel" test --test_output=errors --no//fuzztest:use_riegeli --linkopt=-fsanitize=address --copt=-fsanitize=address --test_env=ASAN_OPTIONS=detect_leaks=0 --platform_suffix=asan --test_timeout=600 centipede/puzzles:all + bazel test --test_output=errors --no//fuzztest:use_riegeli --linkopt=-fsanitize=address --copt=-fsanitize=address --test_env=ASAN_OPTIONS=detect_leaks=0 --platform_suffix=asan --test_timeout=600 centipede/puzzles:all - name: Save new cache based on main - if: github.ref == 'refs/heads/main' + # Save a new cache on nightly scheduled runs (or manual dispatch on + # main), or on a push to main when no previous cache exists for this + # key prefix. This avoids uploading duplicate immutable cache archives + # on every commit pushed to main. + if: >- + github.ref == 'refs/heads/main' && + (github.event_name != 'push' || steps.cache-restore.outputs.cache-matched-key == '') uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: - path: "~/.cache/bazel" + path: "~/.cache/bazel-cache" key: bazel-centipede-cache-mac-${{ github.run_id }} diff --git a/.github/workflows/cargo_test.yml b/.github/workflows/cargo_test.yml index 48aa21573..bd7172870 100644 --- a/.github/workflows/cargo_test.yml +++ b/.github/workflows/cargo_test.yml @@ -56,12 +56,25 @@ jobs: toolchain: nightly - name: Restore Bazel cache + # Skip restoring on scheduled nightly runs so the cache is rebuilt from + # scratch and stale artifacts from older commits/dependencies are + # evicted. + if: github.event_name != 'schedule' + id: cache-restore uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: - path: "~/.cache/bazel" + # Cache only the Bazel repository and disk caches (configured via + # --config=ci in .bazelrc) instead of ~/.cache/bazel + # (output_user_root), which contains the full execroot and linked + # binaries. + path: "~/.cache/bazel-cache" key: bazel-centipede-cache-cargo-test restore-keys: bazel-centipede-cache-cargo-test- + - name: Configure Bazel CI cache + run: | + echo "common --config=ci" >> ~/.bazelrc + - name: Build Centipede static library and binary with Bazel run: | bazel build -c opt //centipede:centipede_engine_static //centipede:centipede @@ -74,8 +87,14 @@ jobs: cargo test --locked --workspace --no-fail-fast -- --test-threads=1 - name: Save Bazel cache based on main - if: github.ref == 'refs/heads/main' + # Save a new cache on nightly scheduled runs (or manual dispatch on + # main), or on a push to main when no previous cache exists for this + # key prefix. This avoids uploading duplicate immutable cache archives + # on every commit pushed to main. + if: >- + github.ref == 'refs/heads/main' && + (github.event_name != 'push' || steps.cache-restore.outputs.cache-matched-key == '') uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: - path: "~/.cache/bazel" + path: "~/.cache/bazel-cache" key: bazel-centipede-cache-cargo-test-${{ github.run_id }} diff --git a/.github/workflows/cmake_test.yml b/.github/workflows/cmake_test.yml index 808402407..a0d535815 100644 --- a/.github/workflows/cmake_test.yml +++ b/.github/workflows/cmake_test.yml @@ -37,6 +37,9 @@ jobs: env: CCACHE_BASEDIR: ${{ github.workspace }} CCACHE_DIR: ${{ github.workspace }}/.ccache + # Cap ccache size so cached objects across clang and gcc builds do not + # grow unboundedly. + CCACHE_MAXSIZE: 2G strategy: matrix: mode: ['default', 'fuzzing', 'codelab'] @@ -51,6 +54,10 @@ jobs: sudo apt-get update && sudo apt-get install -yq \ ccache clang gcc g++ cmake ninja-build libprotobuf-dev protobuf-compiler - name: Restore latest cache + # Skip restoring on scheduled nightly runs so the cache is rebuilt from + # scratch and stale objects from older commits/dependencies are evicted. + if: github.event_name != 'schedule' + id: cache-restore uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ${{ env.CCACHE_DIR }} @@ -116,7 +123,13 @@ jobs: && cmake --build build -j $(nproc) \ && ctest --test-dir build -j $(nproc) --output-on-failure -R "functional_test" - name: Save new cache based on main - if: github.ref == 'refs/heads/main' + # Save a new cache on nightly scheduled runs (or manual dispatch on + # main), or on a push to main when no previous cache exists for this + # key prefix. This avoids uploading duplicate immutable cache archives + # on every commit pushed to main. + if: >- + github.ref == 'refs/heads/main' && + (github.event_name != 'push' || steps.cache-restore.outputs.cache-matched-key == '') uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ${{ env.CCACHE_DIR }}