From c7c4106845f9e37b53d0b69a9ec2e609f675770d Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 16 Oct 2026 14:16:11 -0700 Subject: [PATCH 1/5] build: drop the empty venv include dir from build_ext include_dirs In a virtualenv, setuptools' build_ext appends /include. Under PEP 517 build isolation that is a fresh /tmp/build-env- per build, so every compile command differs and sccache's preprocessor cache, which hashes the -I flags, never hits. Drop the directory when it holds no files; it contributes no headers then, so compilation is unchanged. The helper lives in _build_shared.py and is applied in both packages' build_ext.finalize_options, with tests in the shared build_shared mixins. --- cuda_bindings/_build_shared.py | 27 ++++++++++++- cuda_bindings/setup.py | 8 ++++ cuda_bindings/tests/test_build_hooks.py | 5 +++ cuda_core/setup.py | 5 +++ cuda_core/tests/test_build_hooks.py | 5 +++ .../cuda_python_test_helpers/build_shared.py | 40 +++++++++++++++++++ 6 files changed, 88 insertions(+), 2 deletions(-) diff --git a/cuda_bindings/_build_shared.py b/cuda_bindings/_build_shared.py index 5a1231d1a75..656e7ab5c2d 100644 --- a/cuda_bindings/_build_shared.py +++ b/cuda_bindings/_build_shared.py @@ -18,8 +18,9 @@ ``tweak`` hook for flags a single package needs. Besides the toolchain, this module owns the CUDA path lookup and the machinery that both backends use -around cythonize and build_ext: the opt-in Cython generated-source cache and -the build stamps that force a rebuild when the build configuration changed. +around cythonize and build_ext: the opt-in Cython generated-source cache, the +include-dir cleanup that keeps compile commands stable across isolated builds, +and the build stamps that force a rebuild when the build configuration changed. Note: There is no support guarantee for environment variables like CUDA_PYTHON_TOOLCHAIN. They may be removed or changed in the future. @@ -429,6 +430,28 @@ def _stable_cython_alias(target: Path, alias: Path): pass +# ----------------------------------------------------------------------- +# build_ext include dirs + + +def drop_empty_venv_include_dir(include_dirs): + """Return ``include_dirs`` without the virtualenv include dir if it holds no files. + + In a virtualenv, setuptools' build_ext appends ``/include``. + Under PEP 517 build isolation that prefix is a fresh temporary directory + (``/tmp/build-env-``), so the ``-I`` flag differs on every build, + and compiler caches that key on the command line (sccache's preprocessor + cache) never hit. A directory without files contributes no headers, so + dropping it leaves the compilation unchanged. + """ + if sys.exec_prefix == sys.base_exec_prefix: + return include_dirs + venv_include = os.path.join(sys.exec_prefix, "include") + if any(files for _, _, files in os.walk(venv_include)): + return include_dirs + return [d for d in include_dirs if d != venv_include] + + # ----------------------------------------------------------------------- # Build stamps # diff --git a/cuda_bindings/setup.py b/cuda_bindings/setup.py index 4481adba047..d4495e4ab6b 100644 --- a/cuda_bindings/setup.py +++ b/cuda_bindings/setup.py @@ -10,6 +10,8 @@ from setuptools.command.build_ext import build_ext as _build_ext from setuptools.command.build_py import build_py as _build_py +from _build_shared import drop_empty_venv_include_dir + if os.environ.get("PARALLEL_LEVEL") is not None: warn( "Environment variable PARALLEL_LEVEL is deprecated. Use CUDA_PYTHON_PARALLEL_LEVEL instead", @@ -24,6 +26,12 @@ class build_ext(_build_ext): + def finalize_options(self): + super().finalize_options() + # A per-build venv path on every -I line defeats sccache's + # preprocessor cache; see _build_shared.drop_empty_venv_include_dir(). + self.include_dirs = drop_empty_venv_include_dir(self.include_dirs) + def build_extensions(self): if nthreads > 0: self.parallel = nthreads diff --git a/cuda_bindings/tests/test_build_hooks.py b/cuda_bindings/tests/test_build_hooks.py index b45b58fa102..95c35220b76 100644 --- a/cuda_bindings/tests/test_build_hooks.py +++ b/cuda_bindings/tests/test_build_hooks.py @@ -125,6 +125,7 @@ def test_record_writes_stamp(self, stamp): BuildKeyStampMixin, CheckToolchainAvailableMixin, DistutilsLinkerIntegrationMixin, + DropEmptyVenvIncludeDirMixin, ForceBuildExtReexportMixin, ResolveToolchainMixin, WithCompilerMixin, @@ -161,6 +162,10 @@ class TestBuildKeyStamp(BuildKeyStampMixin): build_shared = _build_shared +class TestDropEmptyVenvIncludeDir(DropEmptyVenvIncludeDirMixin): + build_shared = _build_shared + + class TestForceBuildExtReexport(ForceBuildExtReexportMixin): build_hooks = build_hooks build_shared = _build_shared diff --git a/cuda_core/setup.py b/cuda_core/setup.py index 3127104cc49..1e6c4f9508d 100644 --- a/cuda_core/setup.py +++ b/cuda_core/setup.py @@ -13,6 +13,8 @@ from setuptools.command.build_ext import build_ext as _build_ext from setuptools.command.build_py import build_py as _build_py +from _build_shared import drop_empty_venv_include_dir + nthreads = int(os.environ.get("CUDA_PYTHON_PARALLEL_LEVEL", os.cpu_count() // 2)) coverage_mode = bool(int(os.environ.get("CUDA_PYTHON_COVERAGE", "0"))) _ROOT_DIR = Path(__file__).resolve().parent @@ -56,6 +58,9 @@ def _build_aoti_shim_lib(compiler, plat_name): class build_ext(_build_ext): # noqa: N801 def finalize_options(self): super().finalize_options() + # A per-build venv path on every -I line defeats sccache's + # preprocessor cache; see _build_shared.drop_empty_venv_include_dir(). + self.include_dirs = drop_empty_venv_include_dir(self.include_dirs) # A stale .so from a previous build configuration (CUDA major, # toolchain, debug/coverage) looks perfectly fresh; see # build_hooks._check_build_config(). diff --git a/cuda_core/tests/test_build_hooks.py b/cuda_core/tests/test_build_hooks.py index fd016f9df71..032f19f898c 100644 --- a/cuda_core/tests/test_build_hooks.py +++ b/cuda_core/tests/test_build_hooks.py @@ -908,6 +908,7 @@ def fake_cythonize(ext_modules, **kwargs): BuildKeyStampMixin, CheckToolchainAvailableMixin, DistutilsLinkerIntegrationMixin, + DropEmptyVenvIncludeDirMixin, ForceBuildExtReexportMixin, ResolveToolchainMixin, WithCompilerMixin, @@ -944,6 +945,10 @@ class TestBuildKeyStamp(BuildKeyStampMixin): build_shared = _build_shared +class TestDropEmptyVenvIncludeDir(DropEmptyVenvIncludeDirMixin): + build_shared = _build_shared + + class TestForceBuildExtReexport(ForceBuildExtReexportMixin): build_hooks = build_hooks build_shared = _build_shared diff --git a/cuda_python_test_helpers/cuda_python_test_helpers/build_shared.py b/cuda_python_test_helpers/cuda_python_test_helpers/build_shared.py index ec70b60c76b..bf0596c595e 100644 --- a/cuda_python_test_helpers/cuda_python_test_helpers/build_shared.py +++ b/cuda_python_test_helpers/cuda_python_test_helpers/build_shared.py @@ -470,6 +470,46 @@ def test_record_creates_the_directory_and_writes_the_key(self, stamp_file): assert stamp_file.read_text(encoding="utf-8") == "cu13-gnu-opt\n" +class DropEmptyVenvIncludeDirMixin: + """``drop_empty_venv_include_dir``: remove the venv include dir only when it holds no files.""" + + build_shared = None + + @pytest.fixture + def venv(self, tmp_path, monkeypatch): + venv = tmp_path / "build-env-abc123" + monkeypatch.setattr(sys, "exec_prefix", str(venv)) + monkeypatch.setattr(sys, "base_exec_prefix", str(tmp_path / "python")) + return venv + + @pytest.mark.agent_authored(model="claude-opus-5.5") + def test_outside_a_venv_returns_the_list_unchanged(self, tmp_path, monkeypatch): + monkeypatch.setattr(sys, "exec_prefix", str(tmp_path)) + monkeypatch.setattr(sys, "base_exec_prefix", str(tmp_path)) + dirs = ["/opt/cuda/include", os.path.join(str(tmp_path), "include")] + assert self.build_shared.drop_empty_venv_include_dir(dirs) == dirs + + @pytest.mark.agent_authored(model="claude-opus-5.5") + def test_missing_venv_include_dir_is_dropped(self, venv): + venv_include = os.path.join(str(venv), "include") + dirs = ["/opt/cuda/include", venv_include, "/usr/include/python3"] + assert self.build_shared.drop_empty_venv_include_dir(dirs) == ["/opt/cuda/include", "/usr/include/python3"] + + @pytest.mark.agent_authored(model="claude-opus-5.5") + def test_venv_include_dir_with_only_empty_subdirs_is_dropped(self, venv): + (venv / "include" / "site" / "python3.14").mkdir(parents=True) + dirs = ["/opt/cuda/include", os.path.join(str(venv), "include")] + assert self.build_shared.drop_empty_venv_include_dir(dirs) == ["/opt/cuda/include"] + + @pytest.mark.agent_authored(model="claude-opus-5.5") + def test_venv_include_dir_with_a_header_is_kept(self, venv): + header_dir = venv / "include" / "site" / "python3.14" / "greenlet" + header_dir.mkdir(parents=True) + (header_dir / "greenlet.h").write_text("", encoding="utf-8") + dirs = ["/opt/cuda/include", os.path.join(str(venv), "include")] + assert self.build_shared.drop_empty_venv_include_dir(dirs) == dirs + + class ForceBuildExtReexportMixin: """``build_hooks.force_build_ext`` is a live view of ``build_shared.force_build_ext``. From c0469c9f5366fa806cb857267e7f0249a23f9578 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 16 Oct 2026 14:16:13 -0700 Subject: [PATCH 2/5] build: keep build-env library dirs out of cuda.bindings Extensions Cython copies each Extension's library_dirs into the metadata comment of the generated .cpp. Under PEP 517 build isolation, platlib and /lib are /tmp/build-env- paths, so the generated sources differ per build, and sccache's preprocessor cache, which hashes the raw source, misses even when the object cache hits. Drop platlib: nothing links from site-packages itself. Add /lib (where a conda environment can provide libcudart_static.a) in build_ext.finalize_options instead, so the link line still has it but the generated sources no longer do. --- cuda_bindings/build_hooks.py | 6 ++++-- cuda_bindings/setup.py | 7 +++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/cuda_bindings/build_hooks.py b/cuda_bindings/build_hooks.py index e49cb99d48c..65a9912139b 100644 --- a/cuda_bindings/build_hooks.py +++ b/cuda_bindings/build_hooks.py @@ -284,12 +284,14 @@ def _build_cuda_bindings(debug=False): include_dirs = [ os.path.dirname(sysconfig.get_path("include")), ] + include_path_list - library_dirs = [sysconfig.get_path("platlib"), os.path.join(os.sys.prefix, "lib")] + # Keep build-environment paths out of the Extensions: Cython copies their + # library_dirs into the generated sources. setup.py's build_ext adds + # /lib instead. if sys.platform == "win32": cudalib_subdirs = [r"lib\arm64"] if sysconfig.get_platform() == "win-arm64" else [r"lib\x64"] else: cudalib_subdirs = ["lib64", "lib"] - library_dirs.extend(os.path.join(cuda_path, subdir) for subdir in cudalib_subdirs) + library_dirs = [os.path.join(cuda_path, subdir) for subdir in cudalib_subdirs] # Rename architecture-specific files dst_files = _rename_architecture_specific_files() diff --git a/cuda_bindings/setup.py b/cuda_bindings/setup.py index d4495e4ab6b..9381953281d 100644 --- a/cuda_bindings/setup.py +++ b/cuda_bindings/setup.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 import os +import sys from warnings import warn import build_hooks @@ -31,6 +32,12 @@ def finalize_options(self): # A per-build venv path on every -I line defeats sccache's # preprocessor cache; see _build_shared.drop_empty_venv_include_dir(). self.include_dirs = drop_empty_venv_include_dir(self.include_dirs) + # A conda environment can provide libcudart_static.a in /lib. + # Added here rather than to the Extensions: Cython copies Extension + # library_dirs into the generated sources, and under build isolation + # sys.prefix is a fresh /tmp/build-env- per build, which + # defeats sccache's preprocessor cache. + self.library_dirs.append(os.path.join(sys.prefix, "lib")) def build_extensions(self): if nthreads > 0: From b4a61b71f7fbdbeedc33f6a9a7fe401533b1a78c Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 16 Oct 2026 14:16:14 -0700 Subject: [PATCH 3/5] ci: enable sccache's GHA preprocessor cache in Linux wheel builds rapidsai/sccache can store preprocessor-cache entries in the GHA cache alongside object entries. A preprocessor hit skips running the preprocessor, which the object cache alone always has to do to compute its key. The two preceding commits make the compile commands and the generated sources stable across isolated builds, so these entries can now hit. --- .github/workflows/build-wheel.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-wheel.yml b/.github/workflows/build-wheel.yml index a894387aec2..4e69de2d358 100644 --- a/.github/workflows/build-wheel.yml +++ b/.github/workflows/build-wheel.yml @@ -246,6 +246,7 @@ jobs: CC="/host/${{ env.SCCACHE_PATH }} cc" CXX="/host/${{ env.SCCACHE_PATH }} c++" SCCACHE_GHA_ENABLED=true + SCCACHE_PREPROCESSOR_CACHE_GHA_ENABLED=true ACTIONS_RUNTIME_TOKEN=${{ env.ACTIONS_RUNTIME_TOKEN }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=${{ env.ACTIONS_CACHE_SERVICE_V2 }} @@ -343,6 +344,7 @@ jobs: CC="/host/${{ env.SCCACHE_PATH }} cc" CXX="/host/${{ env.SCCACHE_PATH }} c++" SCCACHE_GHA_ENABLED=true + SCCACHE_PREPROCESSOR_CACHE_GHA_ENABLED=true ACTIONS_RUNTIME_TOKEN=${{ env.ACTIONS_RUNTIME_TOKEN }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=${{ env.ACTIONS_CACHE_SERVICE_V2 }} @@ -620,6 +622,7 @@ jobs: CC="/host/${{ env.SCCACHE_PATH }} cc" CXX="/host/${{ env.SCCACHE_PATH }} c++" SCCACHE_GHA_ENABLED=true + SCCACHE_PREPROCESSOR_CACHE_GHA_ENABLED=true ACTIONS_RUNTIME_TOKEN=${{ env.ACTIONS_RUNTIME_TOKEN }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=${{ env.ACTIONS_CACHE_SERVICE_V2 }} From 02a542acd42468eff34ecf9d1a80a044bb331efb Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 16 Oct 2026 15:21:40 -0700 Subject: [PATCH 4/5] build: follow symlinked dirs when checking the venv include dir drop_empty_venv_include_dir() used os.walk() without followlinks, so a /include whose only content is symlinked directories (for example linked header trees) looked empty. The helper then dropped the -I flag even though headers were reachable through it, breaking compiles that need them. Walk with followlinks=True so headers behind symlinked directories count as files, and add regression tests for a symlinked directory that holds a header and for one that is empty. --- cuda_bindings/_build_shared.py | 5 +++- .../cuda_python_test_helpers/build_shared.py | 25 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/cuda_bindings/_build_shared.py b/cuda_bindings/_build_shared.py index 656e7ab5c2d..aedb3f57993 100644 --- a/cuda_bindings/_build_shared.py +++ b/cuda_bindings/_build_shared.py @@ -443,11 +443,14 @@ def drop_empty_venv_include_dir(include_dirs): and compiler caches that key on the command line (sccache's preprocessor cache) never hit. A directory without files contributes no headers, so dropping it leaves the compilation unchanged. + + Symlinked subdirectories are followed: headers reachable through them + still count as files. """ if sys.exec_prefix == sys.base_exec_prefix: return include_dirs venv_include = os.path.join(sys.exec_prefix, "include") - if any(files for _, _, files in os.walk(venv_include)): + if any(files for _, _, files in os.walk(venv_include, followlinks=True)): return include_dirs return [d for d in include_dirs if d != venv_include] diff --git a/cuda_python_test_helpers/cuda_python_test_helpers/build_shared.py b/cuda_python_test_helpers/cuda_python_test_helpers/build_shared.py index bf0596c595e..f9bfc1e6b1b 100644 --- a/cuda_python_test_helpers/cuda_python_test_helpers/build_shared.py +++ b/cuda_python_test_helpers/cuda_python_test_helpers/build_shared.py @@ -509,6 +509,31 @@ def test_venv_include_dir_with_a_header_is_kept(self, venv): dirs = ["/opt/cuda/include", os.path.join(str(venv), "include")] assert self.build_shared.drop_empty_venv_include_dir(dirs) == dirs + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_venv_include_dir_with_a_header_behind_a_symlinked_dir_is_kept(self, venv, tmp_path): + vendor = tmp_path / "vendor-headers" + vendor.mkdir() + (vendor / "config.h").write_text("", encoding="utf-8") + (venv / "include").mkdir(parents=True) + try: + (venv / "include" / "vendor").symlink_to(vendor, target_is_directory=True) + except OSError as exc: # e.g. Windows without symlink privilege + pytest.skip(f"cannot create symlinks: {exc}") + dirs = ["/opt/cuda/include", os.path.join(str(venv), "include")] + assert self.build_shared.drop_empty_venv_include_dir(dirs) == dirs + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_venv_include_dir_with_a_symlink_to_an_empty_dir_is_dropped(self, venv, tmp_path): + empty = tmp_path / "empty-headers" + empty.mkdir() + (venv / "include").mkdir(parents=True) + try: + (venv / "include" / "vendor").symlink_to(empty, target_is_directory=True) + except OSError as exc: # e.g. Windows without symlink privilege + pytest.skip(f"cannot create symlinks: {exc}") + dirs = ["/opt/cuda/include", os.path.join(str(venv), "include")] + assert self.build_shared.drop_empty_venv_include_dir(dirs) == ["/opt/cuda/include"] + class ForceBuildExtReexportMixin: """``build_hooks.force_build_ext`` is a live view of ``build_shared.force_build_ext``. From a0049af7df90397dd58de6501c02a063aaa2fc27 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 16 Oct 2026 15:25:33 -0700 Subject: [PATCH 5/5] build: do not add /lib to library_dirs twice build_ext.finalize_options unconditionally appended /lib. Skip the append when the directory is already in library_dirs, so a repeated finalize_options call or a pre-populated list does not produce a duplicate -L flag. --- cuda_bindings/setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cuda_bindings/setup.py b/cuda_bindings/setup.py index 9381953281d..1b02b43bace 100644 --- a/cuda_bindings/setup.py +++ b/cuda_bindings/setup.py @@ -37,7 +37,9 @@ def finalize_options(self): # library_dirs into the generated sources, and under build isolation # sys.prefix is a fresh /tmp/build-env- per build, which # defeats sccache's preprocessor cache. - self.library_dirs.append(os.path.join(sys.prefix, "lib")) + prefix_lib = os.path.join(sys.prefix, "lib") + if prefix_lib not in self.library_dirs: + self.library_dirs.append(prefix_lib) def build_extensions(self): if nthreads > 0: