FIX intermittent Windows module enumeration failure (#217)

CHANGES.md

@@ -1,3 +1,12 @@
+3.7.0 (TBD)
+===========
+
+- Fixed an intermittent `OSError` on Windows when DLLs are loaded or unloaded
+  concurrently during library discovery (for example when importing conda-forge
+  OpenCV). Windows module enumeration now uses a snapshot-first approach with
+  graceful per-module fallbacks.
+  https://github.com/joblib/threadpoolctl/issues/217
+
 3.6.0 (2025-03-13)
 ==================
 

tests/test_threadpoolctl.py

@@ -5,13 +5,20 @@
 import subprocess
 import sys
 
+import threadpoolctl
 from threadpoolctl import threadpool_limits, threadpool_info
 from threadpoolctl import ThreadpoolController
 from threadpoolctl import _ALL_PREFIXES, _ALL_USER_APIS
 
 from .utils import cython_extensions_compiled
 from .utils import check_nested_prange_blas
 from .utils import libopenblas_paths
+from .utils import copy_and_load_dll
+from .utils import get_openblas_dll_path
+from .utils import LONG_PATH_OPENBLAS_DLL
+from .utils import make_long_windows_path
+from .utils import normalize_windows_path
+from .utils import TRUNCATED_PATH_OPENBLAS_DLL
 from .utils import scipy
 from .utils import threadpool_info_from_subprocess
 from .utils import select
@@ -613,6 +620,7 @@ def test_architecture():
     expected_openblas_architectures = (
         # XXX: add more as needed by CI or developer laptops
         "armv8",
+        "cooperlake",
         "haswell",
         "neoversen1",
         "prescott",  # see: https://github.com/xianyi/OpenBLAS/pull/3485
@@ -792,3 +800,69 @@ def test_custom_controller():
         assert mylib_controller.num_threads == 1
 
     assert ThreadpoolController().info() == original_info
+
+
+def test_threadpool_controller_repeated_init():
+    """Stress-test repeated library discovery.
+
+    Non-regression test for a Windows-specific problem where DLLs loaded or
+    unloaded concurrently during discovery could raise an OSError; see
+    https://github.com/joblib/threadpoolctl/issues/217
+    """
+    for _ in range(100):
+        ThreadpoolController()
+
+
+@pytest.mark.skipif(sys.platform != "win32", reason="Windows-only test")
+def test_windows_library_path_longer_than_max_path(tmp_path):
+    """OpenBLAS loaded from a path longer than MAX_PATH is discovered.
+
+    Regression test inspired by the local repro in
+    https://github.com/joblib/threadpoolctl/pull/189#issuecomment-2714235916
+    """
+    src_dll = get_openblas_dll_path()
+    if src_dll is None:
+        pytest.skip("Requires OpenBLAS on Windows")
+
+    long_path = make_long_windows_path(tmp_path, LONG_PATH_OPENBLAS_DLL, min_length=261)
+    copy_and_load_dll(src_dll, long_path)
+
+    expected_path = normalize_windows_path(long_path)
+    openblas_info = ThreadpoolController().select(internal_api="openblas").info()
+
+    long_path_entries = [info for info in openblas_info if len(info["filepath"]) > 260]
+    assert len(long_path_entries) >= 1
+    assert any(
+        normalize_windows_path(info["filepath"]) == expected_path
+        for info in long_path_entries
+    )
+
+
+@pytest.mark.skipif(sys.platform != "win32", reason="Windows-only test")
+def test_windows_library_path_exceeds_internal_limit(tmp_path, monkeypatch):
+    """Libraries with a path longer than the internal limit are ignored.
+
+    Regression test inspired by the local repro in
+    https://github.com/joblib/threadpoolctl/pull/189#issuecomment-2714235916
+    """
+    src_dll = get_openblas_dll_path()
+    if src_dll is None:
+        pytest.skip("Requires OpenBLAS on Windows")
+
+    monkeypatch.setattr(threadpoolctl, "_WINDOWS_MAX_LIBRARY_PATH_LENGTH", 300)
+
+    long_path = make_long_windows_path(
+        tmp_path, TRUNCATED_PATH_OPENBLAS_DLL, min_length=400
+    )
+    copy_and_load_dll(src_dll, long_path)
+
+    expected_path = normalize_windows_path(long_path)
+    with pytest.warns(RuntimeWarning, match="path too long"):
+        info = ThreadpoolController().info()
+
+    filepaths = {
+        normalize_windows_path(entry["filepath"])
+        for entry in info
+        if "filepath" in entry
+    }
+    assert expected_path not in filepaths

tests/utils.py

@@ -1,9 +1,12 @@
-import os
 import json
+import os
 import sys
+import ctypes
+import shutil
 import threadpoolctl
 from glob import glob
 from os.path import dirname, normpath
+from pathlib import Path
 from subprocess import check_output
 
 # Path to shipped openblas for libraries such as numpy or scipy
@@ -17,6 +20,17 @@
     np.dot(np.ones(1000), np.ones(1000))
 
     libopenblas_patterns.append(os.path.join(np.__path__[0], ".libs", "libopenblas*"))
+    numpy_site_packages = os.path.dirname(np.__path__[0])
+    libopenblas_patterns.append(
+        os.path.join(numpy_site_packages, "numpy.libs", "libscipy_openblas*.dll")
+    )
+    if sys.platform == "win32":
+        libopenblas_patterns.append(
+            os.path.join(sys.prefix, "Library", "bin", "openblas*.dll")
+        )
+        libopenblas_patterns.append(
+            os.path.join(sys.prefix, "Library", "bin", "libopenblas*.dll")
+        )
 except ImportError:
     pass
 
@@ -30,6 +44,10 @@
     libopenblas_patterns.append(
         os.path.join(scipy.__path__[0], ".libs", "libopenblas*")
     )
+    scipy_site_packages = os.path.dirname(scipy.__path__[0])
+    libopenblas_patterns.append(
+        os.path.join(scipy_site_packages, "scipy.libs", "libscipy_openblas*.dll")
+    )
 except ImportError:
     scipy = None
 
@@ -86,3 +104,73 @@ def select(info, **kwargs):
     ]
 
     return selected_info
+
+
+def get_openblas_dll_path():
+    """Return a path to an OpenBLAS DLL that can be copied for Windows tests."""
+    if libopenblas_paths:
+        return sorted(
+            libopenblas_paths,
+            key=lambda path: (
+                0
+                if "libscipy_openblas" in os.path.basename(path).lower()
+                else 1 if "libopenblas" in os.path.basename(path).lower() else 2
+            ),
+        )[0]
+
+    from threadpoolctl import ThreadpoolController
+
+    controllers = ThreadpoolController().select(internal_api="openblas").lib_controllers
+    if not controllers:
+        return None
+
+    filepath = controllers[0].filepath
+    if os.path.isfile(filepath):
+        return filepath
+    return None
+
+
+def normalize_windows_path(path):
+    """Normalize Windows paths for stable comparisons in tests."""
+    path = os.path.abspath(str(path))
+    if path.startswith("\\\\?\\"):
+        path = path[4:]
+        if path.startswith("UNC\\"):
+            path = "\\\\" + path[4:]
+    return os.path.normcase(os.path.normpath(path))
+
+
+LONG_PATH_OPENBLAS_DLL = "libopenblas_long_path_test.dll"
+TRUNCATED_PATH_OPENBLAS_DLL = "libopenblas_path_too_long.dll"
+
+
+def make_long_windows_path(base_dir, filename, min_length=261):
+    """Nest padded directories under base_dir until base/.../filename >= min_length."""
+    padding_segments = ["a" * 100, "b" * 100, "c" * 100, "d" * 100]
+    current = Path(base_dir)
+    segment_index = 0
+    target = current / filename
+
+    while len(str(target)) < min_length:
+        current = current / padding_segments[segment_index % len(padding_segments)]
+        segment_index += 1
+        target = current / filename
+
+    target.parent.mkdir(parents=True, exist_ok=True)
+    return target
+
+
+def to_extended_windows_path(path):
+    """Return an extended-length path for Windows APIs (> MAX_PATH)."""
+    path = os.path.abspath(str(path))
+    if path.startswith("\\\\?\\"):
+        return path
+    if path.startswith("\\\\"):
+        return "\\\\?\\UNC\\" + path[2:]
+    return "\\\\?\\" + path
+
+
+def copy_and_load_dll(src_dll, destination):
+    """Copy a DLL to destination and load it in the current process."""
+    shutil.copy2(src_dll, to_extended_windows_path(destination))
+    ctypes.CDLL(to_extended_windows_path(destination))