diff --git a/scripts/import_profiler/profiler.py b/scripts/import_profiler/profiler.py index 72b73e1a755e..8ccfd7b218a9 100644 --- a/scripts/import_profiler/profiler.py +++ b/scripts/import_profiler/profiler.py @@ -377,18 +377,69 @@ def validate_module_name(module_name): raise argparse.ArgumentTypeError(f"'{module_name}' is not a valid Python module identifier.") return module_name +IGNORED_TOP_LEVEL_NAMES = { + "tests", "samples", "examples", "benchmark", "benchmarks", "third_party", + "testing", "test_utils", "docs", "build", "dist", "bin", "ci", "scripts", + "cloudbuild", "notebooks", "assets", "scratch", "specs" +} +IGNORED_NAME_PREFIXES = ( + "test_", "tests_", "sample_", "samples_", "bench_", "benchmarks_", + "example_", "examples_", "doc_", "docs_", "notebook_", "notebooks_" +) + + +def _should_process_namespace_package(top_level: str, target_pkg: str) -> bool: + """Determines if a discovered top-level directory should be processed. + + Args: + top_level: Top-level directory component of the package (e.g. 'tests' or 'google'). + target_pkg: The distribution package being profiled (e.g. 'google-cloud-storage'). + + Returns: + True if the top-level directory should be processed, False otherwise. + """ + is_excluded_candidate = ( + top_level in IGNORED_TOP_LEVEL_NAMES + or top_level.startswith(IGNORED_NAME_PREFIXES) + ) + + if is_excluded_candidate: + normalized_target_pkg = target_pkg.replace("-", "").replace("_", "").lower() + normalized_top_level = top_level.replace("-", "").replace("_", "").lower() + # Note: If the top-level folder name is part of the target package name + # (e.g. top_level='test_utils' when target_pkg='google-cloud-testutils'), then it should be processed. + return normalized_top_level in normalized_target_pkg + + return True # Not a candidate for exclusion, process it. + + +def _is_path_allowed(path_obj, pkg_norm: str) -> bool: + """Checks if a path object is allowed based on ignored directory rules.""" + from pathlib import Path + parts = Path(path_obj).parts + return all( + part not in IGNORED_TOP_LEVEL_NAMES + or part.replace("-", "").replace("_", "").lower() in pkg_norm + for part in parts + ) + + def find_module_from_package(pkg): import importlib.metadata import importlib.util + from pathlib import Path # 1. Try to use importlib.metadata.files (works for standard installations from PyPI/wheels) try: files = importlib.metadata.files(pkg) if files: - ignored_parts = {'tests', 'testing', 'samples', 'examples', 'benchmark', 'benchmarks', 'third_party', 'test_utils', 'docs', 'build', 'dist', 'bin', 'ci', 'scripts', 'cloudbuild', 'notebooks', 'assets', 'scratch', 'specs'} - if pkg == "google-cloud-testutils": - ignored_parts.discard('test_utils') - init_files = [str(f) for f in files if str(f).endswith('__init__.py') and '__pycache__' not in str(f) and not any(part in ignored_parts for part in str(f).replace('\\', '/').split('/'))] + pkg_norm = pkg.replace("-", "").replace("_", "").lower() + init_files = [ + str(f) for f in files + if Path(f).name == '__init__.py' + and '__pycache__' not in Path(f).parts + and _is_path_allowed(f, pkg_norm) + ] if init_files: from pathlib import Path shortest_init = min(init_files, key=lambda p: len(Path(p).parts)) @@ -412,18 +463,11 @@ def find_module_from_package(pkg): if abs_where_dir not in sys.path: sys.path.insert(0, abs_where_dir) pkgs = setuptools.find_namespace_packages(where=where_dir) - ignored_prefixes = ("tests", "samples", "examples", "benchmark", "benchmarks", "third_party", "testing", "test_utils", "docs", "build", "dist", "bin", "ci", "scripts", "cloudbuild", "notebooks", "assets", "scratch", "specs") - ignored_starts = ("test_", "tests_", "sample_", "samples_", "bench_", "benchmarks_", "example_", "examples_", "doc_", "docs_", "notebook_", "notebooks_") - filtered = [] for p in pkgs: top = p.split(".")[0] - is_ignored_top = top in ignored_prefixes or top.startswith(ignored_starts) - if is_ignored_top and pkg == "google-cloud-testutils" and top == "test_utils": - is_ignored_top = False - if is_ignored_top or p in ("google", "google.cloud"): - continue - filtered.append(p) + if _should_process_namespace_package(top, pkg): + filtered.append(p) # First preference: packages containing __init__.py for p in sorted(filtered, key=len): diff --git a/scripts/import_profiler/test_profiler.py b/scripts/import_profiler/test_profiler.py index 49e94b602b7a..871f5a0b810e 100644 --- a/scripts/import_profiler/test_profiler.py +++ b/scripts/import_profiler/test_profiler.py @@ -650,10 +650,13 @@ def test_find_module_from_package_metadata_test_utils(): def test_find_module_from_package_setuptools(): sys.modules.setdefault("setuptools", MagicMock()) + def mock_isfile(path): + return "my_pkg" in path with patch("importlib.metadata.files", side_effect=Exception), \ - patch("os.path.exists", return_value=True), \ + patch("profiler.os.path.exists", return_value=True), \ + patch("profiler.os.path.isdir", return_value=True), \ patch("setuptools.find_namespace_packages", return_value=["google", "google.cloud", "tests.dummy", "my_pkg"]), \ - patch("os.path.isfile", return_value=True), \ + patch("profiler.os.path.isfile", side_effect=mock_isfile), \ patch("importlib.util.find_spec", return_value=True): res = find_module_from_package("my-pkg") assert res == "my_pkg" @@ -830,4 +833,38 @@ def test_cli_main_options(): runpy.run_path(profiler_path, run_name="__main__") +def test_should_process_namespace_package(): + from profiler import _should_process_namespace_package + + # Standard non-library top-level directories should NOT be processed + assert _should_process_namespace_package("tests", "google-cloud-storage") is False + assert _should_process_namespace_package("samples", "google-cloud-storage") is False + assert _should_process_namespace_package("test_utils", "google-cloud-storage") is False + assert _should_process_namespace_package("test_helpers", "google-cloud-storage") is False + + # Exception: Target package explicitly contains the top-level directory name (e.g. google-cloud-testutils -> test_utils) + assert _should_process_namespace_package("test_utils", "google-cloud-testutils") is True + + # Valid library package top-level should be processed + assert _should_process_namespace_package("google", "google-cloud-storage") is True + assert _should_process_namespace_package("my_library", "my-library") is True + + +def test_find_module_from_package_testutils(): + """Verifies that google-cloud-testutils correctly resolves to test_utils namespace package.""" + sys.modules.setdefault("setuptools", MagicMock()) + def mock_isfile(path): + return "test_utils" in path + with patch("importlib.metadata.files", side_effect=Exception), \ + patch("profiler.os.path.exists", return_value=True), \ + patch("profiler.os.path.isdir", return_value=True), \ + patch("setuptools.find_namespace_packages", return_value=["google", "google.cloud", "tests", "test_utils"]) as mock_find, \ + patch("profiler.os.path.isfile", side_effect=mock_isfile), \ + patch("importlib.util.find_spec", return_value=True): + res = find_module_from_package("google-cloud-testutils") + assert res == "test_utils" + mock_find.assert_called_once_with(where="src") + + +