From 34c2be91f650273790984b696269404b7fb57c5c Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Wed, 16 Sep 2026 17:39:31 +0300
Subject: [PATCH 1/3] add Python test images to pyspark-core module

---
 dev/sparktestsupport/modules.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
index 2690a0c2578f1..79cd7851cdec5 100644
--- a/dev/sparktestsupport/modules.py
+++ b/dev/sparktestsupport/modules.py
@@ -577,7 +577,10 @@ def __hash__(self):
 pyspark_core = Module(
     name="pyspark-core",
     dependencies=[core],
-    source_file_regexes=["python/(?!pyspark/(ml|mllib|sql|streaming|pandas|resource|testing))"],
+    source_file_regexes=[
+        "python/(?!pyspark/(ml|mllib|sql|streaming|pandas|resource|testing))",
+        "dev/spark-test-image/python-.*/Dockerfile",
+    ],
     python_test_goals=[
         # doctests
         "pyspark.conf",

From af4a760d8ec4aa202a626acfef6a3654b830166d Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Sat, 26 Sep 2026 12:15:30 +0300
Subject: [PATCH 2/3] add shared pyspark base module

---
 dev/sparktestsupport/modules.py | 49 ++++++++++++++++++++++++++++++---
 1 file changed, 45 insertions(+), 4 deletions(-)

diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
index 79cd7851cdec5..599ca8943b30a 100644
--- a/dev/sparktestsupport/modules.py
+++ b/dev/sparktestsupport/modules.py
@@ -574,12 +574,21 @@ def __hash__(self):
     ],
 )
 
+# Shared Python test environment.
+pyspark_base = Module(
+    name="pyspark-base",
+    dependencies=[],
+    source_file_regexes=[
+        "dev/spark-test-image/python-.*/",
+        "pyproject.toml",
+    ],
+)
+
 pyspark_core = Module(
     name="pyspark-core",
-    dependencies=[core],
+    dependencies=[core, pyspark_base],
     source_file_regexes=[
         "python/(?!pyspark/(ml|mllib|sql|streaming|pandas|resource|testing))",
-        "dev/spark-test-image/python-.*/Dockerfile",
     ],
     python_test_goals=[
         # doctests
@@ -934,7 +943,7 @@ def __hash__(self):
 
 pyspark_install = Module(
     name="pyspark-install",
-    dependencies=[],
+    dependencies=[pyspark_base],
     source_file_regexes=[
         # Python package tests will be triggered with this module
         # Any changes in python/ should trigger this module
@@ -1716,7 +1725,7 @@ def __hash__(self):
 
 pyspark_logger = Module(
     name="pyspark-logger",
-    dependencies=[],
+    dependencies=[pyspark_base],
     source_file_regexes=["python/pyspark/logger"],
     python_test_goals=[
         # doctests
@@ -1844,6 +1853,38 @@ def __hash__(self):
 )
 
 
+def pyspark_modules_missing_base() -> list[str]:
+    """
+    Return PySpark modules that do not depend on `pyspark-base`.
+
+    Every `pyspark-*` module except `pyspark-base` itself must reach it,
+    directly or transitively. That is what makes a shared Python test-environment
+    change rerun the module. CI selects these modules by the same name prefix.
+
+    >>> pyspark_modules_missing_base()
+    []
+    """
+    missing = []
+    for module in all_modules:
+        if not module.name.startswith("pyspark-") or module is pyspark_base:
+            continue
+        seen = set()
+        pending = list(module.dependencies)
+        reaches_base = False
+        while pending:
+            dependency = pending.pop()
+            if dependency in seen:
+                continue
+            seen.add(dependency)
+            if dependency is pyspark_base:
+                reaches_base = True
+                break
+            pending.extend(dependency.dependencies)
+        if not reaches_base:
+            missing.append(module.name)
+    return missing
+
+
 def _test():
     import doctest
 

From 3e4904c6c207c5b3a22bd459112fbdfe15dbbad3 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Sat, 26 Sep 2026 12:55:09 +0300
Subject: [PATCH 3/3] undo whitespace change

---
 dev/sparktestsupport/modules.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
index 115e2efbd1e80..3dba198c41f43 100644
--- a/dev/sparktestsupport/modules.py
+++ b/dev/sparktestsupport/modules.py
@@ -587,9 +587,7 @@ def __hash__(self):
 pyspark_core = Module(
     name="pyspark-core",
     dependencies=[core, pyspark_base],
-    source_file_regexes=[
-        "python/(?!pyspark/(ml|mllib|sql|streaming|pandas|resource|testing))",
-    ],
+    source_file_regexes=["python/(?!pyspark/(ml|mllib|sql|streaming|pandas|resource|testing))"],
     python_test_goals=[
         # doctests
         "pyspark.conf",