[SPARK-58388][INFRA] Clean up the dependency list for lint/docs/gen-protos docker image

.github/workflows/build_and_test.yml

@@ -490,6 +490,8 @@ jobs:
         uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
         with:
           context: ./dev/spark-test-image/docs/
+          build-contexts: |
+            root=./
           push: true
           tags: |
             ${{ needs.precondition.outputs.image_docs_url }}
@@ -501,6 +503,8 @@ jobs:
         uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
         with:
           context: ./dev/spark-test-image/lint/
+          build-contexts: |
+            root=./
           push: true
           tags: |
             ${{ needs.precondition.outputs.image_lint_url }}

.github/workflows/build_infra_images_cache.yml

@@ -75,6 +75,8 @@ jobs:
         uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
         with:
           context: ./dev/spark-test-image/docs/
+          build-contexts: |
+            root=./
           push: true
           tags: ghcr.io/apache/spark/apache-spark-github-action-image-docs-cache:${{ github.ref_name }}-static
           cache-from: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-docs-cache:${{ github.ref_name }}
@@ -89,6 +91,8 @@ jobs:
         uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
         with:
           context: ./dev/spark-test-image/lint/
+          build-contexts: |
+            root=./
           push: true
           tags: ghcr.io/apache/spark/apache-spark-github-action-image-lint-cache:${{ github.ref_name }}-static
           cache-from: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-lint-cache:${{ github.ref_name }}

dev/spark-test-image/connect-gen-protos/Dockerfile

@@ -51,11 +51,11 @@ ENV VIRTUAL_ENV=/opt/spark-venv
 RUN python3.12 -m venv $VIRTUAL_ENV
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
-# Keep these versions in sync with the pinned versions in pyproject.toml
-RUN python3.12 -m pip install \
-    'mypy==1.19.1' \
-    'mypy-protobuf==3.3.0' \
-    'ruff==0.14.8'
+COPY --from=root pyproject.toml ./pyproject.toml
+
+RUN python3.12 -m pip install -U pip
+
+RUN python3.12 -m pip install --group ci_lint
 
 # Mount the Spark repo at /spark
 WORKDIR /spark

dev/spark-test-image/docs/Dockerfile

@@ -89,9 +89,9 @@ ENV VIRTUAL_ENV=/opt/spark-venv
 RUN python3.12 -m venv $VIRTUAL_ENV
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
-# See 'docutils<0.18.0' in SPARK-39421
-RUN python3.12 -m pip install 'sphinx==8.2.3' mkdocs 'pydata_sphinx_theme>=0.13' sphinx-copybutton nbsphinx numpydoc markupsafe \
-  ipython sphinx_plotly_directive 'numpy>=1.23.2' 'pyarrow>=23.0.0' 'pandas==2.3.3' 'plotly>=4.8' 'docutils' \
-  'mypy==1.19.1' 'pytest==7.1.3' 'pytest-mypy-plugins==1.9.3' 'ruff==0.14.8' \
-  'pandas-stubs==1.2.0.53' 'grpcio==1.76.0' 'grpcio-status==1.76.0' 'protobuf==6.33.5' 'grpc-stubs==1.24.11' 'googleapis-common-protos-stubs==2.2.0' \
-  && python3.12 -m pip cache purge
+COPY --from=root pyproject.toml ./pyproject.toml
+
+RUN python3.12 -m pip install -U pip
+
+RUN python3.12 -m pip install --group ci_docs && \
+    python3.12 -m pip cache purge

dev/spark-test-image/lint/Dockerfile

@@ -81,27 +81,10 @@ ENV VIRTUAL_ENV=/opt/spark-venv
 RUN python3.12 -m venv $VIRTUAL_ENV
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
-RUN python3.12 -m pip install \
-    'ruff==0.14.8' \
-    'googleapis-common-protos-stubs==2.2.0' \
-    'grpc-stubs==1.24.11' \
-    'grpcio-status==1.76.0' \
-    'grpcio==1.76.0' \
-    'zstandard==0.25.0' \
-    'ipython' \
-    'matplotlib' \
-    'mypy==1.19.1' \
-    'numpy==2.4.1' \
-    'numpydoc' \
-    'pandas==2.3.3' \
-    'pandas-stubs==2.3.3.260113' \
-    'plotly>=4.8' \
-    'pyarrow>=23.0.0' \
-    'pytest-mypy-plugins==1.9.3' \
-    'pytest==7.1.3' \
-    'scipy>=1.8.0' \
-    'scipy-stubs' \
-    'types-protobuf==6.32.1.20260221' \
-    && python3.12 -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu \
-    && python3.12 -m pip install torcheval \
-    && python3.12 -m pip cache purge
+COPY --from=root pyproject.toml ./pyproject.toml
+
+RUN python3.12 -m pip install -U pip
+
+RUN python3.12 -m pip install --group ml_torch --index-url https://download.pytorch.org/whl/cpu && \
+    python3.12 -m pip install --group ci_lint && \
+    python3.12 -m pip cache purge

pyproject.toml

@@ -255,3 +255,25 @@ ci_classic_minimum = [
     # Debug tools
     "psutil",
 ]
+
+ci_lint = [
+    {include-group = "_lint"},
+    {include-group = "ml_extra_base"},
+    {include-group = "pandas_on_spark_extra"},
+    {include-group = "_ci_connect"},
+    "pyarrow>=23.0.0",
+    "pandas==2.3.3",
+    "pandas-stubs==2.3.3.260113",
+    "ipython"
+]
+
+ci_docs = [
+    {include-group = "_docs"},
+    {include-group = "_ci_connect"},
+    {include-group = "_numpy"},
+    # These are required for sphinx to introspect the code
+    "pyarrow>=23.0.0",
+    "pandas==2.3.3",
+    # plotly is used for sphinx_plotly_directive examples
+    "plotly>=4.8"
+]

sql/connect/common/src/main/protobuf/README.md

@@ -14,10 +14,17 @@ reproducible environment.
 
 ### Build the image
 
+From the root of the Spark repository:
+
 ```bash
-docker build -t connect-cg dev/spark-test-image/connect-gen-protos/
+docker build -t connect-cg \
+  --build-context root=. \
+  dev/spark-test-image/connect-gen-protos/
 ```
 
+The `root` build context points at the Spark repository root so the Dockerfile
+can `COPY` `pyproject.toml` for dependency installation.
+
 ### Run the image
 
 From the root of the Spark repository: