From a4ed88ef744c7db945810a98c3f00553511220e8 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 27 Jul 2026 13:37:33 -0700 Subject: [PATCH 1/3] Clean up the dependency list for lint/docs/gen-protos docker image --- .../connect-gen-protos/Dockerfile | 6 +---- dev/spark-test-image/docs/Dockerfile | 7 ++--- dev/spark-test-image/lint/Dockerfile | 27 +++---------------- pyproject.toml | 22 +++++++++++++++ 4 files changed, 28 insertions(+), 34 deletions(-) diff --git a/dev/spark-test-image/connect-gen-protos/Dockerfile b/dev/spark-test-image/connect-gen-protos/Dockerfile index 5e90254f81670..831b266191ddd 100644 --- a/dev/spark-test-image/connect-gen-protos/Dockerfile +++ b/dev/spark-test-image/connect-gen-protos/Dockerfile @@ -51,11 +51,7 @@ ENV VIRTUAL_ENV=/opt/spark-venv RUN python3.12 -m venv $VIRTUAL_ENV ENV PATH="$VIRTUAL_ENV/bin:$PATH" -# Keep these versions in sync with the pinned versions in pyproject.toml -RUN python3.12 -m pip install \ - 'mypy==1.19.1' \ - 'mypy-protobuf==3.3.0' \ - 'ruff==0.14.8' +RUN python3.12 -m pip install --group ci_lint # Mount the Spark repo at /spark WORKDIR /spark diff --git a/dev/spark-test-image/docs/Dockerfile b/dev/spark-test-image/docs/Dockerfile index b88036916f2c0..2053a26f19a8b 100644 --- a/dev/spark-test-image/docs/Dockerfile +++ b/dev/spark-test-image/docs/Dockerfile @@ -90,8 +90,5 @@ RUN python3.12 -m venv $VIRTUAL_ENV ENV PATH="$VIRTUAL_ENV/bin:$PATH" # See 'docutils<0.18.0' in SPARK-39421 -RUN python3.12 -m pip install 'sphinx==8.2.3' mkdocs 'pydata_sphinx_theme>=0.13' sphinx-copybutton nbsphinx numpydoc markupsafe \ - ipython sphinx_plotly_directive 'numpy>=1.23.2' 'pyarrow>=23.0.0' 'pandas==2.3.3' 'plotly>=4.8' 'docutils' \ - 'mypy==1.19.1' 'pytest==7.1.3' 'pytest-mypy-plugins==1.9.3' 'ruff==0.14.8' \ - 'pandas-stubs==1.2.0.53' 'grpcio==1.76.0' 'grpcio-status==1.76.0' 'protobuf==6.33.5' 'grpc-stubs==1.24.11' 'googleapis-common-protos-stubs==2.2.0' \ - && python3.12 -m pip cache purge +RUN python3.12 -m pip install --group ci_docs && \ + python3.12 -m pip cache purge diff --git a/dev/spark-test-image/lint/Dockerfile b/dev/spark-test-image/lint/Dockerfile index a6c7e48d48a11..ff5edcb2b3adb 100644 --- a/dev/spark-test-image/lint/Dockerfile +++ b/dev/spark-test-image/lint/Dockerfile @@ -81,27 +81,6 @@ ENV VIRTUAL_ENV=/opt/spark-venv RUN python3.12 -m venv $VIRTUAL_ENV ENV PATH="$VIRTUAL_ENV/bin:$PATH" -RUN python3.12 -m pip install \ - 'ruff==0.14.8' \ - 'googleapis-common-protos-stubs==2.2.0' \ - 'grpc-stubs==1.24.11' \ - 'grpcio-status==1.76.0' \ - 'grpcio==1.76.0' \ - 'zstandard==0.25.0' \ - 'ipython' \ - 'matplotlib' \ - 'mypy==1.19.1' \ - 'numpy==2.4.1' \ - 'numpydoc' \ - 'pandas==2.3.3' \ - 'pandas-stubs==2.3.3.260113' \ - 'plotly>=4.8' \ - 'pyarrow>=23.0.0' \ - 'pytest-mypy-plugins==1.9.3' \ - 'pytest==7.1.3' \ - 'scipy>=1.8.0' \ - 'scipy-stubs' \ - 'types-protobuf==6.32.1.20260221' \ - && python3.12 -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu \ - && python3.12 -m pip install torcheval \ - && python3.12 -m pip cache purge +RUN python3.12 -m pip install --group ml_torch --index-url https://download.pytorch.org/whl/cpu && \ + python3.12 -m pip install --group ci_lint && \ + python3.12 -m pip cache purge diff --git a/pyproject.toml b/pyproject.toml index 8ef23247ceba5..2e8df5291941d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -255,3 +255,25 @@ ci_classic_minimum = [ # Debug tools "psutil", ] + +ci_lint = [ + {include-group = "_lint"}, + {include-group = "ml_extra_base"}, + {include-group = "pandas_on_spark_extra"}, + {include-group = "_ci_connect"}, + "pyarrow>=23.0.0", + "pandas==2.3.3", + "pandas-stubs==2.3.3.260113", + "ipython" +] + +ci_docs = [ + {include-group = "_docs"}, + {include-group = "_ci_connect"}, + {include-group = "_numpy"}, + # These are required for sphinx to introspect the code + "pyarrow>=23.0.0", + "pandas==2.3.3", + # plotly is used for sphinx_plotly_directive examples + "plotly>=4.8" +] \ No newline at end of file From 45ae576cb3dabd3985cded2e86efaa1c2bfe98b2 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 27 Jul 2026 13:38:48 -0700 Subject: [PATCH 2/3] Add blank line --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2e8df5291941d..35de692cf5b0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -276,4 +276,4 @@ ci_docs = [ "pandas==2.3.3", # plotly is used for sphinx_plotly_directive examples "plotly>=4.8" -] \ No newline at end of file +] From 13afed16ca2c0683e72b65ea6c0a8844078a686e Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 27 Jul 2026 13:58:32 -0700 Subject: [PATCH 3/3] Update github actions and docs --- .github/workflows/build_and_test.yml | 4 ++++ .github/workflows/build_infra_images_cache.yml | 4 ++++ dev/spark-test-image/connect-gen-protos/Dockerfile | 4 ++++ dev/spark-test-image/docs/Dockerfile | 5 ++++- dev/spark-test-image/lint/Dockerfile | 4 ++++ sql/connect/common/src/main/protobuf/README.md | 9 ++++++++- 6 files changed, 28 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 8e2ea2884d1ee..6bcd93d5340e4 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -490,6 +490,8 @@ jobs: uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f with: context: ./dev/spark-test-image/docs/ + build-contexts: | + root=./ push: true tags: | ${{ needs.precondition.outputs.image_docs_url }} @@ -501,6 +503,8 @@ jobs: uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f with: context: ./dev/spark-test-image/lint/ + build-contexts: | + root=./ push: true tags: | ${{ needs.precondition.outputs.image_lint_url }} diff --git a/.github/workflows/build_infra_images_cache.yml b/.github/workflows/build_infra_images_cache.yml index 9800107fc369c..d5063f8db9d2b 100644 --- a/.github/workflows/build_infra_images_cache.yml +++ b/.github/workflows/build_infra_images_cache.yml @@ -75,6 +75,8 @@ jobs: uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f with: context: ./dev/spark-test-image/docs/ + build-contexts: | + root=./ push: true tags: ghcr.io/apache/spark/apache-spark-github-action-image-docs-cache:${{ github.ref_name }}-static cache-from: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-docs-cache:${{ github.ref_name }} @@ -89,6 +91,8 @@ jobs: uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f with: context: ./dev/spark-test-image/lint/ + build-contexts: | + root=./ push: true tags: ghcr.io/apache/spark/apache-spark-github-action-image-lint-cache:${{ github.ref_name }}-static cache-from: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-lint-cache:${{ github.ref_name }} diff --git a/dev/spark-test-image/connect-gen-protos/Dockerfile b/dev/spark-test-image/connect-gen-protos/Dockerfile index 831b266191ddd..a0fa39991a9ab 100644 --- a/dev/spark-test-image/connect-gen-protos/Dockerfile +++ b/dev/spark-test-image/connect-gen-protos/Dockerfile @@ -51,6 +51,10 @@ ENV VIRTUAL_ENV=/opt/spark-venv RUN python3.12 -m venv $VIRTUAL_ENV ENV PATH="$VIRTUAL_ENV/bin:$PATH" +COPY --from=root pyproject.toml ./pyproject.toml + +RUN python3.12 -m pip install -U pip + RUN python3.12 -m pip install --group ci_lint # Mount the Spark repo at /spark diff --git a/dev/spark-test-image/docs/Dockerfile b/dev/spark-test-image/docs/Dockerfile index 2053a26f19a8b..d50201dbbb8b6 100644 --- a/dev/spark-test-image/docs/Dockerfile +++ b/dev/spark-test-image/docs/Dockerfile @@ -89,6 +89,9 @@ ENV VIRTUAL_ENV=/opt/spark-venv RUN python3.12 -m venv $VIRTUAL_ENV ENV PATH="$VIRTUAL_ENV/bin:$PATH" -# See 'docutils<0.18.0' in SPARK-39421 +COPY --from=root pyproject.toml ./pyproject.toml + +RUN python3.12 -m pip install -U pip + RUN python3.12 -m pip install --group ci_docs && \ python3.12 -m pip cache purge diff --git a/dev/spark-test-image/lint/Dockerfile b/dev/spark-test-image/lint/Dockerfile index ff5edcb2b3adb..3864832e7850c 100644 --- a/dev/spark-test-image/lint/Dockerfile +++ b/dev/spark-test-image/lint/Dockerfile @@ -81,6 +81,10 @@ ENV VIRTUAL_ENV=/opt/spark-venv RUN python3.12 -m venv $VIRTUAL_ENV ENV PATH="$VIRTUAL_ENV/bin:$PATH" +COPY --from=root pyproject.toml ./pyproject.toml + +RUN python3.12 -m pip install -U pip + RUN python3.12 -m pip install --group ml_torch --index-url https://download.pytorch.org/whl/cpu && \ python3.12 -m pip install --group ci_lint && \ python3.12 -m pip cache purge diff --git a/sql/connect/common/src/main/protobuf/README.md b/sql/connect/common/src/main/protobuf/README.md index 64b4285995937..f0d227e50a572 100644 --- a/sql/connect/common/src/main/protobuf/README.md +++ b/sql/connect/common/src/main/protobuf/README.md @@ -14,10 +14,17 @@ reproducible environment. ### Build the image +From the root of the Spark repository: + ```bash -docker build -t connect-cg dev/spark-test-image/connect-gen-protos/ +docker build -t connect-cg \ + --build-context root=. \ + dev/spark-test-image/connect-gen-protos/ ``` +The `root` build context points at the Spark repository root so the Dockerfile +can `COPY` `pyproject.toml` for dependency installation. + ### Run the image From the root of the Spark repository: