python-packages.yml

name: Python Packages (test + publish)

on:
  push:
    branches: [master]
    paths:
      - 'packages/**'
      - '.github/workflows/python-packages.yml'
      - '.github/scripts/changed_packages.py'
  pull_request:
    branches: [master]
    paths:
      - 'packages/**'
      - '.github/workflows/python-packages.yml'
      - '.github/scripts/changed_packages.py'
  release:
    types: [published]

# Cancel stale CI runs from earlier pushes on the same branch. Skip on
# release events — those are one-shot and must complete.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name != 'release' }}

jobs:
  # ─── Discover affected packages from the diff ────────────────────────────
  # Only the packages that changed (or that depend on a changed package) get
  # tested. Workflow-file changes and release events expand to "all packages".
  discover:
    runs-on: ubuntu-latest
    outputs:
      packages: ${{ steps.affected.outputs.packages }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0      # need history for git diff

      - id: affected
        name: Compute affected package list
        env:
          GHA_EVENT: ${{ github.event_name }}
          PR_BASE: ${{ github.base_ref }}
          PUSH_BEFORE: ${{ github.event.before }}
        run: |
          set -euo pipefail
          # Pick BASE_REF based on event type. Empty / null SHA => --all.
          if [ "$GHA_EVENT" = "pull_request" ]; then
            BASE_REF="origin/${PR_BASE}"
          elif [ "$GHA_EVENT" = "push" ]; then
            BASE_REF="${PUSH_BEFORE}"
          else
            BASE_REF=""
          fi
          if [ "$GHA_EVENT" = "release" ] \
             || [ -z "${BASE_REF}" ] \
             || [ "${BASE_REF}" = "0000000000000000000000000000000000000000" ]; then
            pkgs=$(python3 .github/scripts/changed_packages.py --all)
          else
            export BASE_REF
            pkgs=$(python3 .github/scripts/changed_packages.py)
          fi
          echo "Affected packages: $pkgs"
          echo "packages=$pkgs" >> "$GITHUB_OUTPUT"

  # ─── Tests run only for affected packages ────────────────────────────────
  test:
    needs: discover
    if: needs.discover.outputs.packages != '[]'
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ['3.11', '3.12']
        package: ${{ fromJSON(needs.discover.outputs.packages) }}

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          # No `cache: pip` — we install via uv, so pip's cache dir
          # (~/.cache/pip) is never populated. setup-uv handles its own
          # cache below.

      - name: Set up uv
        uses: astral-sh/setup-uv@v3
        with:
          enable-cache: true
          cache-dependency-glob: 'packages/**/pyproject.toml'

      # Install every in-repo MIDAS package (excluding the hidden ones) as
      # editable in a single uv resolve. uv resolves the entire dep graph
      # in one pass, replacing the 18 sequential conditional pip-install
      # steps the old workflow had.
      - name: Install all in-repo MIDAS packages (editable, single uv resolve)
        run: |
          set -euo pipefail
          shopt -s nullglob
          args=()
          for d in packages/midas_*/; do
            name=$(basename "$d")
            case "$name" in
              midas_grain_odf|midas_pf_odf)
                # Hidden from CI / PyPI — skip.
                ;;
              *)
                args+=("-e" "$d")
                ;;
            esac
          done
          uv pip install --system "${args[@]}"

      - name: Install dev extras for ${{ matrix.package }}
        run: uv pip install --system -e "packages/${{ matrix.package }}[dev]"

      - name: Run tests for ${{ matrix.package }}
        if: matrix.package != 'midas_suite'
        run: |
          cd packages/${{ matrix.package }}
          # `slow`-marked tests (midas-index regression vs C IndexerOMP)
          # depend on the C binaries which CI doesn't build; skipif handles
          # that, but we exclude them explicitly here for speed.
          python -m pytest tests/ -v --tb=short -m "not slow"

      # midas_suite is a meta-package with no test suite. Run an import +
      # installed() smoke test instead, which verifies (a) the package
      # imports, (b) every declared sub-package was resolved, and (c) the
      # SUBPACKAGES tuple matches what's actually importable.
      - name: Smoke test midas_suite
        if: matrix.package == 'midas_suite'
        run: |
          python -c "
          import midas_suite
          status = midas_suite.installed()
          print(f'midas-suite {midas_suite.__version__} declares {len(midas_suite.SUBPACKAGES)} sub-packages:')
          missing = []
          for name, ver in status.items():
              flag = 'OK ' if ver else 'MISS'
              print(f'  {flag}  {name:30s}  {ver if ver else \"NOT INSTALLED\"}')
              if not ver:
                  missing.append(name)
          if missing:
              raise SystemExit(f'FAIL: {len(missing)} sub-package(s) failed to import: {missing}')
          print('All sub-packages resolved.')
          "

  # ─── Build + publish: triggered by GitHub release only ────────────────────
  # The tag name selects which package to publish:
  #   midas-stress-v*           → midas_stress
  #   midas-params-v*           → midas_params
  #   midas-peakfit-v*          → midas_peakfit
  #   midas-integrate-v*        → midas_integrate
  #   midas-hkls-v*             → midas_hkls
  #   midas-calibrate-v*        → midas_calibrate
  #   midas-diffract-v*         → midas_diffract
  #   midas-index-v*            → midas_index
  #   midas-transforms-v*       → midas_transforms
  #   midas-nf-preprocess-v*    → midas_nf_preprocess
  #   midas-process-grains-v*   → midas_process_grains
  #   midas-fit-grain-v*        → midas_fit_grain
  #   midas-nf-fitorientation-v* → midas_nf_fitorientation
  #   midas-nf-pipeline-v*      → midas_nf_pipeline
  #   midas-parsl-configs-v*    → midas_parsl_configs
  #   midas-ff-pipeline-v*      → midas_ff_pipeline
  #   midas-suite-v*            → midas_suite (meta-package)
  #
  # Release tags without one of these prefixes are ignored (no-op build).
  build:
    needs: test
    runs-on: ubuntu-latest
    if: github.event_name == 'release'
    outputs:
      package: ${{ steps.which.outputs.package }}
      pypi_name: ${{ steps.which.outputs.pypi_name }}
    steps:
      - uses: actions/checkout@v4

      - id: which
        name: Identify package from tag
        run: |
          TAG="${GITHUB_REF##*/}"
          echo "Tag: $TAG"
          if [[ "$TAG" == midas-stress-v* ]]; then
            echo "package=midas_stress" >> "$GITHUB_OUTPUT"
            echo "pypi_name=midas-stress" >> "$GITHUB_OUTPUT"
          elif [[ "$TAG" == midas-params-v* ]]; then
            echo "package=midas_params" >> "$GITHUB_OUTPUT"
            echo "pypi_name=midas-params" >> "$GITHUB_OUTPUT"
          elif [[ "$TAG" == midas-peakfit-v* ]]; then
            echo "package=midas_peakfit" >> "$GITHUB_OUTPUT"
            echo "pypi_name=midas-peakfit" >> "$GITHUB_OUTPUT"
          elif [[ "$TAG" == midas-integrate-v* ]]; then
            echo "package=midas_integrate" >> "$GITHUB_OUTPUT"
            echo "pypi_name=midas-integrate" >> "$GITHUB_OUTPUT"
          elif [[ "$TAG" == midas-hkls-v* ]]; then
            echo "package=midas_hkls" >> "$GITHUB_OUTPUT"
            echo "pypi_name=midas-hkls" >> "$GITHUB_OUTPUT"
          elif [[ "$TAG" == midas-calibrate-v* ]]; then
            echo "package=midas_calibrate" >> "$GITHUB_OUTPUT"
            echo "pypi_name=midas-calibrate" >> "$GITHUB_OUTPUT"
          elif [[ "$TAG" == midas-diffract-v* ]]; then
            echo "package=midas_diffract" >> "$GITHUB_OUTPUT"
            echo "pypi_name=midas-diffract" >> "$GITHUB_OUTPUT"
          elif [[ "$TAG" == midas-index-v* ]]; then
            echo "package=midas_index" >> "$GITHUB_OUTPUT"
            echo "pypi_name=midas-index" >> "$GITHUB_OUTPUT"
          elif [[ "$TAG" == midas-transforms-v* ]]; then
            echo "package=midas_transforms" >> "$GITHUB_OUTPUT"
            echo "pypi_name=midas-transforms" >> "$GITHUB_OUTPUT"
          elif [[ "$TAG" == midas-nf-preprocess-v* ]]; then
            echo "package=midas_nf_preprocess" >> "$GITHUB_OUTPUT"
            echo "pypi_name=midas-nf-preprocess" >> "$GITHUB_OUTPUT"
          elif [[ "$TAG" == midas-process-grains-v* ]]; then
            echo "package=midas_process_grains" >> "$GITHUB_OUTPUT"
            echo "pypi_name=midas-process-grains" >> "$GITHUB_OUTPUT"
          elif [[ "$TAG" == midas-fit-grain-v* ]]; then
            echo "package=midas_fit_grain" >> "$GITHUB_OUTPUT"
            echo "pypi_name=midas-fit-grain" >> "$GITHUB_OUTPUT"
          elif [[ "$TAG" == midas-nf-fitorientation-v* ]]; then
            echo "package=midas_nf_fitorientation" >> "$GITHUB_OUTPUT"
            echo "pypi_name=midas-nf-fitorientation" >> "$GITHUB_OUTPUT"
          elif [[ "$TAG" == midas-nf-pipeline-v* ]]; then
            echo "package=midas_nf_pipeline" >> "$GITHUB_OUTPUT"
            echo "pypi_name=midas-nf-pipeline" >> "$GITHUB_OUTPUT"
          elif [[ "$TAG" == midas-parsl-configs-v* ]]; then
            echo "package=midas_parsl_configs" >> "$GITHUB_OUTPUT"
            echo "pypi_name=midas-parsl-configs" >> "$GITHUB_OUTPUT"
          elif [[ "$TAG" == midas-ff-pipeline-v* ]]; then
            echo "package=midas_ff_pipeline" >> "$GITHUB_OUTPUT"
            echo "pypi_name=midas-ff-pipeline" >> "$GITHUB_OUTPUT"
          elif [[ "$TAG" == midas-suite-v* ]]; then
            echo "package=midas_suite" >> "$GITHUB_OUTPUT"
            echo "pypi_name=midas-suite" >> "$GITHUB_OUTPUT"
          else
            echo "Tag $TAG does not match any known package prefix; skipping."
            echo "package=" >> "$GITHUB_OUTPUT"
            echo "pypi_name=" >> "$GITHUB_OUTPUT"
          fi

      - name: Set up Python
        if: steps.which.outputs.package != ''
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Install build tools
        if: steps.which.outputs.package != ''
        run: pip install build

      - name: Build sdist and wheel
        if: steps.which.outputs.package != ''
        run: |
          cd packages/${{ steps.which.outputs.package }}
          python -m build

      - name: Upload artifacts
        if: steps.which.outputs.package != ''
        uses: actions/upload-artifact@v4
        with:
          name: ${{ steps.which.outputs.pypi_name }}-dist
          path: packages/${{ steps.which.outputs.package }}/dist/

  publish:
    needs: build
    runs-on: ubuntu-latest
    if: github.event_name == 'release' && needs.build.outputs.package != ''
    environment: pypi
    permissions:
      id-token: write  # for trusted publishing (OIDC)

    steps:
      - name: Download artifacts
        uses: actions/download-artifact@v4
        with:
          name: ${{ needs.build.outputs.pypi_name }}-dist
          path: dist/

      - name: Publish to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        # Uses trusted publishing (OIDC). One-time setup per package at pypi.org:
        # Settings > Publishing > Add a new pending publisher
        #   Owner: marinerhemant
        #   Repo: MIDAS
        #   Workflow: python-packages.yml
        #   Environment: pypi
        # The same GitHub environment (pypi) is shared across packages;
        # the PyPI side identifies which project to publish to by the
        # PyPI project name baked into the uploaded dist/*.