Run selective parallel CI on 8-vCPU Blacksmith runners

.github/actionlint.yaml

@@ -0,0 +1,4 @@
+self-hosted-runner:
+  labels:
+    - blacksmith-8vcpu-ubuntu-2404
+    - blacksmith-8vcpu-ubuntu-2404-arm

.github/actions/setup-rust-ci/action.yml

@@ -0,0 +1,55 @@
+name: Set up Rust CI
+description: Build frontend assets and prepare the pinned Rust toolchain and Kafka dependencies.
+inputs:
+  cache-key:
+    description: Separate compiler artifacts by CI workload.
+    required: true
+  tools:
+    description: Cargo tools to install.
+    required: true
+runs:
+  using: composite
+  steps:
+    - uses: actions/setup-node@v4
+      with:
+        node-version: '24'
+        cache: npm
+        cache-dependency-path: frontend/package-lock.json
+    - name: Build frontend
+      shell: bash
+      working-directory: frontend
+      run: npm ci && npm run build
+    - name: Read pinned Rust toolchain
+      id: toolchain
+      shell: bash
+      run: |
+        python3 - <<'PY'
+        import os, tomllib
+        with open('rust-toolchain.toml', 'rb') as source:
+            channel = tomllib.load(source)['toolchain']['channel']
+        with open(os.environ['GITHUB_OUTPUT'], 'a') as output:
+            print(f'channel={channel}', file=output)
+        PY
+    - uses: dtolnay/rust-toolchain@master
+      with:
+        toolchain: ${{ steps.toolchain.outputs.channel }}
+        components: clippy, llvm-tools-preview
+    - uses: Swatinem/rust-cache@v2
+      with:
+        shared-key: ${{ inputs.cache-key }}
+    - uses: taiki-e/install-action@v2
+      with:
+        tool: ${{ inputs.tools }}
+    - name: Install system dependencies
+      shell: bash
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y libsasl2-dev libssl-dev pkg-config build-essential
+    - name: Patch librdkafka CMake compatibility
+      shell: bash
+      run: |
+        cargo fetch --locked
+        RDKAFKA_SYS_DIR=$(find ~/.cargo/registry/src -name 'rdkafka-sys-*' -type d | head -n 1)
+        CMAKE_FILE="$RDKAFKA_SYS_DIR/librdkafka/CMakeLists.txt"
+        test -f "$CMAKE_FILE"
+        sed -i 's/cmake_minimum_required(VERSION 3.2)/cmake_minimum_required(VERSION 3.5)/' "$CMAKE_FILE"

.github/workflows/blacksmith-testbox.yml

@@ -10,7 +10,7 @@ permissions:
   contents: read
 jobs:
   frontend:
-    runs-on: blacksmith-4vcpu-ubuntu-2404
+    runs-on: blacksmith-8vcpu-ubuntu-2404
     timeout-minutes: 60
     steps:
       - name: Begin Testbox

.github/workflows/build-push-edge-debug.yaml

@@ -9,7 +9,7 @@ jobs:
     runs-on: self-hosted
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3

.github/workflows/build-push-edge.yaml

@@ -9,7 +9,7 @@ jobs:
     runs-on: self-hosted
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3

.github/workflows/build.yaml

@@ -4,7 +4,7 @@ on:
   # Temporarily pause automatic builds for the release target matrix.
   workflow_dispatch:
 
-# Cross preserves this absolute workspace path inside its build container.
+# Each job builds the frontend into this absolute path before cargo runs.
 env:
   NEXT_ASSETS_PATH: ${{ github.workspace }}/frontend/dist
 
@@ -13,27 +13,24 @@ jobs:
   build-default:
     name: Build Default ${{matrix.target}}
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 90
     strategy:
       fail-fast: false
       matrix:
         include:
-          # Linux builds - x86_64 native, aarch64 cross-compile
-          - os: ubuntu-latest
+          # Linux builds - both native, matching the release workflow
+          - os: blacksmith-8vcpu-ubuntu-2404
             target: x86_64-unknown-linux-gnu
-            use_cross: false
-          - os: ubuntu-latest
+          - os: blacksmith-8vcpu-ubuntu-2404-arm
             target: aarch64-unknown-linux-gnu
-            use_cross: true
 
           # macOS build - aarch64 only (M1/M2)
           - os: macos-latest
             target: aarch64-apple-darwin
-            use_cross: false
 
           # Windows build
           - os: windows-latest
             target: x86_64-pc-windows-msvc
-            use_cross: false
 
     steps:
       - uses: actions/checkout@v4
@@ -49,9 +46,17 @@ jobs:
         working-directory: frontend
         run: npm ci && npm run build
 
+      - name: Read pinned Rust toolchain
+        id: rust-toolchain
+        shell: bash
+        run: |
+          channel=$(sed -n 's/^channel *= *"\(.*\)"/\1/p' rust-toolchain.toml 2>/dev/null || true)
+          echo "channel=${channel:-stable}" >> "$GITHUB_OUTPUT"
+
       - name: Setup Rust toolchain
-        uses: dtolnay/rust-toolchain@stable
+        uses: dtolnay/rust-toolchain@master
         with:
+          toolchain: ${{ steps.rust-toolchain.outputs.channel }}
           targets: ${{ matrix.target }}
 
       - name: Install GCC 11 on Linux
@@ -60,27 +65,12 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y gcc-11 g++-11
 
-      - name: Install cross
-        if: matrix.use_cross
-        run: cargo install cross --git https://github.com/cross-rs/cross --locked
-
-      - name: Cache dependencies
-        uses: actions/cache@v4
+      - name: Cache Rust build
+        uses: Swatinem/rust-cache@v2
         with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            target
-          key: ${{ runner.os }}-cargo-${{ matrix.target }}-default-${{ hashFiles('**/Cargo.lock') }}
-
-      - name: Build with cross
-        if: matrix.use_cross
-        env:
-          CROSS_NO_WARNINGS: "0"
-        run: cross build --target ${{ matrix.target }} --release
+          key: ${{ matrix.target }}-default-release
 
       - name: Build native
-        if: ${{ !matrix.use_cross }}
         env:
           CC: ${{ runner.os == 'Linux' && 'gcc-11' || '' }}
           CXX: ${{ runner.os == 'Linux' && 'g++-11' || '' }}
@@ -89,7 +79,8 @@ jobs:
   # Kafka build for x86_64 Linux only
   build-kafka:
     name: Build Kafka x86_64-unknown-linux-gnu
-    runs-on: ubuntu-latest
+    runs-on: blacksmith-8vcpu-ubuntu-2404
+    timeout-minutes: 90
     steps:
       - uses: actions/checkout@v4
 
@@ -121,19 +112,23 @@ jobs:
             libsasl2-dev \
             python3
 
+      - name: Read pinned Rust toolchain
+        id: rust-toolchain
+        shell: bash
+        run: |
+          channel=$(sed -n 's/^channel *= *"\(.*\)"/\1/p' rust-toolchain.toml 2>/dev/null || true)
+          echo "channel=${channel:-stable}" >> "$GITHUB_OUTPUT"
+
       - name: Setup Rust toolchain
-        uses: dtolnay/rust-toolchain@stable
+        uses: dtolnay/rust-toolchain@master
         with:
+          toolchain: ${{ steps.rust-toolchain.outputs.channel }}
           targets: x86_64-unknown-linux-gnu
 
-      - name: Cache dependencies
-        uses: actions/cache@v4
+      - name: Cache Rust build
+        uses: Swatinem/rust-cache@v2
         with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            target
-          key: ${{ runner.os }}-cargo-x86_64-unknown-linux-gnu-kafka-${{ hashFiles('**/Cargo.lock') }}
+          key: x86_64-unknown-linux-gnu-kafka-release
 
       - name: Find and fix librdkafka CMakeLists.txt
         run: |
@@ -163,6 +158,7 @@ jobs:
   build-kafka-mac:
     name: Build Kafka aarch64-apple-darwin
     runs-on: macos-latest
+    timeout-minutes: 120
     steps:
       - uses: actions/checkout@v4
 
@@ -188,19 +184,23 @@ jobs:
             cyrus-sasl \
             python3
 
+      - name: Read pinned Rust toolchain
+        id: rust-toolchain
+        shell: bash
+        run: |
+          channel=$(sed -n 's/^channel *= *"\(.*\)"/\1/p' rust-toolchain.toml 2>/dev/null || true)
+          echo "channel=${channel:-stable}" >> "$GITHUB_OUTPUT"
+
       - name: Setup Rust toolchain
-        uses: dtolnay/rust-toolchain@stable
+        uses: dtolnay/rust-toolchain@master
         with:
+          toolchain: ${{ steps.rust-toolchain.outputs.channel }}
           targets: aarch64-apple-darwin
 
-      - name: Cache dependencies
-        uses: actions/cache@v4
+      - name: Cache Rust build
+        uses: Swatinem/rust-cache@v2
         with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            target
-          key: ${{ runner.os }}-cargo-aarch64-apple-darwin-kafka-${{ hashFiles('**/Cargo.lock') }}
+          key: aarch64-apple-darwin-kafka-release
 
       - name: Find and fix librdkafka CMakeLists.txt
         run: |

.github/workflows/ci.yaml

@@ -0,0 +1,110 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+  schedule:
+    - cron: '17 6 * * *'
+  workflow_dispatch:
+    inputs:
+      full:
+        description: Include sustained load tests and Rust coverage
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+  pull-requests: read
+
+concurrency:
+  group: ci-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  changes:
+    runs-on: blacksmith-8vcpu-ubuntu-2404
+    timeout-minutes: 5
+    outputs:
+      frontend: ${{ steps.paths.outputs.frontend }}
+      rust: ${{ steps.paths.outputs.rust }}
+      integration: ${{ steps.paths.outputs.integration }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
+        id: paths
+        with:
+          filters: |
+            frontend:
+              - 'frontend/**'
+              - 'build.rs'
+              - 'build_support/**'
+              - 'scripts/**'
+              - '.github/workflows/frontend.yaml'
+              - '.github/workflows/ci.yaml'
+            rust:
+              - 'src/**'
+              - 'resources/**'
+              - 'Cargo.*'
+              - 'rust-toolchain.toml'
+              - '.cargo/**'
+              - 'build.rs'
+              - 'build_support/**'
+              - 'scripts/**'
+              - '.github/actions/**'
+              - '.github/workflows/coverage.yaml'
+              - '.github/workflows/ci.yaml'
+            integration:
+              - 'src/**'
+              - 'resources/**'
+              - 'Cargo.*'
+              - 'rust-toolchain.toml'
+              - '.cargo/**'
+              - 'build.rs'
+              - 'build_support/**'
+              - 'scripts/**'
+              - '.github/actions/**'
+              - '.github/workflows/ci.yaml'
+              - 'frontend/**'
+              - 'Dockerfile*'
+              - '.dockerignore'
+              - 'docker-compose*'
+              - '.github/workflows/integration-test.yaml'
+
+  frontend:
+    needs: changes
+    if: needs.changes.outputs.frontend == 'true' || github.event_name != 'pull_request'
+    uses: ./.github/workflows/frontend.yaml
+
+  rust:
+    needs: changes
+    if: needs.changes.outputs.rust == 'true' || github.event_name != 'pull_request'
+    uses: ./.github/workflows/coverage.yaml
+    with:
+      coverage: ${{ github.event_name == 'push' || github.event_name == 'schedule' || inputs.full == true }}
+
+  integration:
+    needs: changes
+    if: needs.changes.outputs.integration == 'true' || github.event_name != 'pull_request'
+    uses: ./.github/workflows/integration-test.yaml
+    with:
+      full: ${{ github.event_name == 'push' || github.event_name == 'schedule' || inputs.full == true }}
+
+  ci-ok:
+    name: CI passed
+    if: always()
+    needs: [changes, frontend, rust, integration]
+    runs-on: blacksmith-8vcpu-ubuntu-2404
+    timeout-minutes: 5
+    steps:
+      - name: Check required results
+        env:
+          RESULTS: ${{ toJSON(needs) }}
+        run: |
+          python3 - <<'PY'
+          import json, os
+          jobs = json.loads(os.environ['RESULTS'])
+          assert jobs['changes']['result'] == 'success', 'Change detection failed'
+          for name, job in jobs.items():
+              assert job['result'] in ('success', 'skipped'), f"{name}: {job['result']}"
+          PY