diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 000000000..055e04ef6 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,4 @@ +self-hosted-runner: + labels: + - blacksmith-8vcpu-ubuntu-2404 + - blacksmith-8vcpu-ubuntu-2404-arm diff --git a/.github/actions/setup-rust-ci/action.yml b/.github/actions/setup-rust-ci/action.yml new file mode 100644 index 000000000..76d580e24 --- /dev/null +++ b/.github/actions/setup-rust-ci/action.yml @@ -0,0 +1,55 @@ +name: Set up Rust CI +description: Build frontend assets and prepare the pinned Rust toolchain and Kafka dependencies. +inputs: + cache-key: + description: Separate compiler artifacts by CI workload. + required: true + tools: + description: Cargo tools to install. + required: true +runs: + using: composite + steps: + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: npm + cache-dependency-path: frontend/package-lock.json + - name: Build frontend + shell: bash + working-directory: frontend + run: npm ci && npm run build + - name: Read pinned Rust toolchain + id: toolchain + shell: bash + run: | + python3 - <<'PY' + import os, tomllib + with open('rust-toolchain.toml', 'rb') as source: + channel = tomllib.load(source)['toolchain']['channel'] + with open(os.environ['GITHUB_OUTPUT'], 'a') as output: + print(f'channel={channel}', file=output) + PY + - uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ steps.toolchain.outputs.channel }} + components: clippy, llvm-tools-preview + - uses: Swatinem/rust-cache@v2 + with: + shared-key: ${{ inputs.cache-key }} + - uses: taiki-e/install-action@v2 + with: + tool: ${{ inputs.tools }} + - name: Install system dependencies + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y libsasl2-dev libssl-dev pkg-config build-essential + - name: Patch librdkafka CMake compatibility + shell: bash + run: | + cargo fetch --locked + RDKAFKA_SYS_DIR=$(find ~/.cargo/registry/src -name 'rdkafka-sys-*' -type d | head -n 1) + CMAKE_FILE="$RDKAFKA_SYS_DIR/librdkafka/CMakeLists.txt" + test -f "$CMAKE_FILE" + sed -i 's/cmake_minimum_required(VERSION 3.2)/cmake_minimum_required(VERSION 3.5)/' "$CMAKE_FILE" diff --git a/.github/workflows/blacksmith-testbox.yml b/.github/workflows/blacksmith-testbox.yml index 56bbc5847..2527d614d 100644 --- a/.github/workflows/blacksmith-testbox.yml +++ b/.github/workflows/blacksmith-testbox.yml @@ -10,7 +10,7 @@ permissions: contents: read jobs: frontend: - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 60 steps: - name: Begin Testbox diff --git a/.github/workflows/build-push-edge-debug.yaml b/.github/workflows/build-push-edge-debug.yaml index 075c31358..0b28a350b 100644 --- a/.github/workflows/build-push-edge-debug.yaml +++ b/.github/workflows/build-push-edge-debug.yaml @@ -9,7 +9,7 @@ jobs: runs-on: self-hosted steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v3 diff --git a/.github/workflows/build-push-edge.yaml b/.github/workflows/build-push-edge.yaml index cd787c45a..fd950605b 100644 --- a/.github/workflows/build-push-edge.yaml +++ b/.github/workflows/build-push-edge.yaml @@ -9,7 +9,7 @@ jobs: runs-on: self-hosted steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v3 diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index daf6ba6ff..6a38f28cd 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -4,7 +4,7 @@ on: # Temporarily pause automatic builds for the release target matrix. workflow_dispatch: -# Cross preserves this absolute workspace path inside its build container. +# Each job builds the frontend into this absolute path before cargo runs. env: NEXT_ASSETS_PATH: ${{ github.workspace }}/frontend/dist @@ -13,27 +13,24 @@ jobs: build-default: name: Build Default ${{matrix.target}} runs-on: ${{ matrix.os }} + timeout-minutes: 90 strategy: fail-fast: false matrix: include: - # Linux builds - x86_64 native, aarch64 cross-compile - - os: ubuntu-latest + # Linux builds - both native, matching the release workflow + - os: blacksmith-8vcpu-ubuntu-2404 target: x86_64-unknown-linux-gnu - use_cross: false - - os: ubuntu-latest + - os: blacksmith-8vcpu-ubuntu-2404-arm target: aarch64-unknown-linux-gnu - use_cross: true # macOS build - aarch64 only (M1/M2) - os: macos-latest target: aarch64-apple-darwin - use_cross: false # Windows build - os: windows-latest target: x86_64-pc-windows-msvc - use_cross: false steps: - uses: actions/checkout@v4 @@ -49,9 +46,17 @@ jobs: working-directory: frontend run: npm ci && npm run build + - name: Read pinned Rust toolchain + id: rust-toolchain + shell: bash + run: | + channel=$(sed -n 's/^channel *= *"\(.*\)"/\1/p' rust-toolchain.toml 2>/dev/null || true) + echo "channel=${channel:-stable}" >> "$GITHUB_OUTPUT" + - name: Setup Rust toolchain - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@master with: + toolchain: ${{ steps.rust-toolchain.outputs.channel }} targets: ${{ matrix.target }} - name: Install GCC 11 on Linux @@ -60,27 +65,12 @@ jobs: sudo apt-get update sudo apt-get install -y gcc-11 g++-11 - - name: Install cross - if: matrix.use_cross - run: cargo install cross --git https://github.com/cross-rs/cross --locked - - - name: Cache dependencies - uses: actions/cache@v4 + - name: Cache Rust build + uses: Swatinem/rust-cache@v2 with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-${{ matrix.target }}-default-${{ hashFiles('**/Cargo.lock') }} - - - name: Build with cross - if: matrix.use_cross - env: - CROSS_NO_WARNINGS: "0" - run: cross build --target ${{ matrix.target }} --release + key: ${{ matrix.target }}-default-release - name: Build native - if: ${{ !matrix.use_cross }} env: CC: ${{ runner.os == 'Linux' && 'gcc-11' || '' }} CXX: ${{ runner.os == 'Linux' && 'g++-11' || '' }} @@ -89,7 +79,8 @@ jobs: # Kafka build for x86_64 Linux only build-kafka: name: Build Kafka x86_64-unknown-linux-gnu - runs-on: ubuntu-latest + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 90 steps: - uses: actions/checkout@v4 @@ -121,19 +112,23 @@ jobs: libsasl2-dev \ python3 + - name: Read pinned Rust toolchain + id: rust-toolchain + shell: bash + run: | + channel=$(sed -n 's/^channel *= *"\(.*\)"/\1/p' rust-toolchain.toml 2>/dev/null || true) + echo "channel=${channel:-stable}" >> "$GITHUB_OUTPUT" + - name: Setup Rust toolchain - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@master with: + toolchain: ${{ steps.rust-toolchain.outputs.channel }} targets: x86_64-unknown-linux-gnu - - name: Cache dependencies - uses: actions/cache@v4 + - name: Cache Rust build + uses: Swatinem/rust-cache@v2 with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-x86_64-unknown-linux-gnu-kafka-${{ hashFiles('**/Cargo.lock') }} + key: x86_64-unknown-linux-gnu-kafka-release - name: Find and fix librdkafka CMakeLists.txt run: | @@ -163,6 +158,7 @@ jobs: build-kafka-mac: name: Build Kafka aarch64-apple-darwin runs-on: macos-latest + timeout-minutes: 120 steps: - uses: actions/checkout@v4 @@ -188,19 +184,23 @@ jobs: cyrus-sasl \ python3 + - name: Read pinned Rust toolchain + id: rust-toolchain + shell: bash + run: | + channel=$(sed -n 's/^channel *= *"\(.*\)"/\1/p' rust-toolchain.toml 2>/dev/null || true) + echo "channel=${channel:-stable}" >> "$GITHUB_OUTPUT" + - name: Setup Rust toolchain - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@master with: + toolchain: ${{ steps.rust-toolchain.outputs.channel }} targets: aarch64-apple-darwin - - name: Cache dependencies - uses: actions/cache@v4 + - name: Cache Rust build + uses: Swatinem/rust-cache@v2 with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-aarch64-apple-darwin-kafka-${{ hashFiles('**/Cargo.lock') }} + key: aarch64-apple-darwin-kafka-release - name: Find and fix librdkafka CMakeLists.txt run: | diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 000000000..11c9df89b --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,110 @@ +name: CI + +on: + pull_request: + push: + branches: [main] + schedule: + - cron: '17 6 * * *' + workflow_dispatch: + inputs: + full: + description: Include sustained load tests and Rust coverage + type: boolean + default: false + +permissions: + contents: read + pull-requests: read + +concurrency: + group: ci-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + changes: + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 5 + outputs: + frontend: ${{ steps.paths.outputs.frontend }} + rust: ${{ steps.paths.outputs.rust }} + integration: ${{ steps.paths.outputs.integration }} + steps: + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v3 + id: paths + with: + filters: | + frontend: + - 'frontend/**' + - 'build.rs' + - 'build_support/**' + - 'scripts/**' + - '.github/workflows/frontend.yaml' + - '.github/workflows/ci.yaml' + rust: + - 'src/**' + - 'resources/**' + - 'Cargo.*' + - 'rust-toolchain.toml' + - '.cargo/**' + - 'build.rs' + - 'build_support/**' + - 'scripts/**' + - '.github/actions/**' + - '.github/workflows/coverage.yaml' + - '.github/workflows/ci.yaml' + integration: + - 'src/**' + - 'resources/**' + - 'Cargo.*' + - 'rust-toolchain.toml' + - '.cargo/**' + - 'build.rs' + - 'build_support/**' + - 'scripts/**' + - '.github/actions/**' + - '.github/workflows/ci.yaml' + - 'frontend/**' + - 'Dockerfile*' + - '.dockerignore' + - 'docker-compose*' + - '.github/workflows/integration-test.yaml' + + frontend: + needs: changes + if: needs.changes.outputs.frontend == 'true' || github.event_name != 'pull_request' + uses: ./.github/workflows/frontend.yaml + + rust: + needs: changes + if: needs.changes.outputs.rust == 'true' || github.event_name != 'pull_request' + uses: ./.github/workflows/coverage.yaml + with: + coverage: ${{ github.event_name == 'push' || github.event_name == 'schedule' || inputs.full == true }} + + integration: + needs: changes + if: needs.changes.outputs.integration == 'true' || github.event_name != 'pull_request' + uses: ./.github/workflows/integration-test.yaml + with: + full: ${{ github.event_name == 'push' || github.event_name == 'schedule' || inputs.full == true }} + + ci-ok: + name: CI passed + if: always() + needs: [changes, frontend, rust, integration] + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 5 + steps: + - name: Check required results + env: + RESULTS: ${{ toJSON(needs) }} + run: | + python3 - <<'PY' + import json, os + jobs = json.loads(os.environ['RESULTS']) + assert jobs['changes']['result'] == 'success', 'Change detection failed' + for name, job in jobs.items(): + assert job['result'] in ('success', 'skipped'), f"{name}: {job['result']}" + PY diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index 86845cbbe..127c6a669 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -1,100 +1,73 @@ +name: Rust checks and tests on: - pull_request: - paths-ignore: - - "docs/**" - - "helm/**" - - "assets/**" - - "**.md" - -name: Lint, Test and Coverage Report -# Cross preserves this absolute workspace path inside its build container. + workflow_call: + inputs: + coverage: + description: Collect and upload combined default and Kafka coverage. + type: boolean + default: false +permissions: + contents: read env: NEXT_ASSETS_PATH: ${{ github.workspace }}/frontend/dist jobs: community-ui: - runs-on: ubuntu-latest + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 5 steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 22 + node-version: '24' - name: Test community UI overlay run: node --test scripts/test-community-ui.mjs - coverage: - runs-on: ubuntu-latest + clippy: + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 45 steps: - uses: actions/checkout@v4 - - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: '24' - cache: npm - cache-dependency-path: frontend/package-lock.json - - - name: Build frontend - working-directory: frontend - run: npm ci && npm run build - - uses: dtolnay/rust-toolchain@stable - with: - components: clippy - - - uses: Swatinem/rust-cache@v2 - with: - shared-key: ${{ runner.os }}-cargo - - - uses: taiki-e/install-action@v2 + - uses: ./.github/actions/setup-rust-ci with: - tool: cargo-hack, cargo-llvm-cov, nextest - - - name: Install System Dependencies - run: | - sudo apt-get update - sudo apt-get install -y \ - libsasl2-dev \ - libssl-dev \ - pkg-config \ - build-essential - if: runner.os == 'Linux' - - - name: Find and fix librdkafka CMakeLists.txt - run: | - # Download the package first so it's in the registry - cargo fetch - - # Find the rdkafka-sys package directory - RDKAFKA_SYS_DIR=$(find ~/.cargo/registry/src -name "rdkafka-sys-*" -type d | head -n 1) - echo "Found rdkafka-sys at: $RDKAFKA_SYS_DIR" - - # Find the librdkafka CMakeLists.txt file - CMAKE_FILE="$RDKAFKA_SYS_DIR/librdkafka/CMakeLists.txt" - - if [ -f "$CMAKE_FILE" ]; then - echo "Found CMakeLists.txt at: $CMAKE_FILE" - - # Make a backup of the original file - cp "$CMAKE_FILE" "$CMAKE_FILE.bak" - - # Replace the minimum required version - sed -i 's/cmake_minimum_required(VERSION 3.2)/cmake_minimum_required(VERSION 3.5)/' "$CMAKE_FILE" - - echo "Modified CMakeLists.txt - before and after comparison:" - diff "$CMAKE_FILE.bak" "$CMAKE_FILE" || true - else - echo "Could not find librdkafka CMakeLists.txt file!" - exit 1 - fi - - - name: Check with clippy + cache-key: rust-clippy + tools: cargo-hack + - name: Check all feature configurations with Clippy + # cargo-hack temporarily removes dev dependencies and updates Cargo.lock; + # --locked would reject that temporary manifest before Clippy can run. run: cargo hack clippy --verbose --each-feature --no-dev-deps -- -D warnings + tests: + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-rust-ci + with: + cache-key: rust-tests-${{ inputs.coverage && 'coverage' || 'plain' }} + tools: ${{ inputs.coverage && 'cargo-llvm-cov, nextest' || 'nextest' }} - name: Test default feature set - run: cargo hack llvm-cov --no-report nextest - - - name: Test kafka feature - run: cargo hack --features kafka llvm-cov --no-report nextest --filter-expr 'test(kafka)' - - - name: Generate coverage report + if: ${{ !inputs.coverage }} + run: cargo nextest run --locked + - name: Test Kafka feature + if: ${{ !inputs.coverage }} + run: cargo nextest run --locked --features kafka -E 'test(kafka)' + - name: Clear stale workspace coverage + if: inputs.coverage + run: cargo llvm-cov clean --workspace + - name: Test default feature set with coverage + if: inputs.coverage + run: cargo llvm-cov nextest --locked --no-report + - name: Test Kafka feature with coverage + if: inputs.coverage + run: cargo llvm-cov nextest --locked --features kafka --no-report --filter-expr 'test(kafka)' + - name: Generate combined coverage report + if: inputs.coverage run: cargo llvm-cov report --lcov --output-path coverage.lcov + - name: Upload coverage report + if: inputs.coverage + uses: actions/upload-artifact@v4 + with: + name: rust-coverage + path: coverage.lcov + if-no-files-found: error diff --git a/.github/workflows/frontend.yaml b/.github/workflows/frontend.yaml index 4d8ea4363..af005f31a 100644 --- a/.github/workflows/frontend.yaml +++ b/.github/workflows/frontend.yaml @@ -1,21 +1,17 @@ name: Frontend on: + workflow_call: workflow_dispatch: - pull_request: - paths: ['frontend/**', 'build.rs', '.github/workflows/frontend.yaml'] - push: - paths: ['frontend/**', 'build.rs', '.github/workflows/frontend.yaml'] permissions: contents: read +defaults: + run: + working-directory: frontend jobs: - frontend: - runs-on: ubuntu-latest - # Keep screenshot fonts, Chromium and OS identical to the reviewed baselines. - # Update together with @playwright/test in frontend/package-lock.json. - container: mcr.microsoft.com/playwright:v1.63.0-noble - defaults: - run: - working-directory: frontend + checks: + name: Format, build and unit tests + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 15 steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -23,7 +19,10 @@ jobs: node-version: '24' cache: npm cache-dependency-path: frontend/package-lock.json - - run: npm ci && npm run build + - run: npm ci + - run: npm run format:check + - run: npm run build + - run: npm test - run: npm run package if: github.event_name == 'workflow_dispatch' - name: Upload frontend package @@ -35,11 +34,43 @@ jobs: frontend/build.zip frontend/build.zip.sha1 if-no-files-found: error - - run: npm run check + + browser: + name: ${{ matrix.suite }} + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 20 + # Keep screenshot fonts, Chromium and OS identical to the reviewed baselines. + # Update together with @playwright/test in frontend/package-lock.json. + container: mcr.microsoft.com/playwright:v1.63.0-noble + strategy: + fail-fast: false + matrix: + include: + - suite: ui-1 + command: npm run test:ui -- --shard=1/4 --workers=4 + - suite: ui-2 + command: npm run test:ui -- --shard=2/4 --workers=4 + - suite: ui-3 + command: npm run test:ui -- --shard=3/4 --workers=4 + - suite: ui-4 + command: npm run test:ui -- --shard=4/4 --workers=4 + - suite: production + command: npm run test:prod -- --workers=4 + - suite: storybook + command: npm run test:stories + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: npm + cache-dependency-path: frontend/package-lock.json + - run: npm ci + - run: ${{ matrix.command }} - uses: actions/upload-artifact@v4 if: failure() with: - name: frontend-browser-results + name: frontend-browser-results-${{ matrix.suite }} path: | frontend/playwright-report frontend/test-results diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml index 2e63463e7..dbb4dfa01 100644 --- a/.github/workflows/integration-test.yaml +++ b/.github/workflows/integration-test.yaml @@ -1,35 +1,86 @@ name: Integration Tests with Quest on: - pull_request: - paths-ignore: - - "docs/**" - - "helm/**" - - "assets/**" - - "**.md" + workflow_call: + inputs: + full: + description: Include sustained Quest load tests + type: boolean + default: false +permissions: + contents: read jobs: - - docker-compose-test: - name: Quest Smoke and Load Tests for Standalone deployments - runs-on: ubuntu-latest + build-image: + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 30 steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Start compose - run: docker compose -f docker-compose-test.yaml up --build --exit-code-from quest - - name: Stop compose - if: always() - run: docker compose -f docker-compose-test.yaml down -v + - uses: actions/checkout@v4 + - name: Set up cached Docker builder + uses: useblacksmith/setup-docker-builder@v2 + with: + cache-key: parseable-integration + - name: Build integration image once + uses: useblacksmith/build-push-action@v2 + with: + context: . + file: Dockerfile.debug + platforms: linux/amd64 + tags: parseable-ci:${{ github.sha }} + outputs: type=docker,dest=${{ runner.temp }}/parseable-ci.tar + - uses: actions/upload-artifact@v4 + with: + name: integration-image + path: ${{ runner.temp }}/parseable-ci.tar + compression-level: 0 + retention-days: 1 + if-no-files-found: error - docker-compose-distributed-test: - name: Quest Smoke and Load Tests for Distributed deployments - runs-on: ubuntu-latest + quest: + name: Quest ${{ matrix.deployment }} (${{ inputs.full && 'full' || 'smoke' }}) + needs: build-image + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + include: + - deployment: standalone + compose: docker-compose-test.yaml + server: parseable + - deployment: distributed + compose: docker-compose-distributed-test.yaml + server: parseable-query + env: + PARSEABLE_TEST_IMAGE: parseable-ci:${{ github.sha }} + COMPOSE_FILE: ${{ matrix.compose }} + QUEST_MODE: ${{ inputs.full && 'load' || 'smoke' }} steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Start compose - run: docker compose -f docker-compose-distributed-test.yaml up --build --exit-code-from quest + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: integration-image + path: ${{ runner.temp }} + - name: Load shared integration image + run: docker load --input "$RUNNER_TEMP/parseable-ci.tar" + - name: Select functional suite for PRs + if: ${{ !inputs.full }} + run: echo "COMPOSE_FILE=$COMPOSE_FILE:docker-compose-ci-smoke.yaml" >> "$GITHUB_ENV" + - name: Start services and test embedded frontend + env: + SERVER: ${{ matrix.server }} + run: | + docker compose up --no-build --wait --wait-timeout 180 "$SERVER" + docker compose exec -T "$SERVER" sh -ec 'curl --fail --silent --show-error --location http://localhost:8000/next/ | grep -q "Parseable · Observability"' + - name: Run Quest + run: docker compose up --no-build --exit-code-from quest + - name: Capture service logs + if: failure() + run: docker compose logs --no-color > quest-services.log + - uses: actions/upload-artifact@v4 + if: failure() + with: + name: quest-${{ matrix.deployment }}-logs + path: quest-services.log - name: Stop compose if: always() - run: docker compose -f docker-compose-distributed-test.yaml down -v - + run: docker compose down -v diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 092f0518a..4e880df45 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,6 +7,9 @@ # repository, so that job runs only in parseablehq/parseable. # It also creates a checksum file for the artifacts and publishes it to the release page. # It also builds and pushes a Docker image to Docker Hub with the same version tag. +# Linux jobs run on Blacksmith runners. The build-profile input can select the +# experimental release-thin-lto profile; such runs upload workflow artifacts +# only and never publish to the release page or registry. name: Release builds for Linux(64 bit, Arm), Mac(64bit, Arm), Windows and Docker @@ -14,24 +17,39 @@ on: workflow_dispatch: inputs: tag: - description: 'Existing release tag to build and publish (e.g., v3.2.5)' + description: 'Release tag; thin-LTO experiments may also use a branch or commit containing that profile' required: true type: string - -# Cross preserves this absolute workspace path inside its build container. + build-profile: + description: 'Cargo profile (release-thin-lto is experimental and skips publishing)' + required: false + type: choice + default: release-lto + options: + - release-lto + - release-thin-lto + +# Never cancel a release mid-publish; a second run for the same tag waits. +concurrency: + group: release-${{ inputs.tag }} + cancel-in-progress: false + +# Each job builds the frontend into this absolute path before cargo runs. env: NEXT_ASSETS_PATH: ${{ github.workspace }}/frontend/dist + RELEASE_PROFILE: ${{ inputs.build-profile || 'release-lto' }} jobs: build-frontend: name: Package frontend - runs-on: ubuntu-latest + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 20 permissions: contents: write steps: - uses: actions/checkout@v4 with: - ref: refs/tags/${{ inputs.tag }} + ref: ${{ inputs.build-profile == 'release-thin-lto' && inputs.tag || format('refs/tags/{0}', inputs.tag) }} - name: Set up Node.js uses: actions/setup-node@v4 @@ -58,6 +76,7 @@ jobs: if-no-files-found: error - name: Attach frontend to release + if: env.RELEASE_PROFILE == 'release-lto' uses: softprops/action-gh-release@v0.1.15 with: draft: false @@ -81,17 +100,16 @@ jobs: matrix: include: - target: x86_64-unknown-linux-gnu - os: ubuntu-latest - use-cross: false + os: blacksmith-8vcpu-ubuntu-2404 # A hosted runner lost contact during the cross-compiled LTO build, # so aarch64 builds natively on an Arm runner. - target: aarch64-unknown-linux-gnu - os: ubuntu-24.04-arm - use-cross: false + os: blacksmith-8vcpu-ubuntu-2404-arm + timeout-minutes: 90 steps: - uses: actions/checkout@v4 with: - ref: refs/tags/${{ inputs.tag }} + ref: ${{ inputs.build-profile == 'release-thin-lto' && inputs.tag || format('refs/tags/{0}', inputs.tag) }} - name: Set up Node.js uses: actions/setup-node@v4 @@ -104,38 +122,45 @@ jobs: working-directory: frontend run: npm ci && npm run build - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - profile: minimal # minimal component installation (ie, no documentation) - target: ${{matrix.target}} - override: true + - name: Read pinned Rust toolchain + id: rust-toolchain + shell: bash + run: | + channel=$(sed -n 's/^channel *= *"\(.*\)"/\1/p' rust-toolchain.toml 2>/dev/null || true) + echo "channel=${channel:-stable}" >> "$GITHUB_OUTPUT" - - name: Install cross - if: matrix.use-cross - run: cargo install cross --git https://github.com/cross-rs/cross --locked + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ steps.rust-toolchain.outputs.channel }} + targets: ${{ matrix.target }} - - name: Build with cross - if: matrix.use-cross - run: cross build --profile release-lto --target ${{ matrix.target }} + - name: Cache Rust build + uses: Swatinem/rust-cache@v2 + with: + key: ${{ matrix.target }}-default-${{ env.RELEASE_PROFILE }} # The aarch64 runner lost contact during the fat-LTO build twice, which # GitHub attributes to CPU or memory starvation. Free disk, add swap and # record the resources as annotations, which survive a lost runner. + # Swap is best effort because runner disk layouts differ. - name: Free disk and add swap run: | sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /opt/hostedtoolcache/CodeQL - sudo fallocate -l 16G /swap.release && sudo chmod 600 /swap.release - sudo mkswap /swap.release && sudo swapon /swap.release + if sudo fallocate -l 16G /swap.release && sudo chmod 600 /swap.release \ + && sudo mkswap /swap.release && sudo swapon /swap.release; then + echo "Added 16G swap" + else + echo "::warning title=Swap::Could not add swap; continuing without it" + fi echo "::notice title=Runner resources::$(nproc) CPUs; $(free -h | awk '/Mem/{print $2" RAM"} /Swap/{print $2" swap"}' | paste -sd ' '); $(df -h --output=avail / | tail -1 | tr -d ' ') free on /" - name: Build native - if: ${{ !matrix.use-cross }} - run: cargo build --profile release-lto --target ${{ matrix.target }} + run: cargo build --profile "$RELEASE_PROFILE" --target ${{ matrix.target }} - name: Rename binary run: | - mv target/${{ matrix.target }}/release-lto/parseable Parseable_OSS_${{ matrix.target }} + mv target/${{ matrix.target }}/$RELEASE_PROFILE/parseable Parseable_OSS_${{ matrix.target }} - name: Generate artifact attestation uses: actions/attest-build-provenance@v1 @@ -149,6 +174,7 @@ jobs: path: Parseable_OSS_${{ matrix.target }} - name: Publish Archive to Release Page + if: env.RELEASE_PROFILE == 'release-lto' uses: softprops/action-gh-release@v0.1.15 with: draft: false @@ -159,6 +185,7 @@ jobs: build-windows: runs-on: windows-latest + timeout-minutes: 120 permissions: id-token: write contents: write @@ -167,7 +194,7 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - ref: refs/tags/${{ inputs.tag }} + ref: ${{ inputs.build-profile == 'release-thin-lto' && inputs.tag || format('refs/tags/{0}', inputs.tag) }} - name: Set up Node.js uses: actions/setup-node@v4 @@ -180,20 +207,30 @@ jobs: working-directory: frontend run: npm ci && npm run build - - name: Install latest rust toolchain - uses: actions-rs/toolchain@v1 + - name: Read pinned Rust toolchain + id: rust-toolchain + shell: bash + run: | + channel=$(sed -n 's/^channel *= *"\(.*\)"/\1/p' rust-toolchain.toml 2>/dev/null || true) + echo "channel=${channel:-stable}" >> "$GITHUB_OUTPUT" + + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@master with: - toolchain: stable - profile: minimal - default: true - override: true + toolchain: ${{ steps.rust-toolchain.outputs.channel }} + targets: x86_64-pc-windows-msvc + + - name: Cache Rust build + uses: Swatinem/rust-cache@v2 + with: + key: x86_64-pc-windows-msvc-default-${{ env.RELEASE_PROFILE }} - name: Build - run: cargo build --all --profile release-lto --target x86_64-pc-windows-msvc + run: cargo build --all --profile ${{ env.RELEASE_PROFILE }} --target x86_64-pc-windows-msvc - name: Rename binary run: | - mv target/x86_64-pc-windows-msvc/release-lto/PARSEABLE.exe Parseable_OSS_x86_64-pc-windows-msvc.exe + mv target/x86_64-pc-windows-msvc/${{ env.RELEASE_PROFILE }}/PARSEABLE.exe Parseable_OSS_x86_64-pc-windows-msvc.exe - name: Generate artifact attestation uses: actions/attest-build-provenance@v1 @@ -207,6 +244,7 @@ jobs: path: Parseable_OSS_x86_64-pc-windows-msvc.exe - name: Publish Archive to Release Page + if: env.RELEASE_PROFILE == 'release-lto' uses: softprops/action-gh-release@v0.1.15 with: draft: false @@ -217,6 +255,7 @@ jobs: build-mac: runs-on: macos-latest + timeout-minutes: 120 permissions: id-token: write contents: write @@ -230,7 +269,7 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - ref: refs/tags/${{ inputs.tag }} + ref: ${{ inputs.build-profile == 'release-thin-lto' && inputs.tag || format('refs/tags/{0}', inputs.tag) }} - name: Set up Node.js uses: actions/setup-node@v4 @@ -243,23 +282,32 @@ jobs: working-directory: frontend run: npm ci && npm run build - - name: Install latest rust toolchain - uses: actions-rs/toolchain@v1 + - name: Read pinned Rust toolchain + id: rust-toolchain + shell: bash + run: | + channel=$(sed -n 's/^channel *= *"\(.*\)"/\1/p' rust-toolchain.toml 2>/dev/null || true) + echo "channel=${channel:-stable}" >> "$GITHUB_OUTPUT" + + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ steps.rust-toolchain.outputs.channel }} + targets: ${{ matrix.target }} + + - name: Cache Rust build + uses: Swatinem/rust-cache@v2 with: - toolchain: stable - profile: minimal - target: ${{ matrix.target }} - default: true - override: true + key: ${{ matrix.target }}-default-${{ env.RELEASE_PROFILE }} - name: Build run: | - cargo build --profile release-lto --target ${{ matrix.target }} - strip target/${{ matrix.target }}/release-lto/Parseable + cargo build --profile "$RELEASE_PROFILE" --target ${{ matrix.target }} + strip target/${{ matrix.target }}/$RELEASE_PROFILE/Parseable - name: Rename binary run: | - mv target/${{ matrix.target }}/release-lto/Parseable Parseable_OSS_${{ matrix.target }} + mv target/${{ matrix.target }}/$RELEASE_PROFILE/Parseable Parseable_OSS_${{ matrix.target }} - name: Generate artifact attestation uses: actions/attest-build-provenance@v1 @@ -273,6 +321,7 @@ jobs: path: Parseable_OSS_${{ matrix.target }} - name: Publish Archive to Release Page + if: env.RELEASE_PROFILE == 'release-lto' uses: softprops/action-gh-release@v0.1.15 with: draft: false @@ -283,7 +332,8 @@ jobs: build-kafka-linux: name: Build Kafka for x86_64-unknown-linux-gnu - runs-on: ubuntu-latest + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 90 permissions: id-token: write contents: write @@ -291,7 +341,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: refs/tags/${{ inputs.tag }} + ref: ${{ inputs.build-profile == 'release-thin-lto' && inputs.tag || format('refs/tags/{0}', inputs.tag) }} - name: Set up Node.js uses: actions/setup-node@v4 @@ -320,11 +370,24 @@ jobs: libssl-dev \ python3 + - name: Read pinned Rust toolchain + id: rust-toolchain + shell: bash + run: | + channel=$(sed -n 's/^channel *= *"\(.*\)"/\1/p' rust-toolchain.toml 2>/dev/null || true) + echo "channel=${channel:-stable}" >> "$GITHUB_OUTPUT" + - name: Setup Rust toolchain - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@master with: + toolchain: ${{ steps.rust-toolchain.outputs.channel }} targets: x86_64-unknown-linux-gnu + - name: Cache Rust build + uses: Swatinem/rust-cache@v2 + with: + key: x86_64-unknown-linux-gnu-kafka-${{ env.RELEASE_PROFILE }} + - name: Find and fix librdkafka CMakeLists.txt run: | cargo fetch @@ -346,11 +409,11 @@ jobs: CC: gcc-11 CXX: g++-11 LIBRDKAFKA_SSL_VENDORED: "1" - run: cargo build --profile release-lto --target x86_64-unknown-linux-gnu --features kafka + run: cargo build --profile "$RELEASE_PROFILE" --target x86_64-unknown-linux-gnu --features kafka - name: Rename binary run: | - mv target/x86_64-unknown-linux-gnu/release-lto/parseable Parseable_OSS_x86_64-unknown-linux-gnu-kafka + mv target/x86_64-unknown-linux-gnu/$RELEASE_PROFILE/parseable Parseable_OSS_x86_64-unknown-linux-gnu-kafka - name: Generate artifact attestation uses: actions/attest-build-provenance@v1 @@ -364,6 +427,7 @@ jobs: path: Parseable_OSS_x86_64-unknown-linux-gnu-kafka - name: Publish Archive to Release Page + if: env.RELEASE_PROFILE == 'release-lto' uses: softprops/action-gh-release@v0.1.15 with: draft: false @@ -375,6 +439,7 @@ jobs: build-kafka-mac: name: Build Kafka for aarch64-apple-darwin runs-on: macos-latest + timeout-minutes: 120 permissions: id-token: write contents: write @@ -383,7 +448,7 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - ref: refs/tags/${{ inputs.tag }} + ref: ${{ inputs.build-profile == 'release-thin-lto' && inputs.tag || format('refs/tags/{0}', inputs.tag) }} - name: Set up Node.js uses: actions/setup-node@v4 @@ -406,11 +471,24 @@ jobs: openssl@3.0 \ python3 + - name: Read pinned Rust toolchain + id: rust-toolchain + shell: bash + run: | + channel=$(sed -n 's/^channel *= *"\(.*\)"/\1/p' rust-toolchain.toml 2>/dev/null || true) + echo "channel=${channel:-stable}" >> "$GITHUB_OUTPUT" + - name: Setup Rust toolchain - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@master with: + toolchain: ${{ steps.rust-toolchain.outputs.channel }} targets: aarch64-apple-darwin + - name: Cache Rust build + uses: Swatinem/rust-cache@v2 + with: + key: aarch64-apple-darwin-kafka-${{ env.RELEASE_PROFILE }} + - name: Find and fix librdkafka CMakeLists.txt run: | cargo fetch @@ -431,12 +509,12 @@ jobs: env: LIBRDKAFKA_SSL_VENDORED: "1" run: | - cargo build --profile release-lto --target aarch64-apple-darwin --features kafka - strip target/aarch64-apple-darwin/release-lto/Parseable + cargo build --profile "$RELEASE_PROFILE" --target aarch64-apple-darwin --features kafka + strip target/aarch64-apple-darwin/$RELEASE_PROFILE/Parseable - name: Rename binary run: | - mv target/aarch64-apple-darwin/release-lto/Parseable Parseable_OSS_aarch64-apple-darwin-kafka + mv target/aarch64-apple-darwin/$RELEASE_PROFILE/Parseable Parseable_OSS_aarch64-apple-darwin-kafka - name: Generate artifact attestation uses: actions/attest-build-provenance@v1 @@ -450,6 +528,7 @@ jobs: path: Parseable_OSS_aarch64-apple-darwin-kafka - name: Publish Archive to Release Page + if: env.RELEASE_PROFILE == 'release-lto' uses: softprops/action-gh-release@v0.1.15 with: draft: false @@ -459,7 +538,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} create-checksum: - runs-on: ubuntu-22.04 # or ubuntu-24.04, ubuntu-20.04 + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 10 needs: [build-linux, build-windows, build-mac, build-kafka-linux, build-kafka-mac] permissions: contents: write @@ -480,6 +560,7 @@ jobs: path: checksum.txt - name: Publish Check Sum to Release Page + if: env.RELEASE_PROFILE == 'release-lto' uses: softprops/action-gh-release@v0.1.15 with: draft: false @@ -489,13 +570,14 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} build-push-docker-image: - if: github.repository == 'parseablehq/parseable' + if: github.repository == 'parseablehq/parseable' && (inputs.build-profile || 'release-lto') == 'release-lto' runs-on: self-hosted + timeout-minutes: 120 steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: - ref: refs/tags/${{ inputs.tag }} + ref: ${{ inputs.build-profile == 'release-thin-lto' && inputs.tag || format('refs/tags/{0}', inputs.tag) }} - name: Set up QEMU uses: docker/setup-qemu-action@v3 diff --git a/Cargo.toml b/Cargo.toml index 8674a5b0e..b666b79c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -267,3 +267,8 @@ kafka = [ inherits = "release" lto = "fat" codegen-units = 1 + +# Experimental faster release build; release-lto stays the production profile. +[profile.release-thin-lto] +inherits = "release" +lto = "thin" diff --git a/Dockerfile.debug b/Dockerfile.debug index e5289ea28..62f14b93b 100644 --- a/Dockerfile.debug +++ b/Dockerfile.debug @@ -35,9 +35,11 @@ WORKDIR /parseable COPY Cargo.toml Cargo.lock build.rs ./ COPY build_support ./build_support COPY scripts/community-ui-overlay.json scripts/community-promql-capabilities.js ./scripts/ +RUN mkdir src && echo "fn main() {}" > src/main.rs && cargo build && rm -rf src + +# Embed frontend assets after compiling dependencies so UI edits reuse them. COPY --from=frontend /frontend/dist /parseable/frontend/dist ENV NEXT_ASSETS_PATH=/parseable/frontend/dist -RUN mkdir src && echo "fn main() {}" > src/main.rs && cargo build && rm -rf src # Build the actual binary COPY src ./src diff --git a/Dockerfile.kafka b/Dockerfile.kafka index 19dfc2d87..fd1f323d7 100644 --- a/Dockerfile.kafka +++ b/Dockerfile.kafka @@ -44,8 +44,6 @@ WORKDIR /parseable COPY Cargo.toml Cargo.lock build.rs ./ COPY build_support ./build_support COPY scripts/community-ui-overlay.json scripts/community-promql-capabilities.js ./scripts/ -COPY --from=frontend /frontend/dist /parseable/frontend/dist -ENV NEXT_ASSETS_PATH=/parseable/frontend/dist # Fix librdkafka CMakeLists.txt before building RUN mkdir -p src && echo "fn main() {}" > src/main.rs && \ @@ -70,6 +68,10 @@ RUN mkdir -p src && echo "fn main() {}" > src/main.rs && \ RUN cargo build --release --features kafka && \ rm -rf src +# Embed frontend assets after compiling dependencies so UI edits reuse them. +COPY --from=frontend /frontend/dist /parseable/frontend/dist +ENV NEXT_ASSETS_PATH=/parseable/frontend/dist + # Copy the actual source code COPY src ./src COPY resources ./resources diff --git a/README.md b/README.md index a8cb676fb..c64fdb36c 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,26 @@ For production deployments, refer to the [installation guide ↗︎](https://www ## Contributing +Pull requests use the [CI workflow](.github/workflows/ci.yaml) on Blacksmith +8-vCPU Ubuntu 24.04 runners. It selects frontend, Rust, and integration jobs from +the changed paths and reports one aggregate `CI passed` check. Frontend changes +still exercise the embedded UI through the integration image; Rust-only changes +skip the browser suite. A newer PR commit cancels its superseded CI run. + +Browser tests run in four shards alongside production and Storybook checks. +Clippy runs alongside Rust tests. Integration builds one cached Docker image and +shares it between standalone and distributed Quest jobs. PRs keep Quest's +functional tests but exclude its six sustained k6 tests. Pushes to `main` and the +nightly run (06:17 UTC) run the full Quest load suite and upload Rust coverage. +Manual CI runs can select `full` to exercise those checks before merging. + +Release builds cache Rust dependencies per target, feature set, and profile. +The manual release workflow defaults to the existing fat-LTO profile. Its +`release-thin-lto` option is an experiment that only uploads workflow artifacts; +it accepts a tag, branch, or commit containing that Cargo profile and never +publishes release assets or images. Compare build times and runtime benchmarks +on the same source revision before changing the production profile. + Contributors [Contribution guide ↗︎](https://github.com/parseablehq/parseable/blob/main/CONTRIBUTING.md) diff --git a/docker-compose-ci-smoke.yaml b/docker-compose-ci-smoke.yaml new file mode 100644 index 000000000..b1b7215b5 --- /dev/null +++ b/docker-compose-ci-smoke.yaml @@ -0,0 +1,6 @@ +# CI uses the same Quest functional suite, excluding its sustained k6 phases. +services: + quest: + entrypoint: [sh, /ci/quest-smoke.sh] + volumes: + - ./scripts/ci/quest-smoke.sh:/ci/quest-smoke.sh:ro diff --git a/docker-compose-distributed-test.yaml b/docker-compose-distributed-test.yaml index 52a5352f3..b46810649 100644 --- a/docker-compose-distributed-test.yaml +++ b/docker-compose-distributed-test.yaml @@ -26,6 +26,7 @@ services: - parseable-internal # query server parseable-query: + image: ${PARSEABLE_TEST_IMAGE:-parseable-test:local} build: context: . dockerfile: Dockerfile.debug @@ -64,6 +65,7 @@ services: max_attempts: 3 # ingest server one parseable-ingest-one: + image: ${PARSEABLE_TEST_IMAGE:-parseable-test:local} build: context: . dockerfile: Dockerfile.debug @@ -113,7 +115,7 @@ services: - QUEST_EDITION=oss command: [ - "load", + "${QUEST_MODE:-load}", "http://parseable-query:8000", "parseableadmin", "parseableadmin", diff --git a/docker-compose-test.yaml b/docker-compose-test.yaml index 062cc04dc..ddef891fb 100644 --- a/docker-compose-test.yaml +++ b/docker-compose-test.yaml @@ -25,6 +25,7 @@ services: - parseable-internal parseable: + image: ${PARSEABLE_TEST_IMAGE:-parseable-test:local} build: context: . dockerfile: Dockerfile.debug @@ -69,7 +70,7 @@ services: environment: - QUEST_EDITION=oss command: [ - "load", + "${QUEST_MODE:-load}", "http://parseable:8000", "parseableadmin", "parseableadmin", diff --git a/scripts/ci/quest-smoke.sh b/scripts/ci/quest-smoke.sh new file mode 100644 index 000000000..5681f205d --- /dev/null +++ b/scripts/ci/quest-smoke.sh @@ -0,0 +1,21 @@ +#!/bin/sh +set -eu + +# Match Quest's main.sh positional arguments and pb configuration. Its "smoke" +# mode still includes a multi-minute k6 test, so explicitly exclude load phases. +# Full main/nightly runs continue to use the image's unmodified entrypoint. +if [ "$#" -lt 11 ] || [ "$1" != smoke ]; then + echo 'Expected Quest smoke arguments (endpoint, credentials and MinIO settings)' >&2 + exit 2 +fi +export XDG_CONFIG_HOME="${XDG_CONFIG_HOME:-/tmp/quest-pb-config}" +pb profile add quest "$2" "$3" "$4" -o json +pb profile default quest -o json +stream="ci$(od -An -N8 -tx1 /dev/urandom | tr -d ' \n')" + +exec ./quest.test -test.v -test.parallel=64 -test.timeout=12m \ + -test.skip '^(TestLoad.*WithK6.*|TestSmokeLoadWithK6Streams)$' \ + -edition="${QUEST_EDITION:-oss}" -mode=smoke \ + -query-url="$2" -query-user="$3" -query-pass="$4" -stream="$stream" \ + -minio-url="$8" -minio-user="$9" -minio-pass="${10}" -minio-bucket="${11}" \ + -ingestor-url="${12:-}" -ingestor-user="${13:-}" -ingestor-pass="${14:-}"