Repository navigation
Preserve PR test qualification and record workflow delivery #3
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarks | |
| on: | |
| push: | |
| branches: [main] | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: keyload-benchmarks-${{ github.ref }} | |
| cancel-in-progress: false | |
| jobs: | |
| comparison-build: | |
| name: Build and checks | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 | |
| with: | |
| global-json-file: global.json | |
| - run: dotnet restore KeyLoad.slnx | |
| - run: dotnet build KeyLoad.slnx --no-restore --configuration Release | |
| - name: Verify EditorConfig and analyzer fixes | |
| run: dotnet format KeyLoad.slnx --verify-no-changes --no-restore | |
| - name: Check repository rules | |
| run: node scripts/Features/RepositoryGovernance/verify.mjs | |
| - name: Retain compiler diagnostic reports | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: code-quality-comparisons | |
| path: artifacts/code-quality/** | |
| if-no-files-found: error | |
| comparison-plan: | |
| if: github.repository == 'managedcode/KeyLoad' && github.ref == 'refs/heads/main' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| outputs: | |
| crud: ${{ steps.plan.outputs.crud_matrix }} | |
| specialized: ${{ steps.plan.outputs.specialized_matrix }} | |
| preflight: ${{ steps.plan.outputs.preflight_matrix }} | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 | |
| - name: Freeze the complete intensive Linux cohort | |
| id: plan | |
| run: node scripts/Features/BenchmarkComparisons/isolated-plan.mjs --output="$RUNNER_TEMP/isolated-plan.json" --github-output="$GITHUB_OUTPUT" | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-plan | |
| path: ${{ runner.temp }}/isolated-plan.json | |
| if-no-files-found: error | |
| retention-days: 90 | |
| comparison-images: | |
| name: comparison-images | |
| needs: [comparison-build, comparison-plan] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| permissions: | |
| contents: read | |
| actions: read | |
| outputs: | |
| bundle-artifact-id: ${{ steps.bundle.outputs.artifact-id }} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| KEYLOAD_COMPARISON_JOB_NAME: comparison-images | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 | |
| - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 | |
| with: | |
| global-json-file: global.json | |
| - name: Require the actual Docker runtime | |
| run: docker version | |
| - name: Prepare immutable source-bound runtime images once | |
| id: images | |
| run: node scripts/Features/BenchmarkComparisons/prepare-images.mjs | |
| - name: Bind the verified runtime image receipt | |
| env: | |
| KEYLOAD_SERVER_IMAGE: ${{ steps.images.outputs.server-image }} | |
| KEYLOAD_RUNNER_IMAGE: ${{ steps.images.outputs.load-generator-image }} | |
| run: | | |
| test -n "$KEYLOAD_SERVER_IMAGE" && test -n "$KEYLOAD_RUNNER_IMAGE" | |
| printf 'KeyLoad__ContainerImages__Server=%s\nBenchmarks__ContainerImages__LoadGenerator=%s\nKEYLOAD_IMAGE_RECEIPT=%s\n' "$KEYLOAD_SERVER_IMAGE" "$KEYLOAD_RUNNER_IMAGE" "$RUNNER_TEMP/keyload-images/image-receipt.json" >> "$GITHUB_ENV" | |
| - run: dotnet restore KeyLoad.slnx | |
| - run: dotnet build tests/KeyLoad.ComparisonTests --no-restore --configuration Release | |
| - name: Qualify native image export and import | |
| id: roundtrip | |
| run: dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/ImageBundleRealTests/*' --results-directory TestResults/comparison-images/image-roundtrip | |
| - name: Qualify authenticated current-job identity | |
| run: dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/IsolatedGitHubCurrentJobTests/*' --results-directory TestResults/comparison-images/job-identity | |
| - name: Qualify exact native topology composition models | |
| run: | | |
| dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/Isolated*Resource*/*' --results-directory TestResults/comparison-images/resource-models | |
| dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/MongoReadinessResourceTests/*' --results-directory TestResults/comparison-images/mongo-resource | |
| dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/MongoReadinessIdentityTests/*' --results-directory TestResults/comparison-images/mongo-identity | |
| dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/MongoReadinessTaskFailureTests/*' --results-directory TestResults/comparison-images/mongo-task-failure | |
| dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/IsolatedKurrentVolumeRegressionFailureTests/*' --results-directory TestResults/comparison-images/kurrent-volume-failure | |
| dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/IsolatedKeyLoadAdmissionTests/*' --results-directory TestResults/comparison-images/keyload-admission | |
| dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/ComparisonResourceLogBufferTests/*' --results-directory TestResults/comparison-images/log-buffer | |
| - name: Qualify isolated replay admission and native diagnostics | |
| run: | | |
| dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/IsolatedKeyLoadReplayAdmissionTests/*' --results-directory TestResults/comparison-images/replay-admission | |
| dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/RedisReplicaDiagnosticTests/*' --results-directory TestResults/comparison-images/redis-diagnostics | |
| dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/MongoBootstrapDiagnosticTests/*' --results-directory TestResults/comparison-images/mongo-diagnostics | |
| dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/KurrentCleanupDiagnosticTests/*' --results-directory TestResults/comparison-images/kurrent-diagnostics | |
| dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/ComparisonReplayDiagnosticLogTests/*' --results-directory TestResults/comparison-images/replay-log | |
| dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/ComparisonReplayDiagnosticRetentionTests/*' --results-directory TestResults/comparison-images/replay-retention | |
| - name: Qualify time-series workload and aggregation contracts | |
| run: | | |
| dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/TimeSeriesWorkloadTests/*' --results-directory TestResults/comparison-images/timeseries-workload | |
| dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/TimeSeriesPackageVersionTests/*' --results-directory TestResults/comparison-images/timeseries-package | |
| - name: Qualify isolated TimeSeries resource models | |
| run: | | |
| dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/IsolatedTimeSeriesKeyLoadResourceTests/*' --results-directory TestResults/comparison-images/timeseries-keyload-model | |
| dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/IsolatedTimeSeriesTimescaleResourceTests/*' --results-directory TestResults/comparison-images/timeseries-timescale-model | |
| dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/IsolatedTimeSeriesBenchmarkResourceTests/*' --results-directory TestResults/comparison-images/timeseries-family-model | |
| - name: Retain common image bundle | |
| id: bundle | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-image-bundle | |
| path: ${{ runner.temp }}/keyload-image-bundle/* | |
| if-no-files-found: error | |
| compression-level: 0 | |
| retention-days: 90 | |
| - name: Capture job-owned registry after an incomplete round-trip | |
| if: ${{ always() && steps.roundtrip.outcome != 'success' && (steps.images.outcome == 'success' || steps.images.outcome == 'failure') }} | |
| run: node scripts/Features/BenchmarkComparisons/cleanup-images.mjs | |
| - name: Retain native image and source qualification evidence | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-images-qualification | |
| path: | | |
| ${{ runner.temp }}/keyload-images/** | |
| ${{ runner.temp }}/keyload-images-build/** | |
| ${{ runner.temp }}/keyload-cell-github/** | |
| **/TestResults/** | |
| artifacts/code-quality/** | |
| if-no-files-found: error | |
| retention-days: 90 | |
| comparison-preflight: | |
| name: preflight / ${{ matrix.id }} | |
| needs: [comparison-plan, comparison-images] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 150 | |
| permissions: | |
| contents: read | |
| actions: read | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 6 | |
| matrix: ${{ fromJSON(needs.comparison-plan.outputs.preflight) }} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| KEYLOAD_COMPARISON_CELL_ID: ${{ matrix.id }} | |
| KEYLOAD_COMPARISON_JOB_NAME: preflight / ${{ matrix.id }} | |
| Benchmarks__Target: ${{ matrix.target }} | |
| Benchmarks__NodeCount: ${{ matrix.nodeCount }} | |
| Benchmarks__Scenario: ${{ matrix.scenario }} | |
| Benchmarks__EvidenceProfile: ${{ matrix.profile }} | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 | |
| - name: Prepare isolated native case | |
| id: images | |
| uses: ./.github/workflows/Features/BenchmarkComparisons/IsolatedCellSetup | |
| with: | |
| bundle-artifact-id: ${{ needs.comparison-images.outputs.bundle-artifact-id }} | |
| - run: dotnet restore KeyLoad.slnx | |
| - run: dotnet build tests/KeyLoad.ComparisonTests --no-restore --configuration Release | |
| - name: Run isolated native case | |
| run: dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/IsolatedNativeComparisonTests/*' | |
| - name: Retain isolated worker evidence | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-preflight-${{ matrix.id }} | |
| path: artifacts/comparisons/isolated/workers/${{ matrix.id }}/* | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: Retain native worker lifecycle evidence | |
| if: always() | |
| uses: ./.github/workflows/Features/BenchmarkComparisons/IsolatedCellTeardown | |
| with: | |
| setup-outcome: ${{ steps.images.outcome }} | |
| artifact-name: comparison-preflight-qualification-${{ matrix.id }} | |
| comparison-crud: | |
| name: case / ${{ matrix.id }} | |
| needs: [comparison-plan, comparison-images, comparison-preflight] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 150 | |
| permissions: | |
| contents: read | |
| actions: read | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 6 | |
| matrix: ${{ fromJSON(needs.comparison-plan.outputs.crud) }} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| KEYLOAD_COMPARISON_CELL_ID: ${{ matrix.id }} | |
| KEYLOAD_COMPARISON_JOB_NAME: case / ${{ matrix.id }} | |
| Benchmarks__Target: ${{ matrix.target }} | |
| Benchmarks__NodeCount: ${{ matrix.nodeCount }} | |
| Benchmarks__Scenario: ${{ matrix.scenario }} | |
| Benchmarks__EvidenceProfile: ${{ matrix.profile }} | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 | |
| - name: Prepare isolated native case | |
| id: images | |
| uses: ./.github/workflows/Features/BenchmarkComparisons/IsolatedCellSetup | |
| with: | |
| bundle-artifact-id: ${{ needs.comparison-images.outputs.bundle-artifact-id }} | |
| - run: dotnet restore KeyLoad.slnx | |
| - run: dotnet build tests/KeyLoad.ComparisonTests --no-restore --configuration Release | |
| - name: Run isolated native case | |
| run: dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/IsolatedNativeComparisonTests/*' | |
| - name: Retain isolated worker evidence | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-worker-${{ matrix.id }} | |
| path: artifacts/comparisons/isolated/workers/${{ matrix.id }}/* | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: Retain native worker lifecycle evidence | |
| if: always() | |
| uses: ./.github/workflows/Features/BenchmarkComparisons/IsolatedCellTeardown | |
| with: | |
| setup-outcome: ${{ steps.images.outcome }} | |
| artifact-name: comparison-case-qualification-${{ matrix.id }} | |
| comparison-specialized: | |
| name: case / ${{ matrix.id }} | |
| needs: [comparison-plan, comparison-images, comparison-preflight, comparison-crud] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 150 | |
| permissions: | |
| contents: read | |
| actions: read | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 6 | |
| matrix: ${{ fromJSON(needs.comparison-plan.outputs.specialized) }} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| KEYLOAD_COMPARISON_CELL_ID: ${{ matrix.id }} | |
| KEYLOAD_COMPARISON_JOB_NAME: case / ${{ matrix.id }} | |
| Benchmarks__Target: ${{ matrix.target }} | |
| Benchmarks__NodeCount: ${{ matrix.nodeCount }} | |
| Benchmarks__Scenario: ${{ matrix.scenario }} | |
| Benchmarks__EvidenceProfile: ${{ matrix.profile }} | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 | |
| - name: Prepare isolated native case | |
| id: images | |
| uses: ./.github/workflows/Features/BenchmarkComparisons/IsolatedCellSetup | |
| with: | |
| bundle-artifact-id: ${{ needs.comparison-images.outputs.bundle-artifact-id }} | |
| - run: dotnet restore KeyLoad.slnx | |
| - run: dotnet build tests/KeyLoad.ComparisonTests --no-restore --configuration Release | |
| - name: Run isolated native case | |
| run: dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/IsolatedNativeComparisonTests/*' | |
| - name: Retain isolated worker evidence | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-worker-${{ matrix.id }} | |
| path: artifacts/comparisons/isolated/workers/${{ matrix.id }}/* | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: Retain native worker lifecycle evidence | |
| if: always() | |
| uses: ./.github/workflows/Features/BenchmarkComparisons/IsolatedCellTeardown | |
| with: | |
| setup-outcome: ${{ steps.images.outcome }} | |
| artifact-name: comparison-case-qualification-${{ matrix.id }} | |
| comparison-aggregate: | |
| needs: [comparison-plan, comparison-images, comparison-crud, comparison-specialized] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 150 | |
| permissions: | |
| contents: read | |
| actions: read | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 | |
| - name: Recreate the canonical complete intensive plan | |
| run: node scripts/Features/BenchmarkComparisons/isolated-plan.mjs --output="$RUNNER_TEMP/isolated-plan.json" | |
| - name: Collect exact authenticated native jobs and immutable raw artifacts | |
| run: node scripts/Features/BenchmarkComparisons/isolated-github-collect.mjs --plan="$RUNNER_TEMP/isolated-plan.json" --input="$RUNNER_TEMP/isolated-capture" | |
| - name: Validate and retain all270 byte-preserved workers | |
| run: node scripts/Features/BenchmarkComparisons/aggregate-cli.mjs --plan="$RUNNER_TEMP/isolated-plan.json" --input="$RUNNER_TEMP/isolated-capture/data" --proof="$RUNNER_TEMP/isolated-capture/github/proof.json" --output="$RUNNER_TEMP/isolated-aggregate" | |
| - name: Generate bounded native performance metrics | |
| run: node site/scripts/build.mjs --isolated="$RUNNER_TEMP/isolated-aggregate" --output="$RUNNER_TEMP/isolated-site" --revision="$GITHUB_SHA" --site-revision="$GITHUB_SHA" --evidence-url="https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" | |
| - name: Retain the generated native metrics candidate | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-isolated-site-candidate | |
| path: ${{ runner.temp }}/isolated-site/** | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: Retain the complete qualified comparison cohort | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-isolated-suite | |
| path: ${{ runner.temp }}/isolated-aggregate/** | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: Retain authenticated provider and archive integrity evidence | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-isolated-provider-evidence | |
| path: ${{ runner.temp }}/isolated-capture/github/** | |
| if-no-files-found: error | |
| retention-days: 90 | |
| timeseries-image-facts: | |
| name: TimeSeries image | |
| needs: comparison-build | |
| if: github.repository == 'managedcode/KeyLoad' && github.ref == 'refs/heads/main' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 40 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 | |
| - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 | |
| with: | |
| global-json-file: global.json | |
| - name: Require exact own-main source and actual Docker runtime | |
| shell: bash | |
| run: | | |
| test "$(git rev-parse HEAD)" = "$GITHUB_SHA" | |
| test "$GITHUB_REPOSITORY" = 'managedcode/KeyLoad' | |
| test "$GITHUB_REF" = 'refs/heads/main' | |
| docker version | |
| - run: dotnet restore KeyLoad.slnx | |
| - run: dotnet build tests/KeyLoad.ComparisonTests --no-restore --configuration Release | |
| - name: Observe the actual pinned Timescale image and native bootstrap primitives | |
| env: | |
| KEYLOAD_TIMESERIES_IMAGE_FACTS_DIRECTORY: ${{ runner.temp }}/keyload-timeseries-image-facts | |
| run: dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/TimeSeriesIntensivePinnedImageTests/*' --results-directory TestResults/timeseries-image | |
| - name: Retain actual native image facts and feasibility qualification | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: timeseries-native-pinned-image-facts | |
| path: | | |
| ${{ runner.temp }}/keyload-timeseries-image-facts/** | |
| **/TestResults/** | |
| artifacts/code-quality/** | |
| if-no-files-found: error | |
| retention-days: 90 |