diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml new file mode 100644 index 0000000..8b13269 --- /dev/null +++ b/.github/workflows/quality.yml @@ -0,0 +1,62 @@ +name: Quality + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: quality-${{ github.ref }} + cancel-in-progress: true + +jobs: + baseline-regression: + name: Training calibration and inference drift regressions + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + enable-cache: true + + - name: Install Python + run: uv python install 3.12 + + - name: Install locked dependencies + run: uv sync --locked + + - name: Run pinned baseline regression gates + run: uv run pytest -m baseline_regression tests/regression + + test-suite: + name: Full test suite + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + enable-cache: true + + - name: Install Python + run: uv python install 3.12 + + - name: Install locked dependencies + run: uv sync --locked --extra legacy-ui + + - name: Run tests + run: uv run pytest diff --git a/data/validation/baseline_regression_contract.json b/data/validation/baseline_regression_contract.json new file mode 100644 index 0000000..056cffa --- /dev/null +++ b/data/validation/baseline_regression_contract.json @@ -0,0 +1,89 @@ +{ + "schema": "blueprinting.baseline-regression-contract.v1", + "training": { + "baseline": "Calculon / SeqSel Table 5", + "evidence_revision": "eb1eb9fcc4a6e414e85b0252c23ea9ad2730aae2", + "case_count": 8, + "budgets": { + "workload_max_absolute_error_percent": 1e-9, + "calculon_mean_absolute_error_percent": 1e-9, + "calculon_max_absolute_error_percent": 1e-9, + "paper_mean_absolute_error_percent": 3.7, + "paper_max_absolute_error_percent": 8.9, + "memory_max_absolute_error_bytes": 1.0 + }, + "golden": { + "peak_mean_absolute_error_percent": 12.993831306582473, + "calculon_mean_absolute_error_percent": 6.17654222892984e-15, + "paper_mean_absolute_error_percent": 3.654361437637248, + "paper_max_absolute_error_percent": 8.874452316395821, + "portable_digests": { + "seqsel-tab5/megatron-22B/full": "9ce66dd1c115a1efbb0cad1cc3921c3f28ffc58f", + "seqsel-tab5/megatron-22B/seqsel": "873c82dde8f06f97a619855bf7fa74d5730c7d55", + "seqsel-tab5/gpt3-175B/full": "2c9566d969073f2983b2ca3e35d840c986063058", + "seqsel-tab5/gpt3-175B/seqsel": "d3f1826b4e0b24ee91fc5920c95c1dd3b3ca6782", + "seqsel-tab5/turing-530B/full": "1eca776c1c27807964278cf5e768193a6f06c4cc", + "seqsel-tab5/turing-530B/seqsel": "8095358ace4a3a62466c13b0895e03545c6f5621", + "seqsel-tab5/megatron-1T/full": "5ffc23392d5a374f442ef2f448366e3d9f6118ce", + "seqsel-tab5/megatron-1T/seqsel": "377375178daedbd9655bdccb618eab11ba8b68f3" + } + } + }, + "inference": { + "baseline": "Vidur Phi-2 / A100 / TP1 raw component-profile alignment", + "validation_claim": "drift-detection-not-accuracy-validation", + "fixture": "data/validation/vidur/phi2_a100_tp1", + "fixture_manifest_sha256": "24511af4979e32fdc0d1c610960f93e92ac816a2a5b30f30cf6015d2492b2c33", + "source_repository": "https://github.com/microsoft/vidur", + "source_revision": "8383d2935bc62723a212090baa9f98ada206fc14", + "license_file": "LICENSE.vidur", + "license_sha256": "7df20dcdf9197e9945c14858d41c60f11b52b93e5b69e2b63416b874d598d322", + "baseline_revision": "6a583f1c90969bd82ef82b8cf6dd26bcef914909", + "model_digest": "d67e201b24cc428ac0d3330d7fd597cfe90dcdf6", + "case_count": 3, + "budgets": { + "minimum_semantic_component_coverage": 0.75, + "system_evidence_comparable_subtotal_mean_absolute_error_percent": 24.0, + "system_evidence_comparable_subtotal_max_absolute_error_percent": 28.0, + "minimum_improvement_over_peak_percentage_points": 10.0 + }, + "golden": { + "peak_comparable_subtotal_mean_absolute_error_percent": 39.60396144129218, + "system_evidence_comparable_subtotal_mean_absolute_error_percent": 23.57567170288191, + "system_evidence_component_mean_absolute_error_percent": 54.07702487970536, + "system_evidence_component_max_absolute_error_percent": 99.50036630036631, + "cases": { + "phi2-a100-tp1/decode/b1-c33": { + "distributed_digest": "1d6a4598cff8907554b3c246d3993143d17b3d5a", + "portable_digest": "5bbe78b2017f21215b6e66904ba95fa1ccda5e49", + "component_coverage": 0.75, + "baseline_comparable_block_seconds": 0.000147, + "system_evidence_comparable_block_seconds": 0.00010824697435897437, + "system_evidence_comparable_subtotal_error_percent": -26.36260247688818, + "system_evidence_component_mean_absolute_error_percent": 60.87949445624435, + "system_evidence_component_max_absolute_error_percent": 99.50036630036631 + }, + "phi2-a100-tp1/decode/b1-c129": { + "distributed_digest": "4ea3575ed2cd51d69d789e6addc7197a45d32efb", + "portable_digest": "74335a2d5a3ed2f9da31b334a93519c559a0c6a3", + "component_coverage": 0.75, + "baseline_comparable_block_seconds": 0.00015000000000000001, + "system_evidence_comparable_block_seconds": 0.00010879564102564103, + "system_evidence_comparable_subtotal_error_percent": -27.469572649572648, + "system_evidence_component_mean_absolute_error_percent": 60.549761945956284, + "system_evidence_component_max_absolute_error_percent": 99.50036630036631 + }, + "phi2-a100-tp1/prefill/b1-c128": { + "distributed_digest": "a91b167c1f4644bc7f16b38184f61a16d42d004c", + "portable_digest": "7fe68d256445d118ae842cef0d23e123741a7b62", + "component_coverage": 0.75, + "baseline_comparable_block_seconds": 0.0002015, + "system_evidence_comparable_block_seconds": 0.00023554310256410256, + "system_evidence_comparable_subtotal_error_percent": 16.894839982184898, + "system_evidence_component_mean_absolute_error_percent": 40.80181823691545, + "system_evidence_component_max_absolute_error_percent": 83.93685897435897 + } + } + } + } +} diff --git a/data/validation/vidur/phi2_a100_tp1/LICENSE.vidur b/data/validation/vidur/phi2_a100_tp1/LICENSE.vidur new file mode 100644 index 0000000..22aed37 --- /dev/null +++ b/data/validation/vidur/phi2_a100_tp1/LICENSE.vidur @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Microsoft Corporation. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/data/validation/vidur/phi2_a100_tp1/attention.csv b/data/validation/vidur/phi2_a100_tp1/attention.csv new file mode 100644 index 0000000..a4ef752 --- /dev/null +++ b/data/validation/vidur/phi2_a100_tp1/attention.csv @@ -0,0 +1,5 @@ +n_embd,n_q_head,n_kv_head,block_size,num_tensor_parallel_workers,max_model_len,batch_size,prefill_chunk_size,kv_cache_size,is_prefill,attention_backend,time_stats.attn_prefill.median,time_stats.attn_decode.median +2560,32,32,16,1,4096,1,0,32,False,AttentionBackend.FLASH_ATTENTION,,0.009000000000000001 +2560,32,32,16,1,4096,1,0,128,False,AttentionBackend.FLASH_ATTENTION,,0.012 +2560,32,32,16,1,4096,1,128,0,True,AttentionBackend.FLASH_ATTENTION,0.014, +2560,32,32,16,1,4096,1,128,0,True,AttentionBackend.FLASH_ATTENTION,0.014, diff --git a/data/validation/vidur/phi2_a100_tp1/manifest.json b/data/validation/vidur/phi2_a100_tp1/manifest.json new file mode 100644 index 0000000..c9967dc --- /dev/null +++ b/data/validation/vidur/phi2_a100_tp1/manifest.json @@ -0,0 +1,59 @@ +{ + "schema": "blueprinting.vidur-validation-slice.v1", + "source": { + "repository": "https://github.com/microsoft/vidur", + "revision": "8383d2935bc62723a212090baa9f98ada206fc14", + "license": "MIT", + "license_file": "LICENSE.vidur" + }, + "selection": { + "model": "microsoft/phi-2", + "hardware": "a100", + "datatype": "float16", + "tensor_parallel": 1, + "attention_backend": "AttentionBackend.FLASH_ATTENTION", + "block_size": 16, + "cases": [ + {"phase": "decode", "batch_size": 1, "context_tokens": 33}, + {"phase": "decode", "batch_size": 1, "context_tokens": 129}, + {"phase": "prefill", "batch_size": 1, "context_tokens": 128} + ], + "transformation": "Exact row filtering and column projection only; numeric values are unchanged. Duplicate selected upstream rows are retained." + }, + "blueprinting": { + "model": { + "name": "microsoft/phi-2", + "hidden_size": 2560, + "feedforward_size": 10240, + "sequence_length": 4096, + "attention_heads": 32, + "attention_head_size": 80, + "block_count": 32 + }, + "execution": { + "world_size": 1, + "tensor_parallel": 1, + "pipeline_parallel": 1, + "replicas": 1, + "datatype": "float16", + "tensor_parallel_network": 0, + "pipeline_parallel_network": 0 + }, + "hardware": { + "name": "a100_80g", + "profile": "data/systems/a100_80g.json" + } + }, + "files": { + "attention.csv": { + "upstream_path": "data/profiling/compute/a100/microsoft/phi-2/attention.csv", + "upstream_git_blob": "56c1d8c06ce1f08336fe4e82f445ec1249f5ca71", + "sha256": "3c1b7a28425484f9008307a5e63e9a92e52156d3d6734f2ce8f4a30f83fefb3f" + }, + "mlp.csv": { + "upstream_path": "data/profiling/compute/a100/microsoft/phi-2/mlp.csv", + "upstream_git_blob": "5db2fe4390c7fe2a46e545ab8703f3fc37617a7e", + "sha256": "9271dd3a390d1d1e27489f8ea207ba41ee28e42e4da9df14c7924066f4373256" + } + } +} diff --git a/data/validation/vidur/phi2_a100_tp1/mlp.csv b/data/validation/vidur/phi2_a100_tp1/mlp.csv new file mode 100644 index 0000000..c67f429 --- /dev/null +++ b/data/validation/vidur/phi2_a100_tp1/mlp.csv @@ -0,0 +1,3 @@ +n_embd,n_expanded_embd,n_head,n_kv_head,vocab_size,use_gated_mlp,num_tokens,num_tensor_parallel_workers,time_stats.input_layernorm.median,time_stats.attn_pre_proj.median,time_stats.attn_rope.median,time_stats.attn_post_proj.median,time_stats.mlp_up_proj.median,time_stats.mlp_act.median,time_stats.mlp_down_proj.median,time_stats.add.median +2560,10240,32,32,51200,False,128,1,0.008,0.043000000000000003,0.006,0.017,0.0505,0.005,0.054,0.004 +2560,10240,32,32,51200,False,1,1,0.007,0.03,0.004,0.014,0.038,0.002,0.039,0.004 diff --git a/docs/assets/architecture/inference-planning-path.svg b/docs/assets/architecture/inference-planning-path.svg new file mode 100644 index 0000000..7460442 --- /dev/null +++ b/docs/assets/architecture/inference-planning-path.svg @@ -0,0 +1,79 @@ + + Blueprinting inference planning and serving simulation boundary + A shared model and mapping are specialized into independently verifiable prefill and decode phase plans. Blueprinting costs those plans independently, while Vidur is consulted only by a post-hoc baseline comparison. Static request composition is implemented, and arrivals, batching and scheduling belong to a future discrete-event serving simulator. + + + + + + + + Inference Planning Path · 推理规划路径 + One semantic source, independent costing, external baseline comparison, and an explicit serving-simulation boundary + + + + + SHARED SEMANTICS + PHASE SPECIALIZATION & COST + COMPOSITION LAYERS + + + Model + Mapping + dimensions · TP · PP · replicas + request shape is not model structure + + + ModelIR + decoder_inference + KV state effect + phase-neutral · hardware-neutral + + + + + Prefill Binding + query = context = prompt tokens + ModelIR → DistributedTaskIR + → PortablePlanIR + KV capacity + + Decode Binding + query = 1 · explicit context length + same verified representation boundaries + one plan point per context + + + Blueprinting Cost Model + HardwareProfile · internal cost provider + no comparison-oracle input + + Phase Cost View + component latency · TP / PP transfer + weights · KV · working memory + + + + Vidur Raw Profile Report + post-hoc coverage · error · provenance + + + + Static Request Composition + prefill emits token 1 + decode contexts S+1 … S+O−1 + prefill · decode steps · model execution + implemented · no queueing claim + + + + Serving DES (next layer) + arrival trace · routing · queues + continuous batching · chunked prefill + scheduler / CPU overhead · SLO tails + consumes the same phase cost view + does not redefine operator semantics + + Invariant: Blueprinting costs its own lowered plan; a comparison baseline can measure error but cannot change that plan or estimate. + 不变量：Blueprinting 独立估算自己的 lowered plan；baseline 只能度量误差，不能修改 plan 或 estimate。 + diff --git a/docs/design/performance/database.en.md b/docs/design/performance/database.en.md index 18f2070..a5057d1 100644 --- a/docs/design/performance/database.en.md +++ b/docs/design/performance/database.en.md @@ -3,7 +3,7 @@ The performance database is a revisioned evidence store behind a normalized query protocol. It answers a precise question—how an architecture component or legal implementation is expected to behave in a declared context—without hiding architecture choices or calibration knobs inside a lookup table. !!! note "Design status" - The repository currently loads `HardwareProfile` directly. The request/result/store design on this page is the accepted migration target. + The general estimator still loads `HardwareProfile` directly. Static inference now distinguishes an admissible cost-provider contract from a read-only baseline contract; the Vidur CSV adapter is baseline-only. The request/result/store design on this page remains the accepted generalization target. ## Request contract diff --git a/docs/design/performance/database.zh.md b/docs/design/performance/database.zh.md index f454412..54f69b1 100644 --- a/docs/design/performance/database.zh.md +++ b/docs/design/performance/database.zh.md @@ -3,7 +3,7 @@ 性能数据库是 normalized query protocol 背后的版本化 evidence store。它回答一个精确问题——某个 architecture component 或合法 implementation 在明确 context 中预计如何表现——但不会把 architecture choice 或 calibration knob 隐藏在 lookup table 中。 !!! note "设计状态" - 当前仓库仍直接加载 `HardwareProfile`。本页定义的 request/result/store 是已接受的迁移目标。 + 通用 estimator 仍直接加载 `HardwareProfile`。Static inference 已区分可参与估算的 cost-provider contract 与只读 baseline contract；Vidur CSV adapter 仅属于 baseline。本页的 request/result/store 仍是已接受的通用化目标。 ## Request Contract diff --git a/docs/design/performance/index.en.md b/docs/design/performance/index.en.md index 725c4e9..ad2a330 100644 --- a/docs/design/performance/index.en.md +++ b/docs/design/performance/index.en.md @@ -76,9 +76,9 @@ This does not mean every stage is assigned a wall-clock duration. Early stages a ## Current implementation boundary -The repository currently provides a typed `HardwareProfile`, peak-only and system-evidence efficiency curves, block/iteration estimates, and an auditable Calculon experiment. These form an implemented validation adapter, not yet a general architecture-exploration evidence service. +The repository currently provides a typed `HardwareProfile`, peak-only and system-evidence efficiency curves, block/iteration estimates, and an auditable Calculon experiment. Static inference additionally separates an admissible `InferenceCostProvider.resolve()` contract from a read-only `InferenceBaseline.lookup()` contract. Vidur implements only the latter and is consumed by a post-hoc experiment after Blueprinting lowering and costing. These are implemented validation slices, not yet a general architecture-exploration evidence service. -The normalized request/result protocol, provider registry, evidence store, uncertainty model, discrete-event simulator, observation ingestion, and calibration service are target architecture. Their contracts should wrap and then replace direct `HardwareProfile` coupling without changing `PortablePlanIR`. +The general normalized request/result protocol, resolver/registry, evidence store, uncertainty model, discrete-event simulator, observation ingestion, and calibration service remain target architecture. The inference protocols are migration seams, not the final universal schema. Those general contracts should wrap and then replace direct `HardwareProfile` coupling without changing `PortablePlanIR` or allowing a comparison oracle into derivation. ## Design invariants diff --git a/docs/design/performance/index.zh.md b/docs/design/performance/index.zh.md index 7dd3e48..1510002 100644 --- a/docs/design/performance/index.zh.md +++ b/docs/design/performance/index.zh.md @@ -76,9 +76,9 @@ Planner 可以优化 expected latency、conservative bound 或 risk-adjusted obj ## 当前实现边界 -仓库当前提供强类型 `HardwareProfile`、peak-only 与 system-evidence efficiency curve、block/iteration estimate，以及可审计的 Calculon experiment。这些组成一条已实现 validation adapter，但尚不是通用 architecture-exploration evidence service。 +仓库当前提供强类型 `HardwareProfile`、peak-only 与 system-evidence efficiency curve、block/iteration estimate，以及可审计的 Calculon experiment。Static inference 进一步区分可参与估算的 `InferenceCostProvider.resolve()` 与只读的 `InferenceBaseline.lookup()`；Vidur 只实现后者，并在 Blueprinting lowering 和 costing 全部完成后由 post-hoc experiment 使用。这些是已实现的 validation slice，但尚不是通用 architecture-exploration evidence service。 -Normalized request/result protocol、provider registry、evidence store、uncertainty model、discrete-event simulator、observation ingestion 与 calibration service 仍属于目标架构。它们应先包裹、再替代对 `HardwareProfile` 的直接耦合，而不改变 `PortablePlanIR`。 +通用 normalized request/result protocol、resolver/registry、evidence store、uncertainty model、discrete-event simulator、observation ingestion 与 calibration service 仍属于目标架构。Inference protocol 是迁移 seam，不是最终 universal schema。通用 contract 应先包裹、再替代对 `HardwareProfile` 的直接耦合，而不改变 `PortablePlanIR`，也不允许 comparison oracle 进入推导。 ## 设计不变量 diff --git a/docs/experiments/calculon-calibration.en.md b/docs/experiments/calculon-calibration.en.md index 91adf58..f4f2123 100644 --- a/docs/experiments/calculon-calibration.en.md +++ b/docs/experiments/calculon-calibration.en.md @@ -139,8 +139,13 @@ The two largest seqsel cases err in the same direction. The next investigation s uv run python examples/calculon_calibration.py uv run python examples/calculon_calibration.py \ --output examples/calculon_calibration_result.json + +# Run the mandatory training and inference baseline gate used by CI. +uv run pytest -m baseline_regression tests/regression ``` +The original eight parametrized training regressions remain in `tests/compiler/test_calculon_calibration.py`. The repository-level gate additionally runs all eight cases as one experiment and evaluates `data/validation/baseline_regression_contract.json`: workload and Calculon equivalence, memory, paper-error budgets, evidence revision, case identity, aggregate goldens, and every `PortablePlanIR` digest are frozen together. Updating a golden is a reviewed contract change; the gate has no automatic accept-current-output mode. + Implementation map: - `compiler/models/transformer.py`: typed frontend and execution facts; @@ -148,5 +153,6 @@ Implementation map: - `compiler/lowering/transformer.py`: the two canonical derivation passes; - `compiler/analysis/cost_model.py`: peak-only and evidence-backed views; - `compiler/experiments/calculon.py`: oracle adapter, audit, and report. +- `compiler/experiments/regression.py`: strict cross-domain baseline gate and diagnostics. This is the repository's single Blueprinting/Calculon calibration path. Future comparisons must keep oracle data unavailable until workload construction and estimation complete. See [Transformer workload derivation](../design/passes/transformer.md) for the internal transformation contracts and [performance evidence](../design/performance/index.md) for the intended provider migration. diff --git a/docs/experiments/calculon-calibration.zh.md b/docs/experiments/calculon-calibration.zh.md index de6abaf..faae8b7 100644 --- a/docs/experiments/calculon-calibration.zh.md +++ b/docs/experiments/calculon-calibration.zh.md @@ -139,8 +139,13 @@ System profile：`a100_80g`；8 个 case 共用一个 evidence revision。 uv run python examples/calculon_calibration.py uv run python examples/calculon_calibration.py \ --output examples/calculon_calibration_result.json + +# 执行 CI 使用的强制训练/推理 baseline gate。 +uv run pytest -m baseline_regression tests/regression ``` +原有的 8 组参数化训练回归仍保留在 `tests/compiler/test_calculon_calibration.py`。仓库级 gate 还会把 8 个 case 作为一个完整实验运行，并检查 `data/validation/baseline_regression_contract.json`：workload/Calculon 等价性、memory、论文误差预算、evidence revision、case identity、aggregate golden 与每个 `PortablePlanIR` digest 被一起冻结。更新 golden 是必须经过 review 的 contract 变更；gate 不提供自动“接受当前输出”的模式。 + 实现映射： - `compiler/models/transformer.py`：typed frontend 与 execution facts； @@ -148,5 +153,6 @@ uv run python examples/calculon_calibration.py \ - `compiler/lowering/transformer.py`：两个 canonical derivation pass； - `compiler/analysis/cost_model.py`：peak-only 与 evidence-backed view； - `compiler/experiments/calculon.py`：oracle adapter、audit 与 report。 +- `compiler/experiments/regression.py`：严格的跨域 baseline gate 与诊断。 这是仓库唯一的 Blueprinting/Calculon calibration path。未来对比仍必须保证 workload construction 与 estimation 完成前无法访问 oracle data。内部 transformation contract 参见 [Transformer 工作负载推导](../design/passes/transformer.md)，未来 provider 迁移参见 [performance evidence](../design/performance/index.md)。 diff --git a/docs/experiments/vidur-baseline.en.md b/docs/experiments/vidur-baseline.en.md new file mode 100644 index 0000000..22328bb --- /dev/null +++ b/docs/experiments/vidur-baseline.en.md @@ -0,0 +1,86 @@ +# Vidur Raw Component-Profile Alignment + +This experiment asks a deliberately falsifiable question: **after Blueprinting independently lowers an inference phase and evaluates the resulting work, how close is its component cost to a compatible Vidur profile?** Vidur is a reference result, not an implementation dependency or a latency provider for the production analysis path. + +## Experimental boundary + +The order is fixed: + +```text +Transformer semantics + mapping + phase context + -> Blueprinting ModelIR + -> Blueprinting DistributedTaskIR + -> Blueprinting PortablePlanIR + -> Blueprinting peak-only and system-evidence costs + -> freeze plan digests and compiled estimates + -> exact Vidur baseline lookup + -> coverage and error report +``` + +`InferenceCostProvider.resolve()` is the extension point for an admissible Blueprinting performance database or hardware simulator. `InferenceBaseline.lookup()` is the external-oracle interface. `VidurProfileBaseline` implements only `lookup()`, so it cannot be supplied to `estimate_inference_phase()` by accident. + +The experiment report records `oracle_read_during_lowering = false`, `oracle_read_during_costing = false`, and `fit_against_case_outputs = false`. Per-case correction factors and Vidur durations are forbidden inputs to lowering and costing. + +## Comparison contract + +The adapter performs exact lookup over model and hardware identity, dtype, model dimensions, maximum sequence length, TP degree, batch/token shape, phase, context, attention backend, and cache block size. It also records a digest over the input CSV files and pinned upstream revision. + +Vidur's decode `kv_cache_size` is the number of cached tokens before the current token. Blueprinting's decode `context_tokens` is the number of keys visible after the current token is appended. Therefore the exact relation is: + +```text +vidur.kv_cache_size = blueprinting.context_tokens - 1 +``` + +No nearest-neighbor match, interpolation, or silent zero fill is allowed. An absent or semantically incompatible component is `not-covered`. + +## Reported metrics + +Each phase report exposes: + +- component count, matched count, and semantic coverage; +- Blueprinting's complete block cost; +- Blueprinting's subtotal over only matched components; +- Vidur's subtotal over the same component intersection; +- compiled cost excluded from comparison; +- signed absolute and relative errors at component and comparable-subtotal levels; +- non-cancelling component MAPE and maximum component error; +- model, distributed-plan, portable-plan, hardware-evidence, and baseline revisions. + +Both `PEAK_ONLY` and `SYSTEM_EVIDENCE` estimates are compared. This separates errors caused by workload lowering from errors caused by target-wide efficiency curves. + +## Regression drift gate + +The repository carries a minimal offline validation slice from Vidur commit `8383d2935bc62723a212090baa9f98ada206fc14`: Phi-2 on A100 with TP1, covering prefill at 128 tokens and decode at visible contexts 33 and 129. The fixture manifest records the upstream paths, Git blob IDs, MIT license, row/column projection rule, and SHA-256 of each local CSV. The full Vidur corpus remains external. + +`data/validation/baseline_regression_contract.json` freezes two kinds of constraint: + +- semantic-intersection and comparable-subtotal drift budgets: coverage at least 75%, comparable-subtotal MAPE at most 24%, maximum comparable-subtotal case error at most 28%, and at least 10 percentage points of improvement over peak-only; +- reviewed goldens: model/distributed/portable digests, baseline revision, comparable subtotals, per-case errors, aggregate comparable-subtotal MAPE, component MAPE, and maximum component error. + +The current three-case system-evidence comparable-subtotal MAPE is 23.58%, but the non-cancelling component MAPE is 54.08% and the maximum component error is 99.50%. The lower subtotal number contains substantial cross-component cancellation and must not be presented as component accuracy or end-to-end Vidur accuracy. The gate therefore claims only provenance, semantic-coverage, and drift detection. A legitimate model improvement must update reviewed goldens while tightening causal component errors; an automatic “accept current output” update is not allowed. + +Run the same mandatory CI gate locally with: + +```bash +uv run pytest -m baseline_regression tests/regression +``` + +## Known semantic gaps + +The current inference dialect models one dense-MHA, non-gated-MLP decoder template. Vidur profiles for GQA/MQA or gated-MLP models are intentionally rejected at incompatible components. The adapter does not yet consume Vidur all-reduce, send/receive, or CPU-overhead profiles. + +The pinned Phi-2 rows are raw component profiles, not proof that Blueprinting reproduces Phi-2's full decoder topology. Norm placement, parallel-residual structure, fusion/layout choices, and embedding/LM-head work are not represented in the current model schema. The experiment policy therefore records `topology_equivalence = not-claimed-by-raw-component-profile-alignment`. + +Vidur's public block aggregation contributes one `add_time`, while Blueprinting retains the attention residual and MLP residual as two explicit operations. Only the final MLP residual has a direct Vidur component peer; the other remains visible as excluded compiled work. This is reported as a semantic coverage gap instead of being hidden by double-counting the same baseline value. + +## How alignment should improve + +An error is fixed at the lowest causal layer: + +1. correct operation, byte, KV, or collective derivation when the workload audit is wrong; +2. add an explicit implementation choice when FlashAttention, paged attention, fusion, or kernel family changes cost; +3. improve target-wide size/shape response models using versioned observations and held-out validation; +4. extend the normalized context for runtime, topology, or concurrency effects; +5. leave a result uncovered when the two systems do not describe the same semantic object. + +The next coverage steps are gated MLP and GQA/MQA lowering, followed by collective-profile ingestion. Scheduler and queueing alignment belongs to the later serving discrete-event experiment, not this static phase baseline. diff --git a/docs/experiments/vidur-baseline.zh.md b/docs/experiments/vidur-baseline.zh.md new file mode 100644 index 0000000..fdc71d7 --- /dev/null +++ b/docs/experiments/vidur-baseline.zh.md @@ -0,0 +1,86 @@ +# Vidur 原始组件 Profile 对齐 + +这个实验回答一个刻意可证伪的问题：**Blueprinting 独立完成推理 phase 的 lowering 和 cost evaluation 之后，其 component cost 与兼容的 Vidur profile 相差多少？** Vidur 是 reference result，不是 production analysis path 的实现依赖或 latency provider。 + +## 实验边界 + +执行顺序固定为： + +```text +Transformer semantics + mapping + phase context + -> Blueprinting ModelIR + -> Blueprinting DistributedTaskIR + -> Blueprinting PortablePlanIR + -> Blueprinting peak-only and system-evidence costs + -> freeze plan digests and compiled estimates + -> exact Vidur baseline lookup + -> coverage and error report +``` + +`InferenceCostProvider.resolve()` 是 Blueprinting 自有性能数据库或硬件仿真器的扩展点；`InferenceBaseline.lookup()` 是外部 oracle 接口。`VidurProfileBaseline` 只实现 `lookup()`，因此不能被意外传入 `estimate_inference_phase()`。 + +Experiment report 会记录 `oracle_read_during_lowering = false`、`oracle_read_during_costing = false` 与 `fit_against_case_outputs = false`。Per-case correction factor 和 Vidur duration 都是 lowering/costing 的禁止输入。 + +## 对比契约 + +Adapter 对 model/hardware identity、dtype、model dimension、maximum sequence length、TP degree、batch/token shape、phase、context、attention backend 与 cache block size 做 exact lookup，并记录输入 CSV 与固定 upstream revision 的组合 digest。 + +Vidur 的 decode `kv_cache_size` 是当前 token 加入前已经缓存的 token 数；Blueprinting 的 decode `context_tokens` 是加入当前 token 后 attention 可见的 key 数。因此精确关系是： + +```text +vidur.kv_cache_size = blueprinting.context_tokens - 1 +``` + +不允许 nearest-neighbor match、interpolation 或 silent zero fill。缺失或语义不兼容的 component 保持 `not-covered`。 + +## 报告指标 + +每个 phase report 暴露： + +- component 总数、matched 数量与 semantic coverage； +- Blueprinting 完整 block cost； +- Blueprinting 在 matched component 上的 subtotal； +- Vidur 在相同 component intersection 上的 subtotal； +- 未进入比较的 compiled cost； +- component 与 comparable-subtotal 层的 signed absolute/relative error； +- 不可相互抵消的 component MAPE 与最大 component error； +- model、distributed plan、portable plan、hardware evidence 与 baseline revision。 + +实验同时比较 `PEAK_ONLY` 和 `SYSTEM_EVIDENCE` estimate，用于区分 workload lowering 错误与 target-wide efficiency curve 错误。 + +## 回归漂移门禁 + +仓库保存了来自 Vidur commit `8383d2935bc62723a212090baa9f98ada206fc14` 的最小离线 validation slice：A100 上的 Phi-2、TP1，覆盖 128-token prefill，以及 visible context 为 33 和 129 的 decode。Fixture manifest 记录 upstream path、Git blob ID、MIT license、行列投影规则和每个本地 CSV 的 SHA-256；完整 Vidur corpus 仍保持外部依赖。 + +`data/validation/baseline_regression_contract.json` 固定两类约束： + +- semantic intersection 与 comparable-subtotal drift budget：coverage 至少 75%，comparable-subtotal MAPE 不超过 24%，单 case 最大 comparable-subtotal error 不超过 28%，并且相比 peak-only 至少改善 10 个百分点； +- reviewed golden：model/distributed/portable digest、baseline revision、comparable subtotal、逐 case error、aggregate comparable-subtotal MAPE、component MAPE 与最大 component error。 + +当前三个 case 的 system-evidence comparable-subtotal MAPE 为 23.58%，但不可相互抵消的 component MAPE 是 54.08%，最大 component error 是 99.50%。较低的 subtotal 数值包含显著的跨 component 误差抵消，不能被描述为 component accuracy，更不是 Vidur end-to-end accuracy。这个 gate 只声明 provenance、semantic coverage 与 drift detection。合法改进必须更新 reviewed golden，并收紧有因果意义的 component error；不允许自动执行“接受当前输出”。 + +本地执行与 CI 相同的强制 gate： + +```bash +uv run pytest -m baseline_regression tests/regression +``` + +## 已知语义缺口 + +当前 inference dialect 建模一个 dense-MHA、non-gated-MLP decoder template。对于 GQA/MQA 或 gated-MLP model 的 Vidur profile，不兼容 component 会被明确拒绝。Adapter 尚未消费 Vidur all-reduce、send/receive 与 CPU-overhead profile。 + +固定的 Phi-2 数据是 raw component profile，并不能证明 Blueprinting 已复现 Phi-2 的完整 decoder topology。当前 model schema 还没有表达 norm placement、parallel-residual structure、fusion/layout choice 与 embedding/LM-head work。因此 experiment policy 显式记录 `topology_equivalence = not-claimed-by-raw-component-profile-alignment`。 + +Vidur 公开的 block aggregation 只贡献一个 `add_time`，而 Blueprinting 将 attention residual 与 MLP residual 保留为两个显式 operation。只有最终 MLP residual 拥有直接的 Vidur component peer；另一个仍作为 excluded compiled work 可见。系统把它报告为 semantic coverage gap，而不会通过重复使用同一个 baseline value 来掩盖差异。 + +## 如何改进对齐 + +误差应在最低的因果层修复： + +1. workload audit 错误时，修正 operation、byte、KV 或 collective 推导； +2. FlashAttention、paged attention、fusion 或 kernel family 改变 cost 时，引入显式 implementation choice； +3. 使用版本化 observation 与 held-out validation 改进 target-wide size/shape response model； +4. runtime、topology 或 concurrency 影响结果时，扩展 normalized context； +5. 两套系统没有描述同一个 semantic object 时，保持 uncovered。 + +下一步 coverage 是 gated MLP 与 GQA/MQA lowering，随后接入 collective profile。Scheduler 和 queueing 的对齐属于之后的 serving discrete-event experiment，不属于这个静态 phase baseline。 diff --git a/docs/modeling/inference.en.md b/docs/modeling/inference.en.md new file mode 100644 index 0000000..772f7e5 --- /dev/null +++ b/docs/modeling/inference.en.md @@ -0,0 +1,122 @@ +# Inference Planning and Serving Simulation + +Blueprinting now has a runnable decoder-inference slice, but its boundary is intentionally narrower than a serving-system simulator. The implemented path answers a hardware-planning question: **what work, communication, state capacity, and static latency does one prefill or decode phase point induce under a declared mapping?** It does not yet claim to predict queueing, continuous batching, or SLO tails. + +![Inference planning path and serving-simulation boundary](../assets/architecture/inference-planning-path.svg) + +## What we adopt from related work + +[LLMCompass](https://arxiv.org/abs/2312.03134) demonstrates why LLM inference hardware evaluation needs separate software, hardware, mapping, and cost concerns, plus an explicit mapping search rather than a single closed-form model. Blueprinting adopts that separation. Its canonical representations preserve workload and mapping facts before a hardware profile or measured latency is consulted. LLMCompass's area/cost and architecture design-space machinery remains future provider and exploration work; its artifact code is not copied into the canonical IR. + +[Vidur](https://github.com/microsoft/vidur) demonstrates a complementary boundary: request arrivals, replica scheduling, batching, and event progression are a discrete-event layer, while execution time is supplied by component predictors trained from profiling data. Blueprinting adopts that split. Phase plans are the stable cost subjects; a future serving simulator will schedule requests and batches against them rather than redefining Transformer work inside scheduler code. + +The resulting boundary is deliberate: + +| Concern | Current owner | Status | +|---|---|---| +| Transformer operation/byte/collective derivation | canonical inference analysis | Implemented slice | +| Prefill and decode specialization | workload binding + lowering passes | Implemented slice | +| KV-cache state and capacity | ModelIR effect + portable state buffer + memory view | Implemented slice | +| Analytical component cost | `HardwareProfile` fallback | Implemented slice | +| Vidur profiling CSV reuse | post-hoc exact-match baseline | Implemented experiment | +| Static decoder-block phase composition | inference application service | Implemented slice | +| Arrivals, queues, continuous batching, scheduling | serving discrete-event simulator | Planned | +| Hardware area, power, cost, and mapping search | architecture providers and exploration session | Planned | + +## Implemented derivation path + +The model frontend emits one phase-neutral `transformer.decoder_inference` operation with an explicit KV-cache state effect. A phase workload binding then specializes it: + +```text +TransformerModelSpec + + TransformerInferenceExecutionSpec(TP, PP, replicas, dtype, network tiers) + + WorkloadBinding(INFERENCE, PREFILL | DECODE, batch, context) + -> ModelIR + -> DistributeTransformerInferencePass + -> DistributedTaskIR + -> PlanTransformerInferencePass + -> PortablePlanIR + -> estimate_inference_phase(Blueprinting cost provider | analytical model) + -> optional post-hoc Vidur comparison +``` + +The component tasks are input norm, QKV projection, RoPE, KV save, attention core, output projection, TP all-reduce, residual, post-attention norm, MLP up/activation/down, a second all-reduce, and the final residual. This boundary is fine enough to inspect work conservation and broad enough to match observable kernel families in profiling systems. + +Prefill binds `query_tokens = context_tokens = prompt_tokens`. Decode binds `query_tokens = 1` and treats `context_tokens` as the number of keys visible after the current token is appended. Every phase plan carries exact operations, read/write bytes, collective volume, phase, primitive, source layer, query length, context length, block weight capacity, KV capacity, and a conservative workspace buffer. It carries no duration. Costing reconstructs its task view from `PlanTask.workload` and explicit buffers; a hidden lowering object is not allowed to become a second workload truth. + +## Request composition semantics + +For a cohort with prompt length `S` and requested output length `O`, the current decoder-block dialect composes one prefill phase and `O-1` decode phases: + +```text +prefill model time = cost(prefill(batch=B, query=S, context=S)) +decode contexts = S+1, S+2, ..., S+O-1 +model execution time = prefill model time + sum(cost(decode(batch=B, query=1, context=c))) +mean decode-step model time = decode total / (O-1), when O > 1 +``` + +Compiling each decode context separately is intentional in this first slice: size-dependent efficiency need not be linear in context. The report retains every decode plan digest and latency, while the full IR snapshots are exposed for prefill and the final decode context. Because embedding, LM head, sampling, host work, and queueing are absent, these values are deliberately not named TTFT, TPOT, or E2E. This is static model-phase composition, not a serving trace simulation. + +Per-device capacity is derived from the mapping. Dense MHA KV storage for one local block shard is: + +```text +2 × batch × context × (hidden / TP) × bytes_per_element +``` + +It is multiplied by the number of blocks in one pipeline stage. Weight storage is likewise sharded by TP and partitioned by PP; pipeline-boundary buffers remain explicit. Because `PortablePlanIR` has not selected FlashAttention, paged attention, or an unfused implementation yet, working memory is reported as a conservative unfused score-materialization upper bound. Target binding must replace that bound with implementation-specific workspace. The current path requires TP to divide hidden, FFN, and head dimensions, and PP to divide the block count. + +## Using Vidur as a baseline, not as the estimator + +`VidurProfileBaseline.from_csv(...)` consumes user-supplied Vidur `attention.csv` and compute/MLP CSV files. The caller must pin an upstream revision, hardware identity, attention backend, and cache block size. The adapter hashes the inputs and identity into a baseline revision, converts Vidur's millisecond medians to seconds, and only returns a reference when model dimensions, maximum sequence length, TP, batch/token shape, phase, backend, block size, and context match exactly. Vidur records decode `kv_cache_size` before the current token is appended; Blueprinting records the visible context after append, so the adapter makes the explicit relation `vidur_kv_cache_size = context_tokens - 1`. + +```python +from blueprinting.compiler.analysis import HardwareProfile, VidurProfileBaseline +from blueprinting.compiler.bindings import InferencePhase +from blueprinting.compiler.experiments import VidurExperimentCase, run_vidur_experiment +from blueprinting.compiler.models import TransformerInferenceExecutionSpec, TransformerModelSpec + +baseline = VidurProfileBaseline.from_csv( + attention_csv="/profiles/attention.csv", + compute_csv="/profiles/mlp.csv", + model_name="", + hardware_name="a100_80g", + attention_backend="AttentionBackend.FLASH_ATTENTION", + block_size=16, + source_revision="", +) +case = VidurExperimentCase( + name="decode/context-128", + model=TransformerModelSpec(...), + execution=TransformerInferenceExecutionSpec(...), + hardware=HardwareProfile(...), + phase=InferencePhase.DECODE, + batch_size=1, + context_tokens=128, +) +report = run_vidur_experiment((case,), baseline) +``` + +The API boundary is intentional: an admissible internal `InferenceCostProvider` exposes `resolve()`, while an external `InferenceBaseline` exposes `lookup()`. `run_vidur_experiment()` completes lowering and both Blueprinting cost modes before calling `lookup()`. Vidur therefore cannot alter operations, bytes, dependencies, the plan digest, or the compiled latency. + +Comparison is over an explicit semantic intersection. The report contains matched component count, coverage, Blueprinting's comparable subtotal, Vidur's comparable subtotal, excluded Blueprinting work, signed comparable-subtotal error, and non-cancelling component MAPE/max error. Missing records remain `not-covered`; they are never converted to zero. This matters because Vidur's public block aggregation has one `add_time`, whereas Blueprinting deliberately keeps both residual additions explicit, and the current CSV adapter does not yet ingest collective profiles. + +There is no nearest-neighbor or hidden interpolation. An MHA workload requires equal query/KV head counts in both compute and attention records. Matching raw component-profile keys does not establish full decoder-topology equivalence: the current model spec does not yet encode norm placement, residual topology, or gated-MLP choice. The current production inference estimate remains entirely Blueprinting-owned; Vidur is an oracle for measuring where that estimate must improve. + +Blueprinting does not vendor the full upstream profiling corpus. A minimal MIT-licensed Phi-2/A100 validation slice is retained for offline CI, with a pinned upstream commit, source blob IDs, an explicit projection rule, and local file digests. Larger experiments keep Vidur data external. A future ingestion command should add environment manifests, units, runtime/kernel versions, and raw-record IDs before profiles enter the general performance database. + +## What the serving layer must add + +A Vidur-like serving simulator should be a consumer of phase plans and evidence, not another Transformer estimator. Its minimum state is: + +- immutable request records: arrival, prompt/output lengths, priority, and SLO; +- replica and KV allocation state; +- scheduler policy with explicit batching/chunked-prefill decisions; +- an event queue for admission, batch start/end, transfer, preemption, and completion; +- CPU/scheduler overhead evidence separated from device component evidence; +- TTFT, inter-token latency, E2E, throughput, utilization, and tail distributions. + +The scheduler produces a concrete batch context and asks the cost resolver for that context. This allows vLLM-, Orca-, Sarathi-, or future LPU-oriented policies to share the same canonical workload semantics and hardware evidence interfaces. + +## Current limitations + +The implemented dialect covers one dense-MHA, non-gated-MLP decoder template. Embedding, LM head, sampler, explicit norm/residual topology, GQA/MQA, gated MLP, MoE, prefix caching, paged allocation, chunked prefill, speculative decoding, disaggregated prefill/decode, scheduler overhead, and resource contention are not modeled yet. PP and replica structure are not fully materialized in `DistributedTaskIR`; PP latency/memory composition is currently analytical after the local-TP block plan. Each decode context is recompiled rather than algebraically specialized from a parametric plan. `replicas` currently participates only in mapping validation and world-size accounting; reported latency and static model token rate remain single-replica views, not multi-replica serving capacity. Consequently, the current output is suitable for inspecting derivation, analytical memory fit, and first-order hardware sensitivity—not for claiming production serving SLO accuracy. diff --git a/docs/modeling/inference.zh.md b/docs/modeling/inference.zh.md new file mode 100644 index 0000000..3919ed7 --- /dev/null +++ b/docs/modeling/inference.zh.md @@ -0,0 +1,122 @@ +# 推理规划与 Serving 仿真 + +Blueprinting 现在已经具备一条可运行的 decoder inference 切片，但它的边界刻意窄于完整 serving-system simulator。当前路径回答的是硬件规划问题：**在明确 mapping 下，一个 prefill 或某个 context length 的 decode phase 会产生多少计算、通信、状态容量和静态延迟？** 它暂不声称能够预测排队、continuous batching 或 SLO tail。 + +![推理规划路径与 serving-simulation 边界](../assets/architecture/inference-planning-path.svg) + +## 从相关工作中吸收什么 + +[LLMCompass](https://arxiv.org/abs/2312.03134)说明，LLM 推理硬件评估需要分离 software、hardware、mapping 与 cost，并通过显式 mapping search 取代单一闭式模型。Blueprinting 吸收了这个分层：canonical representation 先保存 workload 与 mapping 事实，之后才允许 hardware profile 或测量 latency 参与。LLMCompass 的 area/cost 与 architecture design-space machinery 属于后续 provider 和 exploration 工作；本项目没有把其 artifact code 复制进 canonical IR。 + +[Vidur](https://github.com/microsoft/vidur)提供了另一条关键边界：request arrival、replica scheduling、batching 和 event progression 属于离散事件层，execution time 则由基于 profiling data 的 component predictor 提供。Blueprinting 吸收了这层分离：phase plan 是稳定的 cost subject；未来 serving simulator 在其上调度 request/batch，而不是在 scheduler 代码里重新定义 Transformer work。 + +由此得到的职责边界如下： + +| 关注点 | 当前所有者 | 状态 | +|---|---|---| +| Transformer operation/byte/collective 推导 | canonical inference analysis | Implemented slice | +| Prefill 与 decode 特化 | workload binding + lowering passes | Implemented slice | +| KV-cache state 与容量 | ModelIR effect + portable state buffer + memory view | Implemented slice | +| 解析式 component cost | `HardwareProfile` fallback | Implemented slice | +| Vidur profiling CSV 复用 | post-hoc exact-match baseline | Implemented experiment | +| 静态 decoder-block phase composition | inference application service | Implemented slice | +| Arrival、queue、continuous batching、scheduling | serving discrete-event simulator | Planned | +| 硬件 area、power、cost 与 mapping search | architecture provider 与 exploration session | Planned | + +## 已实现的推导路径 + +Frontend 生成一个 phase-neutral 的 `transformer.decoder_inference` operation，并显式声明 KV-cache state effect。Phase workload binding 再对其特化： + +```text +TransformerModelSpec + + TransformerInferenceExecutionSpec(TP, PP, replicas, dtype, network tiers) + + WorkloadBinding(INFERENCE, PREFILL | DECODE, batch, context) + -> ModelIR + -> DistributeTransformerInferencePass + -> DistributedTaskIR + -> PlanTransformerInferencePass + -> PortablePlanIR + -> estimate_inference_phase(Blueprinting cost provider | analytical model) + -> optional post-hoc Vidur comparison +``` + +Component task 包括 input norm、QKV projection、RoPE、KV save、attention core、output projection、TP all-reduce、residual、post-attention norm、MLP up/activation/down、第二次 all-reduce 与最终 residual。这个边界既足以检查 work conservation，也能与 profiling system 中可观测的 kernel family 对齐。 + +Prefill 绑定 `query_tokens = context_tokens = prompt_tokens`。Decode 绑定 `query_tokens = 1`，并把 `context_tokens` 定义为追加当前 token 后 attention 可见的 key 数。每个 phase plan 保存精确 operations、read/write bytes、collective volume、phase、primitive、source layer、query/context length、block weight capacity、KV capacity 与保守 workspace buffer；它不携带 duration。Costing 只从 `PlanTask.workload` 与显式 buffer 重建 task view，不允许隐藏 lowering object 成为第二份 workload 真值。 + +## Request composition 语义 + +对于 prompt 长度 `S`、请求输出长度 `O` 的 cohort，当前 decoder-block dialect 组合一个 prefill phase 与 `O-1` 个 decode phase： + +```text +prefill model time = cost(prefill(batch=B, query=S, context=S)) +decode contexts = S+1, S+2, ..., S+O-1 +model execution time = prefill model time + sum(cost(decode(batch=B, query=1, context=c))) +mean decode-step model time = decode total / (O-1), when O > 1 +``` + +第一版逐个编译 decode context 是有意选择：size-dependent efficiency 不必关于 context 线性。Report 保留每个 decode plan digest 与 latency，并暴露 prefill 和最终 decode context 的完整 IR snapshot。由于 embedding、LM head、sampling、host work 与 queueing 都未建模，这些数值刻意不命名为 TTFT、TPOT 或 E2E。这是静态 model-phase composition，不是 serving trace simulation。 + +每设备容量从 mapping 推导。Dense MHA 的单个 local block shard KV 存储为： + +```text +2 × batch × context × (hidden / TP) × bytes_per_element +``` + +该值乘以每个 pipeline stage 的 block 数。Weight storage 同样按 TP sharding、按 PP partition，pipeline boundary buffer 显式保留。由于 `PortablePlanIR` 尚未选择 FlashAttention、paged attention 或 unfused implementation，working memory 当前报告未融合 score materialization 的保守上界；target binding 必须用 implementation-specific workspace 替换该上界。当前路径要求 TP 整除 hidden、FFN 和 head dimension，并要求 PP 整除 block count。 + +## 把 Vidur 作为基线，而不是 estimator + +`VidurProfileBaseline.from_csv(...)` 读取用户提供的 Vidur `attention.csv` 与 compute/MLP CSV。调用者必须固定 upstream revision、hardware identity、attention backend 与 cache block size。Adapter 把输入文件和 identity 一起哈希为 baseline revision，将 Vidur 的毫秒 median 转为秒；只有 model dimension、maximum sequence length、TP、batch/token shape、phase、backend、block size 与 context 完全匹配时才返回 reference。Vidur 的 decode `kv_cache_size` 表示当前 token 写入前的长度，而 Blueprinting 的 context 表示写入后 attention 可见的长度，因此 adapter 显式使用 `vidur_kv_cache_size = context_tokens - 1`。 + +```python +from blueprinting.compiler.analysis import HardwareProfile, VidurProfileBaseline +from blueprinting.compiler.bindings import InferencePhase +from blueprinting.compiler.experiments import VidurExperimentCase, run_vidur_experiment +from blueprinting.compiler.models import TransformerInferenceExecutionSpec, TransformerModelSpec + +baseline = VidurProfileBaseline.from_csv( + attention_csv="/profiles/attention.csv", + compute_csv="/profiles/mlp.csv", + model_name="", + hardware_name="a100_80g", + attention_backend="AttentionBackend.FLASH_ATTENTION", + block_size=16, + source_revision="", +) +case = VidurExperimentCase( + name="decode/context-128", + model=TransformerModelSpec(...), + execution=TransformerInferenceExecutionSpec(...), + hardware=HardwareProfile(...), + phase=InferencePhase.DECODE, + batch_size=1, + context_tokens=128, +) +report = run_vidur_experiment((case,), baseline) +``` + +这个 API 边界是刻意设计的：Blueprinting 内部可接受的 `InferenceCostProvider` 暴露 `resolve()`，外部 `InferenceBaseline` 只暴露 `lookup()`。`run_vidur_experiment()` 会先完成 lowering 和两种 Blueprinting cost mode，再调用 `lookup()`；因此 Vidur 无法改变 operations、bytes、dependency、plan digest 或 compiled latency。 + +Comparison 只发生在显式 semantic intersection 上。Report 给出 matched component count、coverage、Blueprinting comparable subtotal、Vidur comparable subtotal、被排除的 Blueprinting work、signed comparable-subtotal error，以及不可相互抵消的 component MAPE/max error。缺失 record 保持 `not-covered`，绝不会被当作零。这个区别很重要：Vidur 公开的 block aggregation 只有一个 `add_time`，而 Blueprinting 刻意保留两个 residual addition；当前 CSV adapter 也尚未读取 collective profile。 + +Adapter 不做 nearest-neighbor 或隐藏插值。MHA workload 还要求 compute 与 attention record 的 query/KV head 数相等。Raw component-profile key 匹配并不证明完整 decoder topology 等价：当前 model spec 还没有编码 norm placement、residual topology 与 gated-MLP choice。Production inference estimate 完全由 Blueprinting 自己产生；Vidur 只是衡量这套机制还应在哪里改进的 oracle。 + +Blueprinting 不复制完整 upstream profiling corpus。仓库只保留一份 MIT-licensed Phi-2/A100 最小 validation slice，用于离线 CI，并固定 upstream commit、source blob ID、显式 projection rule 与本地文件 digest；更大规模实验继续让 Vidur 数据保持外部依赖。未来 ingestion command 应在 profile 进入通用性能数据库前补齐 environment manifest、unit、runtime/kernel version 与 raw-record ID。 + +## Serving 层还必须增加什么 + +Vidur 风格的 serving simulator 应当消费 phase plan 与 evidence，而不是成为另一套 Transformer estimator。它的最小状态包括： + +- immutable request record：arrival、prompt/output length、priority 与 SLO； +- replica 与 KV allocation state； +- 具有显式 batching/chunked-prefill 决策的 scheduler policy； +- admission、batch start/end、transfer、preemption 和 completion event queue； +- 与 device component evidence 分开的 CPU/scheduler overhead evidence； +- TTFT、inter-token latency、E2E、throughput、utilization 与 tail distribution。 + +Scheduler 产生 concrete batch context，再使用该 context 查询 cost resolver。这样 vLLM、Orca、Sarathi 或未来面向 LPU 的策略可以共享同一份 canonical workload semantic 与 hardware evidence interface。 + +## 当前限制 + +已实现 dialect 覆盖一个 dense-MHA、non-gated-MLP decoder template。Embedding、LM head、sampler、显式 norm/residual topology、GQA/MQA、gated MLP、MoE、prefix caching、paged allocation、chunked prefill、speculative decoding、prefill/decode disaggregation、scheduler overhead 与 resource contention 尚未建模。PP 与 replica structure 还没有完整物化到 `DistributedTaskIR`；PP latency/memory 当前是在 local-TP block plan 之后做解析式组合。每个 decode context 仍会重新编译，而不是从 parametric plan 做代数特化。`replicas` 当前只参与 mapping 合法性与 world-size 记账；报告的 latency 与 static model token rate 仍是单 replica 视角，不代表多 replica serving capacity。因此当前输出适合检查推导、解析显存 fit 和一阶硬件敏感性，不能作为 production serving SLO accuracy 的声明。 diff --git a/docs/project/status.en.md b/docs/project/status.en.md index f9a209a..614fbb1 100644 --- a/docs/project/status.en.md +++ b/docs/project/status.en.md @@ -19,8 +19,10 @@ This page separates Blueprinting's hardware-exploration product goals from the e | Hardware-exploration capability | Status | Current evidence or gap | |---|---|---| | Typed Transformer training workload accounting | **Implemented** | exact block operations, bytes, collectives, recomputation, and phases | +| Static Transformer inference phase planning | **Implemented slice** | independently verified prefill/decode plans, KV capacity, and decoder-block phase composition | | Target-neutral workload/mapping plan | **Implemented slice** | Transformer path reaches `PortablePlanIR` | | Versioned compute/memory/network efficiency profile | **Implemented adapter** | `HardwareProfile` and two analytical estimate modes | +| Vidur raw component-profile alignment | **Implemented experiment** | exact-key CSV lookup after independent lowering/costing, with component coverage and non-cancelling error attribution | | Calculon/SeqSel workload and cost calibration | **Implemented experiment** | eight-case reproducible report and tests | | First-class hierarchical `ArchitectureBlueprint` | **Planned** | documented component model; no production schema/API | | Hardware design variables and constraint-aware candidate generation | **Planned** | no design-space generator or search session | @@ -29,7 +31,7 @@ This page separates Blueprinting's hardware-exploration product goals from the e | Architecture-bound placement, schedule, and memory plan | **Experimental Contract / Planned** | `ConcretePlanIR` has only a generic queue-oriented schema and structural verifier; producer, route/occupancy semantics, and typed target extensions do not exist | | Discrete-event compute/memory/resource simulation | **Planned** | current result is analytical composition, not event simulation | | Timeline analysis/replay bundle | **Planned** | `TimingProjection`, `SimulationTraceIR`, and `TimelineBundle` are design contracts only | -| Network/hardware simulator adapters | **Planned** | normalized provider protocol is documented only | +| General network/hardware simulator adapters | **Implemented slice / Planned** | inference cost/baseline protocols exist; general resolver, validity/uncertainty model, and simulator adapters do not | | Bottleneck, utilization, sensitivity, and what-if reports | **Planned** | no general architecture report product | | Energy, area, power, thermal, and cost models | **Planned** | dimensions are specified but no providers exist | | Multi-objective Pareto architecture search | **Planned** | no candidate frontier API | @@ -44,7 +46,7 @@ The five current IR classes use `1.0.0` as an internal canonical serialization v ## Connected analysis path -The current runnable slice is: +The current runnable slices are: ```text TransformerModelSpec + TransformerExecutionSpec @@ -54,15 +56,23 @@ TransformerModelSpec + TransformerExecutionSpec -> PortablePlanIR -> HardwareProfile analytical estimate -> Calculon / paper comparison report + +TransformerModelSpec + inference mapping + request cohort + -> phase-neutral inference ModelIR + -> independently bound prefill and decode DistributedTaskIR + -> phase-local PortablePlanIR with KV state/capacity + -> Blueprinting HardwareProfile/cost-provider estimate + -> optional post-hoc Vidur baseline comparison + -> static prefill / decode-step model time and analytical memory report ``` `PassManager` verifies each staged derivation and exposes immutable checkpoints. `HardwareProfile` supplies evidence after the portable plan. The path does not yet construct an architecture hierarchy, bind physical resources, execute a discrete-event simulation, or search hardware candidates. ## What the current result can claim -The repository can claim that selected Transformer training workloads are decomposed into auditable target-neutral work and compared against one versioned system evidence profile without case-specific timing coefficients. It also provides the stable identities, verifier gates, and pass-level checkpoint hooks needed for later simulation correlation. +The repository can claim that selected Transformer training workloads are decomposed into auditable target-neutral work and compared against one versioned system evidence profile without case-specific timing coefficients. It can also compile dense-MHA inference prefill and decode phase points, derive KV capacity, and compose a homogeneous request cohort with explicit evidence provenance. It provides the stable identities, verifier gates, and pass-level checkpoint hooks needed for later simulation correlation. -It cannot yet claim that Blueprinting explores compute/memory/interconnect parameters, predicts NoC or network contention, models energy/area/cost, constructs a legal concrete hardware schedule, produces sensitivity/Pareto results, or closes a calibration loop on real GPU/LPU observations. +It cannot yet claim serving-system SLO accuracy: arrivals, queueing, continuous batching, scheduler overhead, contention, and tail distributions are absent. Nor can it yet claim that Blueprinting explores compute/memory/interconnect parameters, predicts NoC behavior, models energy/area/cost, constructs a legal concrete hardware schedule, produces sensitivity/Pareto results, or closes a calibration loop on real GPU/LPU observations. ## Engineering foundation @@ -75,13 +85,16 @@ It cannot yet claim that Blueprinting explores compute/memory/interconnect param | Transformer semantic frontend and workload algebra | **Implemented slice** | `models/transformer.py`, `analysis/transformer_workload.py` | | Distributed and portable mapping derivations | **Implemented slice** | `lowering/transformer.py` | | Current evidence adapter | **Implemented slice** | `analysis/cost_model.py` | +| Static inference frontend, lowering, cost, and request composition | **Implemented slice** | `models/transformer_inference.py`, `analysis/{transformer_inference,inference_cost}.py`, `lowering/transformer_inference.py`, `application/inference.py` | +| Vidur raw component-profile alignment | **Implemented experiment** | `analysis/vidur.py` + `experiments/vidur.py`; a minimal licensed CI slice is pinned locally and the full upstream corpus remains external | | Calculon experiment | **Implemented experiment** | `experiments/calculon.py` | +| External-baseline regression gate | **Implemented** | frozen contract and licensed offline fixtures under `data/validation/`; `experiments/regression.py`; `.github/workflows/quality.yml` | These typed representations, verifiers, derivation transactions, and analyses are the formal foundation for hardware exploration. New architecture models, simulator providers, and analysis products should extend this one semantic foundation rather than establish parallel workload truth. ## Verification baseline -The current test suite covers binding consistency, canonical serialization, verifier rejection, pass transaction rollback, checkpoint observers, workload conservation, and the Calculon calibration results. Documentation checks enforce complete bilingual page pairs and strict site builds. +The current test suite covers binding consistency, canonical serialization, verifier rejection, pass transaction rollback, checkpoint observers, workload conservation, Calculon calibration, prefill/decode scaling, KV capacity, static request composition, and baseline-only Vidur comparison. A dedicated CI job runs the eight-case Calculon/SeqSel and three-case pinned Vidur gates on every main-branch pull request and push. It freezes provenance, semantic policy, coverage, comparable-subtotal drift budgets, non-cancelling component errors, aggregate results, and IR digests; it cannot silently regenerate goldens. The Vidur gate is drift detection, not an accuracy certification. Documentation checks enforce complete bilingual page pairs and strict site builds. Status promotion requires an end-to-end product test. For example, introducing `ArchitectureBlueprint` as a dataclass is Contract Only; constructing two different candidates, mapping the same workload, producing comparable results, and preserving provenance is the minimum product-level evidence. diff --git a/docs/project/status.zh.md b/docs/project/status.zh.md index 7d84944..d386d07 100644 --- a/docs/project/status.zh.md +++ b/docs/project/status.zh.md @@ -19,8 +19,10 @@ | 硬件探索能力 | 状态 | 当前证据或差距 | |---|---|---| | Typed Transformer training workload accounting | **Implemented** | 精确 block operation、byte、collective、recomputation 与 phase | +| Static Transformer inference phase planning | **Implemented slice** | 独立验证的 prefill/decode plan、KV 容量以及 decoder-block phase composition | | Target-neutral workload/mapping plan | **Implemented slice** | Transformer path 到达 `PortablePlanIR` | | 版本化 compute/memory/network efficiency profile | **Implemented adapter** | `HardwareProfile` 与两种 analytical estimate mode | +| Vidur raw component-profile 对齐 | **Implemented experiment** | 独立 lowering/costing 后进行 exact-key CSV lookup，并报告 component coverage 与不可抵消的误差归因 | | Calculon/SeqSel workload 与 cost calibration | **Implemented experiment** | 8 case 可复现 report 与 test | | First-class hierarchical `ArchitectureBlueprint` | **Planned** | 已定义 component model；无 production schema/API | | Hardware design variable 与 constraint-aware candidate generation | **Planned** | 无 design-space generator 或 search session | @@ -29,7 +31,7 @@ | Architecture-bound placement、schedule 与 memory plan | **Experimental Contract / Planned** | `ConcretePlanIR` 只有通用 queue-oriented schema 与 structural verifier；producer、route/occupancy semantic 和 typed target extension 尚无 | | Discrete-event compute/memory/resource simulation | **Planned** | 当前结果是 analytical composition，不是 event simulation | | Timeline analysis/replay bundle | **Planned** | `TimingProjection`、`SimulationTraceIR`、`TimelineBundle` 只有 design contract | -| Network/hardware simulator adapter | **Planned** | normalized provider protocol 只有文档设计 | +| 通用 network/hardware simulator adapter | **Implemented slice / Planned** | inference cost/baseline protocol 已存在；通用 resolver、validity/uncertainty model 和 simulator adapter 尚无 | | Bottleneck、utilization、sensitivity 与 what-if report | **Planned** | 无通用 architecture report product | | Energy、area、power、thermal 与 cost model | **Planned** | 已定义维度，但无 provider | | Multi-objective Pareto architecture search | **Planned** | 无 candidate frontier API | @@ -44,7 +46,7 @@ ## 已贯通分析路径 -当前可运行切片是： +当前存在两条可运行切片： ```text TransformerModelSpec + TransformerExecutionSpec @@ -54,15 +56,23 @@ TransformerModelSpec + TransformerExecutionSpec -> PortablePlanIR -> HardwareProfile analytical estimate -> Calculon / paper comparison report + +TransformerModelSpec + inference mapping + request cohort + -> phase-neutral inference ModelIR + -> 分别绑定的 prefill/decode DistributedTaskIR + -> 携带 KV state/capacity 的 phase-local PortablePlanIR + -> Blueprinting HardwareProfile/cost-provider estimate + -> optional post-hoc Vidur baseline comparison + -> 静态 prefill / decode-step model time 与解析 memory report ``` `PassManager` 验证每次 staged derivation 并暴露 immutable checkpoint。`HardwareProfile` 在 portable plan 后提供 evidence。当前路径尚未构造 architecture hierarchy、绑定 physical resource、执行 discrete-event simulation 或搜索 hardware candidate。 ## 当前结果可以声称什么 -仓库可以声称：选定 Transformer training workload 被分解为可审计 target-neutral work，并在不使用 case-specific timing coefficient 的前提下与一个版本化 system evidence profile 比较；系统也提供了后续 simulation correlation 所需的 stable identity、verifier gate 与 pass-level checkpoint hook。 +仓库可以声称：选定 Transformer training workload 被分解为可审计 target-neutral work，并在不使用 case-specific timing coefficient 的前提下与一个版本化 system evidence profile 比较；系统也可以编译 dense-MHA inference 的 prefill/decode phase point、推导 KV 容量，并以显式 evidence provenance 组合 homogeneous request cohort。系统提供了后续 simulation correlation 所需的 stable identity、verifier gate 与 pass-level checkpoint hook。 -当前还不能声称 Blueprinting 已经探索 compute/memory/interconnect parameter、预测 NoC/network contention、建模 energy/area/cost、构造合法 concrete hardware schedule、产生 sensitivity/Pareto result，或用真实 GPU/LPU observation 闭合 calibration loop。 +当前还不能声称 serving-system SLO accuracy：arrival、queueing、continuous batching、scheduler overhead、contention 与 tail distribution 均未实现。也不能声称 Blueprinting 已经探索 compute/memory/interconnect parameter、预测 NoC 行为、建模 energy/area/cost、构造合法 concrete hardware schedule、产生 sensitivity/Pareto result，或用真实 GPU/LPU observation 闭合 calibration loop。 ## 工程基础 @@ -75,13 +85,16 @@ TransformerModelSpec + TransformerExecutionSpec | Transformer semantic frontend 与 workload algebra | **Implemented slice** | `models/transformer.py`、`analysis/transformer_workload.py` | | Distributed/portable mapping derivation | **Implemented slice** | `lowering/transformer.py` | | 当前 evidence adapter | **Implemented slice** | `analysis/cost_model.py` | +| Static inference frontend、lowering、cost 与 request composition | **Implemented slice** | `models/transformer_inference.py`、`analysis/{transformer_inference,inference_cost}.py`、`lowering/transformer_inference.py`、`application/inference.py` | +| Vidur raw component-profile 对齐 | **Implemented experiment** | `analysis/vidur.py` + `experiments/vidur.py`；最小带许可证 CI slice 固定在本地，完整 upstream corpus 仍保持外部依赖 | | Calculon experiment | **Implemented experiment** | `experiments/calculon.py` | +| 外部 baseline 回归门禁 | **Implemented** | `data/validation/` 下的冻结 contract 与带许可证离线 fixture、`experiments/regression.py`、`.github/workflows/quality.yml` | 这些 typed representation、verifier、derivation transaction 与 analysis 构成 hardware exploration 的形式化基础。新的 architecture model、simulator provider 与 analysis product 应扩展这一份 semantic foundation，而不是建立平行 workload truth。 ## 验证基线 -当前 test suite 覆盖 binding consistency、canonical serialization、verifier rejection、pass transaction rollback、checkpoint observer、workload conservation 与 Calculon calibration result。文档检查强制完整双语 page pair 与 strict site build。 +当前 test suite 覆盖 binding consistency、canonical serialization、verifier rejection、pass transaction rollback、checkpoint observer、workload conservation、Calculon calibration、prefill/decode scaling、KV 容量、static request composition 与 baseline-only Vidur comparison。独立 CI job 会在每次面向 main 的 PR 和 push 上执行 8-case Calculon/SeqSel 与 3-case 固定 Vidur gate，同时冻结 provenance、semantic policy、coverage、comparable-subtotal drift budget、不可抵消的 component error、aggregate result 与 IR digest，且不能静默重生成 golden。Vidur gate 只用于 drift detection，不是 accuracy certification。文档检查强制完整双语 page pair 与 strict site build。 能力升级需要端到端 product test。例如只增加 `ArchitectureBlueprint` dataclass 仍是 Contract Only；至少要构造两个不同 candidate、映射同一 workload、产生可比较 result 并保持 provenance，才能形成产品级证据。 diff --git a/mkdocs.yml b/mkdocs.yml index f1d627e..51bec1a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -52,11 +52,13 @@ nav: - Models and Simulation: - Hardware Architecture Model: modeling/hardware.md - Workload and Mapping Model: modeling/workload.md + - Inference Planning and Serving Simulation: modeling/inference.md - Performance Evidence and Cost Models: design/performance/index.md - Performance Database: design/performance/database.md - Simulation and Calibration: design/performance/simulation.md - Experiments: - Calculon Calibration: experiments/calculon-calibration.md + - Vidur Raw Component-Profile Alignment: experiments/vidur-baseline.md - Formal Analysis Foundations: - Formal Analysis Architecture: design/index.md - Derivation and Verification Model: design/compilation-model.md @@ -107,11 +109,13 @@ plugins: Models and Simulation: 模型与仿真 Hardware Architecture Model: 硬件架构模型 Workload and Mapping Model: 工作负载与映射模型 + Inference Planning and Serving Simulation: 推理规划与 Serving 仿真 Performance Evidence and Cost Models: 性能证据与 Cost Model Performance Database: 性能数据库 Simulation and Calibration: 仿真与校准 Experiments: 实验 Calculon Calibration: Calculon 校准 + Vidur Raw Component-Profile Alignment: Vidur 原始组件 Profile 对齐 Formal Analysis Foundations: 形式化分析基础 Formal Analysis Architecture: 形式化分析架构 Derivation and Verification Model: 推导与验证模型 diff --git a/pyproject.toml b/pyproject.toml index 7649a2a..155f6e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -130,6 +130,7 @@ features = ["dev", "full"] [tool.hatch.envs.default.scripts] test = "pytest {args:tests}" test-cov = "pytest --cov=blueprinting --cov-report=term-missing {args:tests}" +baseline-regression = "pytest -m baseline_regression {args:tests/regression}" lint = "ruff check src/ tests/" fmt = "ruff format src/ tests/" @@ -204,6 +205,7 @@ addopts = [ testpaths = ["tests"] pythonpath = ["src"] markers = [ + "baseline_regression: pinned external-baseline regression gates required in CI", "slow: marks tests as slow (deselect with '-m \"not slow\"')", ] filterwarnings = [ diff --git a/src/blueprinting/application/__init__.py b/src/blueprinting/application/__init__.py index dcdeb40..42668c4 100644 --- a/src/blueprinting/application/__init__.py +++ b/src/blueprinting/application/__init__.py @@ -13,6 +13,13 @@ SweepRequest, TaskReport, ) +from .inference import ( + DecodeStepReport, + InferenceAnalysisDraft, + InferenceAnalysisOutcome, + InferenceAnalysisReport, + InferenceAnalysisService, +) __all__ = [ "AnalysisDiagnostic", @@ -21,6 +28,11 @@ "AnalysisReport", "BlueprintingService", "DiagnosticLevel", + "DecodeStepReport", + "InferenceAnalysisDraft", + "InferenceAnalysisOutcome", + "InferenceAnalysisReport", + "InferenceAnalysisService", "IRStageReport", "SweepCase", "SweepReport", diff --git a/src/blueprinting/application/analysis.py b/src/blueprinting/application/analysis.py index 5be8ca4..9fe5933 100644 --- a/src/blueprinting/application/analysis.py +++ b/src/blueprinting/application/analysis.py @@ -14,7 +14,7 @@ from dataclasses import dataclass, replace from enum import Enum from itertools import product -from typing import Any +from typing import TYPE_CHECKING, Any from blueprinting.compiler.analysis import CalibrationMode, HardwareProfile, estimate_iteration from blueprinting.compiler.codec import content_digest @@ -36,6 +36,11 @@ LOGGER = logging.getLogger(__name__) +if TYPE_CHECKING: + from blueprinting.compiler.analysis import InferenceCostProvider + + from .inference import InferenceAnalysisDraft, InferenceAnalysisOutcome, InferenceAnalysisService + class DiagnosticLevel(Enum): """Presentation-neutral diagnostic severity.""" @@ -195,6 +200,10 @@ class TaskReport: memory_seconds: float network_seconds: float total_seconds: float + analytical_seconds: float = 0.0 + evidence_provider: str = "" + evidence_revision: str = "" + evidence_match: str = "" @dataclass(frozen=True) @@ -236,9 +245,7 @@ class AnalysisOutcome: @property def ok(self) -> bool: - return self.report is not None and not any( - item.level is DiagnosticLevel.ERROR for item in self.diagnostics - ) + return self.report is not None and not any(item.level is DiagnosticLevel.ERROR for item in self.diagnostics) @dataclass(frozen=True) @@ -254,15 +261,15 @@ class SweepRequest: def __post_init__(self) -> None: for field_name in ("tensor_parallel", "pipeline_parallel", "data_parallel"): values = tuple(dict.fromkeys(getattr(self, field_name))) - if not values or any(isinstance(value, bool) or not isinstance(value, int) or value <= 0 for value in values): + if not values or any( + isinstance(value, bool) or not isinstance(value, int) or value <= 0 for value in values + ): raise ValueError(f"{field_name} must contain positive integers") object.__setattr__(self, field_name, values) if self.max_candidates <= 0: raise ValueError("max_candidates must be positive") if self.candidate_count > self.max_candidates: - raise ValueError( - f"strategy sweep has {self.candidate_count} candidates; limit is {self.max_candidates}" - ) + raise ValueError(f"strategy sweep has {self.candidate_count} candidates; limit is {self.max_candidates}") @property def candidate_count(self) -> int: @@ -359,12 +366,28 @@ def _stage_report( class BlueprintingService: """Single supported orchestration entry point for Blueprinting clients.""" - def __init__(self, analyses: AnalysisStore | None = None) -> None: + def __init__( + self, + analyses: AnalysisStore | None = None, + *, + inference_cost_provider: InferenceCostProvider | None = None, + ) -> None: + from .inference import InferenceAnalysisService + self._manager = PassManager(analyses=analyses) self._pipeline = PassPipeline.of( DistributeTransformerTrainingPass(), PlanTransformerTrainingPass(), ) + self._inference: InferenceAnalysisService = InferenceAnalysisService( + analyses, + cost_provider=inference_cost_provider, + ) + + def analyze_inference(self, draft: InferenceAnalysisDraft) -> InferenceAnalysisOutcome: + """Run the canonical static inference path through the same service boundary.""" + + return self._inference.analyze(draft) def analyze(self, draft: AnalysisDraft) -> AnalysisOutcome: try: @@ -525,8 +548,7 @@ def _analyze(self, draft: AnalysisDraft) -> AnalysisOutcome: AnalysisDiagnostic( code="capacity.device_memory_exceeded", message=( - f"每设备预计需要 {estimate.memory.total} bytes，" - f"超过容量 {hardware.memory.capacity_bytes} bytes" + f"每设备预计需要 {estimate.memory.total} bytes，超过容量 {hardware.memory.capacity_bytes} bytes" ), level=DiagnosticLevel.WARNING, path=("memory", "total"), diff --git a/src/blueprinting/application/inference.py b/src/blueprinting/application/inference.py new file mode 100644 index 0000000..0d1ba8d --- /dev/null +++ b/src/blueprinting/application/inference.py @@ -0,0 +1,553 @@ +"""Application service for static decoder inference planning. + +This layer composes independently compiled prefill and decode phase points +into one homogeneous request-cohort report. It deliberately excludes request +arrival, queueing, continuous batching and scheduler policy; those belong to +the future serving-simulation layer. +""" + +from __future__ import annotations + +import logging +import time +from collections.abc import Mapping +from dataclasses import dataclass, replace +from typing import Any + +from blueprinting.compiler.analysis import ( + CalibrationMode, + HardwareProfile, + InferenceCostProvider, + InferencePhaseEstimate, + estimate_inference_phase, +) +from blueprinting.compiler.bindings import InferencePhase +from blueprinting.compiler.codec import content_digest +from blueprinting.compiler.errors import CompilerError, IRVerificationError, PassExecutionError +from blueprinting.compiler.frozen import FrozenDict, freeze, thaw +from blueprinting.compiler.ir import ModelIR, PortablePlanIR +from blueprinting.compiler.lowering import DistributeTransformerInferencePass, PlanTransformerInferencePass +from blueprinting.compiler.models import ( + TransformerInferenceExecutionSpec, + TransformerInferenceRequestSpec, + TransformerModelSpec, + build_transformer_inference_model_ir, + inference_compilation_session_for, +) +from blueprinting.compiler.passes import AnalysisStore, PassCheckpoint, PassManager, PassPipeline + +from .analysis import ( + AnalysisDiagnostic, + DiagnosticLevel, + IRStageReport, + TaskReport, + _stage_report, +) + +LOGGER = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class InferenceAnalysisDraft: + """Immutable input for one homogeneous inference request cohort.""" + + model_name: str + model_data: FrozenDict + execution_name: str + execution_data: FrozenDict + request_data: FrozenDict + hardware_name: str + hardware_data: FrozenDict + calibration_mode: CalibrationMode = CalibrationMode.SYSTEM_EVIDENCE + seed: int = 0 + + def __post_init__(self) -> None: + for field_name in ("model_name", "execution_name", "hardware_name"): + if not getattr(self, field_name): + raise ValueError(f"{field_name} must not be empty") + for field_name in ("model_data", "execution_data", "request_data", "hardware_data"): + value = freeze(getattr(self, field_name)) + if not isinstance(value, FrozenDict): + raise TypeError(f"{field_name} must be a mapping") + object.__setattr__(self, field_name, value) + if not isinstance(self.calibration_mode, CalibrationMode): + raise TypeError("calibration_mode must be CalibrationMode") + if isinstance(self.seed, bool) or not isinstance(self.seed, int) or self.seed < 0: + raise ValueError("seed must be a non-negative integer") + + @classmethod + def from_mappings( + cls, + *, + model_name: str, + model_data: Mapping[str, Any], + execution_name: str, + execution_data: Mapping[str, Any], + request_data: Mapping[str, Any], + hardware_name: str, + hardware_data: Mapping[str, Any], + calibration_mode: CalibrationMode = CalibrationMode.SYSTEM_EVIDENCE, + seed: int = 0, + ) -> InferenceAnalysisDraft: + return cls( + model_name=model_name, + model_data=FrozenDict(model_data), + execution_name=execution_name, + execution_data=FrozenDict(execution_data), + request_data=FrozenDict(request_data), + hardware_name=hardware_name, + hardware_data=FrozenDict(hardware_data), + calibration_mode=calibration_mode, + seed=seed, + ) + + @property + def fingerprint(self) -> str: + return content_digest( + FrozenDict( + { + "model_name": self.model_name, + "model_data": self.model_data, + "execution_name": self.execution_name, + "execution_data": self.execution_data, + "request_data": self.request_data, + "hardware_name": self.hardware_name, + "hardware_data": self.hardware_data, + "calibration_mode": self.calibration_mode.value, + "seed": self.seed, + } + ), + "blueprinting-inference-analysis-request-v1", + ) + + def normalized_execution(self) -> dict[str, Any]: + data = thaw(self.execution_data) + replicas = data.get("replicas", data.get("data_par", 1)) + data["replicas"] = replicas + data["num_procs"] = data["tensor_par"] * data["pipeline_par"] * replicas + return data + + +@dataclass(frozen=True) +class DecodeStepReport: + context_tokens: int + plan_digest: str + session_fingerprint: str + seconds: float + memory_bytes: int + evidence_revisions: FrozenDict + + +@dataclass(frozen=True) +class InferenceAnalysisReport: + """Static model-phase composition with inspectable portable plans.""" + + schema: str + request_digest: str + model_name: str + execution_name: str + hardware_name: str + calibration_mode: str + world_size: int + analytical_memory_fits: bool + prefill_seconds: float + mean_decode_step_seconds: float + first_decode_step_seconds: float + last_decode_step_seconds: float + model_execution_seconds: float + static_model_tokens_per_second: float | None + latency: FrozenDict + memory: FrozenDict + workload: FrozenDict + evidence: FrozenDict + configuration: FrozenDict + stages: tuple[IRStageReport, ...] + tasks: tuple[TaskReport, ...] + decode_steps: tuple[DecodeStepReport, ...] + limitations: tuple[str, ...] + + +@dataclass(frozen=True) +class InferenceAnalysisOutcome: + request_digest: str + diagnostics: tuple[AnalysisDiagnostic, ...] + report: InferenceAnalysisReport | None = None + + @property + def ok(self) -> bool: + return self.report is not None and not any(item.level is DiagnosticLevel.ERROR for item in self.diagnostics) + + +@dataclass(frozen=True) +class _CompiledPhase: + session_fingerprint: str + plan: PortablePlanIR + estimate: InferencePhaseEstimate + checkpoints: tuple[PassCheckpoint, ...] + + +def _task_reports(plan: PortablePlanIR, estimate: InferencePhaseEstimate) -> tuple[TaskReport, ...]: + return tuple( + TaskReport( + task_id=str(task.id), + operation=str(task.operation), + kind=task.kind.value, + phase=str(task.workload.attributes.get("phase", "unknown")), + engine=str(task.workload.attributes.get("engine", "unknown")), + source_layer=str(task.workload.attributes.get("source_layer", "")), + dependencies=tuple(str(item) for item in task.dependencies), + concurrency_group=task.concurrency_group or "", + operations=task.workload.operations, + read_bytes=task.workload.read_bytes, + write_bytes=task.workload.write_bytes, + message_bytes=task.workload.message_bytes, + compute_seconds=task_estimate.compute_seconds, + memory_seconds=task_estimate.memory_seconds, + network_seconds=task_estimate.network_seconds, + total_seconds=task_estimate.total_seconds, + analytical_seconds=task_estimate.analytical_seconds, + evidence_provider=task_estimate.evidence_provider, + evidence_revision=task_estimate.evidence_revision, + evidence_match=task_estimate.evidence_match, + ) + for task, task_estimate in zip(plan.tasks, estimate.tasks) + ) + + +class InferenceAnalysisService: + """Compile and compose static inference phase points.""" + + def __init__( + self, + analyses: AnalysisStore | None = None, + *, + cost_provider: InferenceCostProvider | None = None, + ) -> None: + self._manager = PassManager(analyses=analyses) + self._pipeline = PassPipeline.of( + DistributeTransformerInferencePass(), + PlanTransformerInferencePass(), + ) + self._cost_provider = cost_provider + + def analyze(self, draft: InferenceAnalysisDraft) -> InferenceAnalysisOutcome: + try: + return self._analyze(draft) + except IRVerificationError as error: + diagnostics = tuple( + AnalysisDiagnostic( + code=item.code, + message=item.message, + path=item.path, + hint=item.hint, + ) + for item in error.diagnostics + ) + return InferenceAnalysisOutcome(draft.fingerprint, diagnostics) + except PassExecutionError as error: + return InferenceAnalysisOutcome( + draft.fingerprint, + ( + AnalysisDiagnostic( + code="inference.lowering.pass_failed", + message=str(error.cause), + path=(error.pass_name,), + hint="检查 phase workload、并行整除关系与网络层级。", + ), + ), + ) + except KeyError as error: + field = str(error.args[0]) if error.args else "" + return InferenceAnalysisOutcome( + draft.fingerprint, + ( + AnalysisDiagnostic( + code="inference.configuration.missing_field", + message=f"推理配置缺少字段 {field!r}", + path=(field,), + ), + ), + ) + except (ValueError, TypeError, IndexError, ZeroDivisionError) as error: + return InferenceAnalysisOutcome( + draft.fingerprint, + ( + AnalysisDiagnostic( + code="inference.configuration.invalid", + message=str(error), + hint="检查请求长度、模型上下文、TP/PP 整除关系和硬件 profile。", + ), + ), + ) + except CompilerError as error: + return InferenceAnalysisOutcome( + draft.fingerprint, + ( + AnalysisDiagnostic( + code="inference.analysis.compiler_failure", + message=str(error), + ), + ), + ) + except Exception as error: # pragma: no cover - defensive application boundary + LOGGER.exception("unexpected Blueprinting inference analysis failure") + return InferenceAnalysisOutcome( + draft.fingerprint, + ( + AnalysisDiagnostic( + code="inference.analysis.internal_error", + message=f"{type(error).__name__}: {error}", + ), + ), + ) + + def _compile_phase( + self, + source: ModelIR, + model: TransformerModelSpec, + execution: TransformerInferenceExecutionSpec, + hardware: HardwareProfile, + draft: InferenceAnalysisDraft, + *, + phase: InferencePhase, + batch_size: int, + context_tokens: int, + ) -> _CompiledPhase: + session = replace( + inference_compilation_session_for( + model, + execution, + phase=phase, + batch_size=batch_size, + context_tokens=context_tokens, + ), + seed=draft.seed, + ) + pipeline = self._manager.run(self._pipeline, source, session=session) + plan = pipeline.ir + if not isinstance(plan, PortablePlanIR): + raise TypeError(f"inference pipeline returned {type(plan).__name__}, expected PortablePlanIR") + estimate = estimate_inference_phase( + plan, + hardware, + draft.calibration_mode, + cost_provider=self._cost_provider, + ) + return _CompiledPhase(session.fingerprint, plan, estimate, pipeline.checkpoints) + + def _analyze(self, draft: InferenceAnalysisDraft) -> InferenceAnalysisOutcome: + model_data = thaw(draft.model_data) + execution_data = draft.normalized_execution() + request_data = thaw(draft.request_data) + hardware_data = thaw(draft.hardware_data) + model = TransformerModelSpec.from_mapping(draft.model_name, model_data) + execution = TransformerInferenceExecutionSpec.from_mapping(execution_data) + request = TransformerInferenceRequestSpec.from_mapping(request_data) + execution.validate_model(model) + request.validate_model(model) + hardware = HardwareProfile.from_mapping( + draft.hardware_name, + hardware_data, + datatype=execution.datatype, + ) + + frontend_started = time.perf_counter_ns() + source = build_transformer_inference_model_ir(model, datatype=execution.datatype) + frontend_duration = time.perf_counter_ns() - frontend_started + prefill = self._compile_phase( + source, + model, + execution, + hardware, + draft, + phase=InferencePhase.PREFILL, + batch_size=request.batch_size, + context_tokens=request.prompt_tokens, + ) + decode = tuple( + self._compile_phase( + source, + model, + execution, + hardware, + draft, + phase=InferencePhase.DECODE, + batch_size=request.batch_size, + context_tokens=context_tokens, + ) + for context_tokens in request.decode_contexts() + ) + + prefill_seconds = prefill.estimate.total_seconds + decode_total = sum(item.estimate.total_seconds for item in decode) + model_execution_seconds = prefill_seconds + decode_total + mean_decode_step = decode_total / request.decode_iterations if request.decode_iterations else 0.0 + first_decode_step = decode[0].estimate.total_seconds if decode else 0.0 + last_decode_step = decode[-1].estimate.total_seconds if decode else 0.0 + phase_estimates = (prefill.estimate,) + tuple(item.estimate for item in decode) + peak = max(phase_estimates, key=lambda item: item.memory.total) + analytical_memory_fits = peak.memory.total <= hardware.memory.capacity_bytes + + stages = [ + _stage_report("model", "推理模型语义", "frontend-import", source, frontend_duration), + _stage_report( + "prefill.distributed", + "Prefill 分布式任务", + prefill.checkpoints[0].record.pass_name, + prefill.checkpoints[0].ir, + prefill.checkpoints[0].record.duration_ns, + ), + _stage_report( + "prefill.portable", + "Prefill 可移植计划", + prefill.checkpoints[1].record.pass_name, + prefill.checkpoints[1].ir, + prefill.checkpoints[1].record.duration_ns, + ), + ] + representative = decode[-1] if decode else None + if representative is not None: + stages.extend( + ( + _stage_report( + "decode.distributed", + "Decode 分布式任务（最终 context）", + representative.checkpoints[0].record.pass_name, + representative.checkpoints[0].ir, + representative.checkpoints[0].record.duration_ns, + ), + _stage_report( + "decode.portable", + "Decode 可移植计划（最终 context）", + representative.checkpoints[1].record.pass_name, + representative.checkpoints[1].ir, + representative.checkpoints[1].record.duration_ns, + ), + ) + ) + + tasks = _task_reports(prefill.plan, prefill.estimate) + if representative is not None: + tasks += _task_reports(representative.plan, representative.estimate) + decode_steps = tuple( + DecodeStepReport( + context_tokens=item.estimate.context_tokens, + plan_digest=item.plan.digest, + session_fingerprint=item.session_fingerprint, + seconds=item.estimate.total_seconds, + memory_bytes=item.estimate.memory.total, + evidence_revisions=item.estimate.evidence_revisions, + ) + for item in decode + ) + diagnostics: tuple[AnalysisDiagnostic, ...] = () + if not analytical_memory_fits: + diagnostics = ( + AnalysisDiagnostic( + code="capacity.device_memory_exceeded", + message=( + f"每设备推理峰值预计需要 {peak.memory.total} bytes，" + f"超过容量 {hardware.memory.capacity_bytes} bytes" + ), + level=DiagnosticLevel.WARNING, + path=("memory", "total"), + hint="增加 TP/PP、减小 batch/context，或选择更大容量硬件。", + ), + ) + + latency = FrozenDict( + { + "prefill": prefill_seconds, + "decode_total": decode_total, + "mean_decode_step": mean_decode_step, + "first_decode_step": first_decode_step, + "last_decode_step": last_decode_step, + "model_execution": model_execution_seconds, + } + ) + memory = FrozenDict( + { + "weights": peak.memory.weights, + "kv_cache": peak.memory.kv_cache, + "max_kv_cache": max(item.memory.kv_cache for item in phase_estimates), + "working_upper_bound": peak.memory.working_upper_bound, + "max_working_upper_bound": max(item.memory.working_upper_bound for item in phase_estimates), + "pipeline_buffers": peak.memory.pipeline_buffers, + "total": peak.memory.total, + "capacity": hardware.memory.capacity_bytes, + "peak_phase": peak.phase.value, + "peak_context_tokens": peak.context_tokens, + } + ) + workload = FrozenDict( + { + "prefill_task_count": len(prefill.plan.tasks), + "decode_task_count": len(representative.plan.tasks) if representative else 0, + "decode_iterations": request.decode_iterations, + "prefill_operations_per_block": sum(task.workload.operations for task in prefill.plan.tasks), + "final_decode_operations_per_block": ( + sum(task.workload.operations for task in representative.plan.tasks) if representative else 0 + ), + } + ) + evidence_revisions: dict[str, str] = {} + for estimate in phase_estimates: + evidence_revisions.update(dict(estimate.evidence_revisions.items())) + evidence = FrozenDict( + { + "hardware_name": hardware.name, + "hardware_revision": hardware.evidence_revision, + "mode": draft.calibration_mode.value, + "cost_provider_revision": self._cost_provider.revision if self._cost_provider is not None else "none", + "revisions": FrozenDict(evidence_revisions), + } + ) + configuration = FrozenDict( + { + "model": model_data, + "execution": execution_data, + "request": request_data, + "hardware": { + "name": hardware.name, + "datatype": hardware.datatype, + "processing_mode": hardware.processing_mode, + }, + } + ) + report = InferenceAnalysisReport( + schema="blueprinting.inference-analysis-report.v1", + request_digest=draft.fingerprint, + model_name=draft.model_name, + execution_name=draft.execution_name, + hardware_name=draft.hardware_name, + calibration_mode=draft.calibration_mode.value, + world_size=execution.world_size, + analytical_memory_fits=analytical_memory_fits, + prefill_seconds=prefill_seconds, + mean_decode_step_seconds=mean_decode_step, + first_decode_step_seconds=first_decode_step, + last_decode_step_seconds=last_decode_step, + model_execution_seconds=model_execution_seconds, + static_model_tokens_per_second=( + request.batch_size * request.generated_tokens / model_execution_seconds + if model_execution_seconds > 0 + else None + ), + latency=latency, + memory=memory, + workload=workload, + evidence=evidence, + configuration=configuration, + stages=tuple(stages), + tasks=tasks, + decode_steps=decode_steps, + limitations=( + "这是 homogeneous cohort 的静态 phase composition；不包含请求到达、排队、continuous batching 或 scheduler 开销。", + "prefill/decode 时间只覆盖当前 decoder-block dialect；由于 embedding、LM head 与 sampler 未建模，不能解释为 TTFT、TPOT 或 E2E serving latency。", + "replicas 只参与映射合法性与 world-size 记账；当前报告是单 replica cohort latency，不估算跨 replica serving capacity。", + "当前 workload dialect 支持 dense multi-head attention 与非 gated MLP；embedding、LM head 和 sampler 尚未建模。", + "PortablePlanIR 尚未绑定 attention implementation；working memory 使用未融合 score materialization 的保守上界。", + "除非提供 Blueprinting cost provider，组件耗时使用共享 hardware profile 的解析 roofline 证据；comparison baseline 不参与该选择。", + ), + ) + return InferenceAnalysisOutcome(draft.fingerprint, diagnostics, report) diff --git a/src/blueprinting/compiler/analysis/__init__.py b/src/blueprinting/compiler/analysis/__init__.py index b3b916b..085acf0 100644 --- a/src/blueprinting/compiler/analysis/__init__.py +++ b/src/blueprinting/compiler/analysis/__init__.py @@ -9,6 +9,24 @@ estimate_block, estimate_iteration, ) +from .inference_cost import ( + InferencePhaseEstimate, + InferencePhaseMemory, + InferenceTaskEstimate, + estimate_inference_phase, + inference_evidence_query_for, +) +from .inference_evidence import ( + InferenceBaseline, + InferenceCostProvider, + InferenceEvidenceQuery, + InferenceEvidenceResult, +) +from .transformer_inference import ( + InferenceBlockMemoryFacts, + InferenceInvocation, + compile_transformer_inference_block, +) from .transformer_workload import ( BlockMemoryFacts, EngineKind, @@ -17,6 +35,7 @@ TrainingPhase, compile_transformer_block, ) +from .vidur import VidurProfileBaseline __all__ = [ "BlockEstimate", @@ -24,12 +43,25 @@ "CalibrationMode", "EngineKind", "HardwareProfile", + "InferenceBlockMemoryFacts", + "InferenceBaseline", + "InferenceCostProvider", + "InferenceEvidenceQuery", + "InferenceEvidenceResult", + "InferenceInvocation", + "InferencePhaseEstimate", + "InferencePhaseMemory", + "InferenceTaskEstimate", "IterationEstimate", "IterationMemory", "PhaseWork", "PrimitiveInvocation", "TrainingPhase", "compile_transformer_block", + "compile_transformer_inference_block", "estimate_block", + "estimate_inference_phase", "estimate_iteration", + "inference_evidence_query_for", + "VidurProfileBaseline", ] diff --git a/src/blueprinting/compiler/analysis/inference_cost.py b/src/blueprinting/compiler/analysis/inference_cost.py new file mode 100644 index 0000000..e01a998 --- /dev/null +++ b/src/blueprinting/compiler/analysis/inference_cost.py @@ -0,0 +1,307 @@ +"""Evidence-aware cost view for a static inference phase plan.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from ..bindings import InferencePhase +from ..frozen import FrozenDict +from ..ir import CollectiveKind, PlanBuffer, PlanTask, PortablePlanIR +from ..models.transformer import TransformerModelSpec +from ..models.transformer_inference import TransformerInferenceExecutionSpec +from .cost_model import CalibrationMode, HardwareProfile +from .inference_evidence import InferenceCostProvider, InferenceEvidenceQuery +from .transformer_inference import InferenceInvocation +from .transformer_workload import EngineKind, PhaseWork + + +@dataclass(frozen=True) +class InferenceTaskEstimate: + invocation: InferenceInvocation + compute_seconds: float + memory_seconds: float + network_seconds: float + analytical_seconds: float + total_seconds: float + evidence_provider: str + evidence_revision: str + evidence_match: str + + +@dataclass(frozen=True) +class InferencePhaseMemory: + weights: int + kv_cache: int + working_upper_bound: int + pipeline_buffers: int + + @property + def total(self) -> int: + return self.weights + self.kv_cache + self.working_upper_bound + self.pipeline_buffers + + +@dataclass(frozen=True) +class InferencePhaseEstimate: + mode: CalibrationMode + phase: InferencePhase + batch_size: int + query_tokens: int + context_tokens: int + tasks: tuple[InferenceTaskEstimate, ...] + memory: InferencePhaseMemory + block_seconds: float + transformer_seconds: float + pipeline_seconds: float + total_seconds: float + evidence_revisions: FrozenDict + + +def inference_evidence_query_for( + invocation: InferenceInvocation, + *, + hardware: HardwareProfile, + execution: TransformerInferenceExecutionSpec, + model: TransformerModelSpec, + batch_size: int, + query_tokens: int, + context_tokens: int, +) -> InferenceEvidenceQuery: + """Build the one normalized query shared by cost and baseline paths.""" + + return InferenceEvidenceQuery( + phase=invocation.phase, + primitive=invocation.primitive, + source_layer=invocation.source_layer, + model_name=model.name, + hardware_name=hardware.name, + model_sequence_length=model.sequence_length, + hidden_size=model.hidden_size, + feedforward_size=model.feedforward_size, + attention_heads=model.attention_heads, + batch_size=batch_size, + query_tokens=query_tokens, + context_tokens=context_tokens, + tensor_parallel=execution.tensor_parallel, + datatype=execution.datatype, + ) + + +def _task_estimate( + task: PlanTask, + *, + hardware: HardwareProfile, + execution: TransformerInferenceExecutionSpec, + model: TransformerModelSpec, + batch_size: int, + query_tokens: int, + context_tokens: int, + mode: CalibrationMode, + cost_provider: InferenceCostProvider | None, +) -> InferenceTaskEstimate: + invocation = _invocation_from_plan_task(task) + work = invocation.work + processor = hardware.matrix if invocation.engine is EngineKind.MATRIX else hardware.vector + compute_seconds = work.operations / processor.throughput(work.operations, mode) if work.operations else 0.0 + memory_seconds = ( + work.memory_bytes / hardware.memory.throughput(work.memory_bytes, mode) if work.memory_bytes else 0.0 + ) + network_seconds = 0.0 + if invocation.engine is EngineKind.COLLECTIVE: + if invocation.network_tier is None or invocation.collective is None: + raise ValueError("collective invocation is missing network facts") + try: + network = hardware.networks[invocation.network_tier] + except IndexError as error: + raise ValueError(f"hardware profile does not define network tier {invocation.network_tier}") from error + network_seconds = network.time( + invocation.collective.value, + work.message_bytes, + execution.tensor_parallel, + mode, + ) + analytical_seconds = hardware.processing_time(compute_seconds, memory_seconds) + network_seconds + provider_name = "analytical-system-profile" + revision = hardware.evidence_revision + match = mode.value + total_seconds = analytical_seconds + if cost_provider is not None: + evidence = cost_provider.resolve( + inference_evidence_query_for( + invocation, + model=model, + execution=execution, + hardware=hardware, + batch_size=batch_size, + query_tokens=query_tokens, + context_tokens=context_tokens, + ) + ) + if evidence is not None: + total_seconds = evidence.seconds + provider_name = evidence.provider + revision = evidence.revision + match = evidence.match + return InferenceTaskEstimate( + invocation=invocation, + compute_seconds=compute_seconds, + memory_seconds=memory_seconds, + network_seconds=network_seconds, + analytical_seconds=analytical_seconds, + total_seconds=total_seconds, + evidence_provider=provider_name, + evidence_revision=revision, + evidence_match=match, + ) + + +def _invocation_from_plan_task(task: PlanTask) -> InferenceInvocation: + """Reconstruct comparison metadata from canonical portable workload facts. + + Costing reads operation and byte counts only from ``PlanTask.workload``. + The earlier lowering-only ``InferenceInvocation`` is deliberately not + embedded in ``PortablePlanIR`` as a second source of workload truth. + """ + + attributes = task.workload.attributes + try: + phase = InferencePhase(attributes["phase"]) + engine = EngineKind(attributes["engine"]) + name = attributes["name"] + primitive = attributes["primitive"] + source_layer = attributes["source_layer"] + except (KeyError, ValueError) as error: + raise ValueError(f"portable inference task {task.id} has invalid semantic workload metadata") from error + for field_name, value in (("name", name), ("primitive", primitive), ("source_layer", source_layer)): + if not isinstance(value, str) or not value: + raise ValueError(f"portable inference task {task.id} has invalid {field_name}") + + collective_value = attributes.get("collective", "") + network_tier_value = attributes.get("network_tier", -1) + collective = None + network_tier = None + if engine is EngineKind.COLLECTIVE: + try: + collective = CollectiveKind(collective_value) + except ValueError as error: + raise ValueError(f"portable inference task {task.id} has invalid collective metadata") from error + if isinstance(network_tier_value, bool) or not isinstance(network_tier_value, int) or network_tier_value < 0: + raise ValueError(f"portable inference task {task.id} has invalid network tier") + network_tier = network_tier_value + elif collective_value != "" or network_tier_value != -1: + raise ValueError(f"local portable inference task {task.id} carries collective metadata") + + return InferenceInvocation( + name=name, + source_layer=source_layer, + primitive=primitive, + phase=phase, + engine=engine, + work=PhaseWork( + operations=task.workload.operations, + read_bytes=task.workload.read_bytes, + write_bytes=task.workload.write_bytes, + message_bytes=task.workload.message_bytes, + ), + collective=collective, + network_tier=network_tier, + ) + + +def _concrete_buffer_size(buffer: PlanBuffer) -> int: + size = buffer.size_bytes + if isinstance(size, bool) or not isinstance(size, int) or size < 0: + raise TypeError(f"portable inference buffer {buffer.id} must have a concrete non-negative size") + return size + + +def _semantic_buffer_size(plan: PortablePlanIR, semantic: str) -> int: + buffers = tuple(buffer for buffer in plan.buffers if buffer.attributes.get("semantic") == semantic) + if len(buffers) != 1: + raise ValueError(f"portable inference plan must contain exactly one {semantic!r} buffer") + return _concrete_buffer_size(buffers[0]) + + +def estimate_inference_phase( + plan: PortablePlanIR, + hardware: HardwareProfile, + mode: CalibrationMode = CalibrationMode.SYSTEM_EVIDENCE, + *, + cost_provider: InferenceCostProvider | None = None, +) -> InferencePhaseEstimate: + """Cost one prefill or decode phase point without queueing assumptions.""" + + model = plan.attributes.get("model_spec") + execution = plan.attributes.get("inference_execution_spec") + phase = plan.attributes.get("inference_phase") + if not isinstance(model, TransformerModelSpec): + raise TypeError("portable inference plan is missing TransformerModelSpec") + if not isinstance(execution, TransformerInferenceExecutionSpec): + raise TypeError("portable inference plan is missing TransformerInferenceExecutionSpec") + if not isinstance(phase, InferencePhase): + raise TypeError("portable inference plan is missing InferencePhase") + if hardware.datatype != execution.datatype: + raise ValueError("hardware profile datatype does not match inference execution datatype") + batch_size = plan.attributes.get("batch_size") + query_tokens = plan.attributes.get("query_tokens") + context_tokens = plan.attributes.get("context_tokens") + if any(not isinstance(value, int) for value in (batch_size, query_tokens, context_tokens)): + raise TypeError("portable inference plan has non-concrete workload facts") + + task_estimates = [] + for task in plan.tasks: + task_estimates.append( + _task_estimate( + task, + hardware=hardware, + execution=execution, + model=model, + batch_size=batch_size, + query_tokens=query_tokens, + context_tokens=context_tokens, + mode=mode, + cost_provider=cost_provider, + ) + ) + + block_seconds = sum(item.total_seconds for item in task_estimates) + transformer_seconds = block_seconds * model.block_count + pipeline_seconds = 0.0 + if execution.pipeline_parallel > 1: + try: + network = hardware.networks[execution.pipeline_parallel_network] + except IndexError as error: + raise ValueError( + f"hardware profile does not define network tier {execution.pipeline_parallel_network}" + ) from error + pipeline_seconds = (execution.pipeline_parallel - 1) * network.time( + "p2p", + _concrete_buffer_size(next(buffer for buffer in plan.buffers if buffer.id == plan.inputs[0])), + 2, + mode, + ) + + blocks_per_stage = model.block_count // execution.pipeline_parallel + boundary_bytes = _concrete_buffer_size(next(buffer for buffer in plan.buffers if buffer.id == plan.inputs[0])) + memory = InferencePhaseMemory( + weights=_semantic_buffer_size(plan, "block_weights") * blocks_per_stage, + kv_cache=_semantic_buffer_size(plan, "kv_cache") * blocks_per_stage, + working_upper_bound=_semantic_buffer_size(plan, "block_working_upper_bound"), + pipeline_buffers=boundary_bytes * (2 if execution.pipeline_parallel > 1 else 1), + ) + revisions = {hardware.evidence_revision: "analytical-system-profile"} + for item in task_estimates: + revisions[item.evidence_revision] = item.evidence_provider + return InferencePhaseEstimate( + mode=mode, + phase=phase, + batch_size=batch_size, + query_tokens=query_tokens, + context_tokens=context_tokens, + tasks=tuple(task_estimates), + memory=memory, + block_seconds=block_seconds, + transformer_seconds=transformer_seconds, + pipeline_seconds=pipeline_seconds, + total_seconds=transformer_seconds + pipeline_seconds, + evidence_revisions=FrozenDict(revisions), + ) diff --git a/src/blueprinting/compiler/analysis/inference_evidence.py b/src/blueprinting/compiler/analysis/inference_evidence.py new file mode 100644 index 0000000..a856526 --- /dev/null +++ b/src/blueprinting/compiler/analysis/inference_evidence.py @@ -0,0 +1,104 @@ +"""Normalized latency evidence contracts with an explicit oracle boundary. + +Cost providers are admissible inputs to Blueprinting's estimator. Baselines +are read-only comparison oracles and therefore expose a different method name; +they cannot be passed accidentally as cost providers. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import Protocol, runtime_checkable + +from ..bindings import InferencePhase + + +@dataclass(frozen=True) +class InferenceEvidenceQuery: + """A component query with every shape fact needed for reproducibility.""" + + phase: InferencePhase + primitive: str + source_layer: str + model_name: str + hardware_name: str + model_sequence_length: int + hidden_size: int + feedforward_size: int + attention_heads: int + batch_size: int + query_tokens: int + context_tokens: int + tensor_parallel: int + datatype: str + + def __post_init__(self) -> None: + if not isinstance(self.phase, InferencePhase): + raise TypeError("phase must be InferencePhase") + for field_name in ("primitive", "source_layer", "model_name", "hardware_name", "datatype"): + value = getattr(self, field_name) + if not isinstance(value, str) or not value: + raise ValueError(f"{field_name} must not be empty") + for field_name in ( + "model_sequence_length", + "hidden_size", + "feedforward_size", + "attention_heads", + "batch_size", + "query_tokens", + "context_tokens", + "tensor_parallel", + ): + value = getattr(self, field_name) + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError(f"{field_name} must be a positive integer") + if self.query_tokens > self.context_tokens: + raise ValueError("query_tokens cannot exceed context_tokens") + if self.context_tokens > self.model_sequence_length: + raise ValueError("context_tokens cannot exceed model_sequence_length") + if self.phase is InferencePhase.DECODE and self.query_tokens != 1: + raise ValueError("decode evidence queries require query_tokens == 1") + + +@dataclass(frozen=True) +class InferenceEvidenceResult: + """Measured or simulated component latency and its provenance.""" + + seconds: float + provider: str + revision: str + match: str + + def __post_init__(self) -> None: + if ( + isinstance(self.seconds, bool) + or not isinstance(self.seconds, (int, float)) + or not math.isfinite(self.seconds) + or self.seconds < 0 + ): + raise ValueError("evidence seconds must be finite and non-negative") + for field_name in ("provider", "revision", "match"): + value = getattr(self, field_name) + if not isinstance(value, str) or not value: + raise ValueError(f"{field_name} must not be empty") + + +@runtime_checkable +class InferenceCostProvider(Protocol): + """Admissible Blueprinting cost source, such as its performance database.""" + + @property + def revision(self) -> str: ... + + def resolve(self, query: InferenceEvidenceQuery) -> InferenceEvidenceResult | None: ... + + +@runtime_checkable +class InferenceBaseline(Protocol): + """External comparison oracle that must never participate in lowering or costing.""" + + @property + def revision(self) -> str: ... + + def lookup(self, query: InferenceEvidenceQuery) -> InferenceEvidenceResult | None: ... diff --git a/src/blueprinting/compiler/analysis/transformer_inference.py b/src/blueprinting/compiler/analysis/transformer_inference.py new file mode 100644 index 0000000..e54d695 --- /dev/null +++ b/src/blueprinting/compiler/analysis/transformer_inference.py @@ -0,0 +1,288 @@ +"""Exact per-block work derivation for decoder-only Transformer inference. + +The task boundaries intentionally follow observable implementation families +(projection, RoPE, KV save, fused attention, MLP and collectives). This keeps +the analytical model inspectable and permits post-hoc comparison with external +baselines without allowing those baselines to change workload semantics. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from ..bindings import InferencePhase +from ..codec import record_type +from ..ir import CollectiveKind +from ..models.transformer import TransformerModelSpec +from ..models.transformer_inference import TransformerInferenceExecutionSpec +from .transformer_workload import EngineKind, PhaseWork + + +@record_type("compiler.analysis.inference_invocation.v1") +@dataclass(frozen=True) +class InferenceInvocation: + """One target-neutral component invocation for a single decoder block.""" + + name: str + source_layer: str + primitive: str + phase: InferencePhase + engine: EngineKind + work: PhaseWork + collective: CollectiveKind | None = None + network_tier: int | None = None + + def __post_init__(self) -> None: + for field_name in ("name", "source_layer", "primitive"): + value = getattr(self, field_name) + if not isinstance(value, str) or not value: + raise ValueError(f"{field_name} must not be empty") + if not isinstance(self.phase, InferencePhase): + raise TypeError("phase must be InferencePhase") + if not isinstance(self.engine, EngineKind): + raise TypeError("engine must be EngineKind") + if not isinstance(self.work, PhaseWork): + raise TypeError("work must be PhaseWork") + if self.engine is EngineKind.COLLECTIVE: + if self.collective is None or self.network_tier is None: + raise ValueError("collective invocations require kind and network tier") + elif self.collective is not None or self.network_tier is not None: + raise ValueError("local invocations cannot carry collective metadata") + + +@record_type("compiler.analysis.inference_block_memory_facts.v1") +@dataclass(frozen=True) +class InferenceBlockMemoryFacts: + """Per-rank storage for one tensor-parallel block shard and phase.""" + + weights: int + kv_cache: int + working_upper_bound: int + boundary: int + + def __post_init__(self) -> None: + for field_name in self.__dataclass_fields__: + value = getattr(self, field_name) + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError(f"{field_name} must be a non-negative integer") + + +def _work(*, operations: int = 0, read: int = 0, write: int = 0, message: int = 0) -> PhaseWork: + return PhaseWork(operations=operations, read_bytes=read, write_bytes=write, message_bytes=message) + + +def compile_transformer_inference_block( + model: TransformerModelSpec, + execution: TransformerInferenceExecutionSpec, + *, + phase: InferencePhase, + batch_size: int, + context_tokens: int, +) -> tuple[tuple[InferenceInvocation, ...], InferenceBlockMemoryFacts]: + """Derive exact work for one local block at one inference phase point. + + ``context_tokens`` is the number of keys visible to attention. Prefill has + ``query_tokens == context_tokens``; decode has one query token and includes + the newly appended token in the context. + """ + + execution.validate_model(model) + if not isinstance(phase, InferencePhase): + raise TypeError("phase must be InferencePhase") + for name, value in (("batch_size", batch_size), ("context_tokens", context_tokens)): + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError(f"{name} must be a positive integer") + + b = batch_size + q = context_tokens if phase is InferencePhase.PREFILL else 1 + c = context_tokens + h = model.hidden_size + f = model.feedforward_size + tp = execution.tensor_parallel + heads = model.attention_heads // tp + local_h = h // tp + local_f = f // tp + element_bytes = execution.bytes_per_element + token_elements = b * q * h + local_token_elements = b * q * local_h + score_elements = b * heads * q * c + + invocations: list[InferenceInvocation] = [] + + def local( + name: str, + source_layer: str, + primitive: str, + engine: EngineKind, + work: PhaseWork, + ) -> None: + invocations.append( + InferenceInvocation( + name=f"{phase.value}.{name}", + source_layer=source_layer, + primitive=primitive, + phase=phase, + engine=engine, + work=work, + ) + ) + + def all_reduce(name: str, source_layer: str) -> None: + if tp == 1: + return + invocations.append( + InferenceInvocation( + name=f"{phase.value}.{name}", + source_layer=source_layer, + primitive="all_reduce", + phase=phase, + engine=EngineKind.COLLECTIVE, + work=_work(message=token_elements * element_bytes), + collective=CollectiveKind.ALL_REDUCE, + network_tier=execution.tensor_parallel_network, + ) + ) + + norm_parameter_elements = 2 * h + norm_work = _work( + operations=5 * token_elements, + read=(token_elements + norm_parameter_elements) * element_bytes, + write=token_elements * element_bytes, + ) + local("input_norm", "attention.input_norm", "input_layernorm", EngineKind.VECTOR, norm_work) + + qkv_weight_elements = 3 * h * h // tp + local( + "attention_pre_projection", + "attention.qkv", + "attention_pre_projection", + EngineKind.MATRIX, + _work( + operations=6 * b * q * h * h // tp, + read=(token_elements + qkv_weight_elements) * element_bytes, + write=3 * local_token_elements * element_bytes, + ), + ) + local( + "attention_rope", + "attention.rope", + "attention_rope", + EngineKind.VECTOR, + _work( + operations=12 * local_token_elements, + read=2 * local_token_elements * element_bytes, + write=2 * local_token_elements * element_bytes, + ), + ) + local( + "attention_kv_cache_save", + "attention.kv_cache", + "attention_kv_cache_save", + EngineKind.VECTOR, + _work( + read=2 * local_token_elements * element_bytes, + write=2 * local_token_elements * element_bytes, + ), + ) + local( + "attention_core", + "attention.core", + "attention_core", + EngineKind.MATRIX, + _work( + operations=4 * b * q * c * h // tp + 5 * score_elements, + read=(local_token_elements + 2 * b * c * local_h) * element_bytes, + write=local_token_elements * element_bytes, + ), + ) + output_weight_elements = h * h // tp + local( + "attention_post_projection", + "attention.output", + "attention_post_projection", + EngineKind.MATRIX, + _work( + operations=2 * b * q * h * h // tp, + read=(local_token_elements + output_weight_elements) * element_bytes, + write=token_elements * element_bytes, + ), + ) + all_reduce("attention_all_reduce", "attention.output") + local( + "attention_residual", + "attention.residual", + "residual_add", + EngineKind.VECTOR, + _work( + operations=token_elements, + read=2 * token_elements * element_bytes, + write=token_elements * element_bytes, + ), + ) + local("post_attention_norm", "mlp.input_norm", "post_attention_layernorm", EngineKind.VECTOR, norm_work) + + mlp_weight_elements = h * f // tp + local( + "mlp_up_projection", + "mlp.up", + "mlp_up_projection", + EngineKind.MATRIX, + _work( + operations=2 * b * q * h * f // tp, + read=(token_elements + mlp_weight_elements) * element_bytes, + write=b * q * local_f * element_bytes, + ), + ) + local( + "mlp_activation", + "mlp.activation", + "mlp_activation", + EngineKind.VECTOR, + _work( + operations=8 * b * q * local_f, + read=b * q * local_f * element_bytes, + write=b * q * local_f * element_bytes, + ), + ) + local( + "mlp_down_projection", + "mlp.down", + "mlp_down_projection", + EngineKind.MATRIX, + _work( + operations=2 * b * q * h * f // tp, + read=(b * q * local_f + mlp_weight_elements) * element_bytes, + write=token_elements * element_bytes, + ), + ) + all_reduce("mlp_all_reduce", "mlp.down") + local( + "mlp_residual", + "mlp.residual", + "residual_add", + EngineKind.VECTOR, + _work( + operations=token_elements, + read=2 * token_elements * element_bytes, + write=token_elements * element_bytes, + ), + ) + + matrix_weight_elements = (4 * h * h + 2 * h * f) // tp + replicated_norm_elements = 4 * h + block_weights = (matrix_weight_elements + replicated_norm_elements) * element_bytes + block_kv_cache = 2 * b * c * local_h * element_bytes + boundary = token_elements * element_bytes + qkv_working = 3 * local_token_elements + attention_working = score_elements + mlp_working = b * q * local_f + # PortablePlanIR has not selected a fused attention implementation yet. + # Retain the unfused score materialization as a conservative capacity + # bound; target binding may replace it with implementation workspace. + working_upper_bound = boundary + max(qkv_working, attention_working, mlp_working) * element_bytes + return tuple(invocations), InferenceBlockMemoryFacts( + weights=block_weights, + kv_cache=block_kv_cache, + working_upper_bound=working_upper_bound, + boundary=boundary, + ) diff --git a/src/blueprinting/compiler/analysis/vidur.py b/src/blueprinting/compiler/analysis/vidur.py new file mode 100644 index 0000000..fe3e376 --- /dev/null +++ b/src/blueprinting/compiler/analysis/vidur.py @@ -0,0 +1,283 @@ +"""Strict adapter for user-supplied Vidur profiling CSV files. + +The adapter intentionally performs exact-key lookup only. It does not copy +Vidur datasets into Blueprinting and it never silently interpolates across a +different model, tensor-parallel degree, batch size or context length. +""" + +from __future__ import annotations + +import csv +import hashlib +import math +import statistics +from pathlib import Path + +from ..bindings import InferencePhase +from ..codec import content_digest +from ..frozen import FrozenDict +from .inference_evidence import InferenceEvidenceQuery, InferenceEvidenceResult + +_COMPUTE_COLUMNS = { + "input_layernorm": "time_stats.input_layernorm.median", + "attention_pre_projection": "time_stats.attn_pre_proj.median", + "attention_rope": "time_stats.attn_rope.median", + "attention_post_projection": "time_stats.attn_post_proj.median", + "post_attention_layernorm": "time_stats.post_attention_layernorm.median", + "mlp_up_projection": "time_stats.mlp_up_proj.median", + "mlp_activation": "time_stats.mlp_act.median", + "mlp_down_projection": "time_stats.mlp_down_proj.median", + "residual_add": "time_stats.add.median", +} + + +def _read_rows(path: Path, *, required: frozenset[str], timing_columns: frozenset[str]) -> tuple[dict[str, str], ...]: + with path.open(newline="", encoding="utf-8") as stream: + reader = csv.DictReader(stream) + if reader.fieldnames is None: + raise ValueError(f"Vidur profile {path} has no CSV header") + fields = frozenset(reader.fieldnames) + missing = required - fields + if missing: + raise ValueError(f"Vidur profile {path} is missing columns: {', '.join(sorted(missing))}") + if fields.isdisjoint(timing_columns): + raise ValueError(f"Vidur profile {path} has no supported timing column") + rows = tuple(dict(row) for row in reader) + if not rows: + raise ValueError(f"Vidur profile {path} has no data rows") + return rows + + +def _integer(row: dict[str, str], key: str) -> int | None: + value = row.get(key, "") + if value == "": + return None + try: + numeric = float(value) + except ValueError: + return None + if not math.isfinite(numeric) or not numeric.is_integer(): + return None + return int(numeric) + + +def _boolean(row: dict[str, str], key: str) -> bool | None: + value = row.get(key, "").strip().lower() + if value in {"true", "1"}: + return True + if value in {"false", "0"}: + return False + return None + + +def _milliseconds(row: dict[str, str], key: str) -> float | None: + value = row.get(key, "") + if value == "": + return None + try: + result = float(value) + except ValueError: + return None + return result if math.isfinite(result) and result >= 0 else None + + +class VidurProfileBaseline: + """Exact-match comparison baseline for Vidur's public CSV schema.""" + + def __init__( + self, + *, + attention_rows: tuple[dict[str, str], ...], + compute_rows: tuple[dict[str, str], ...], + model_name: str, + hardware_name: str, + attention_backend: str, + block_size: int, + datatype: str, + source_revision: str, + data_digest: str, + ) -> None: + if any( + not value + for value in (model_name, hardware_name, attention_backend, datatype, source_revision, data_digest) + ): + raise ValueError("Vidur baseline identity fields must not be empty") + if isinstance(block_size, bool) or not isinstance(block_size, int) or block_size <= 0: + raise ValueError("Vidur block_size must be a positive integer") + self._attention_rows = attention_rows + self._compute_rows = compute_rows + self._model_name = model_name + self._hardware_name = hardware_name + self._attention_backend = attention_backend + self._block_size = block_size + self._datatype = datatype + self._revision = content_digest( + FrozenDict( + { + "adapter": "blueprinting-vidur-baseline-v1", + "upstream_revision": source_revision, + "data_digest": data_digest, + "model_name": model_name, + "hardware_name": hardware_name, + "attention_backend": attention_backend, + "block_size": block_size, + "datatype": datatype, + } + ), + "inference-baseline", + ) + + @classmethod + def from_csv( + cls, + *, + attention_csv: str | Path, + compute_csv: str | Path, + model_name: str, + hardware_name: str, + attention_backend: str, + block_size: int, + source_revision: str, + datatype: str = "float16", + ) -> VidurProfileBaseline: + """Load baseline profiles bound to an explicit upstream revision.""" + + attention_path = Path(attention_csv) + compute_path = Path(compute_csv) + digester = hashlib.sha256() + for path in (attention_path, compute_path): + payload = path.read_bytes() + digester.update(path.name.encode("utf-8")) + digester.update(len(payload).to_bytes(8, "big")) + digester.update(payload) + return cls( + attention_rows=_read_rows( + attention_path, + required=frozenset( + { + "n_embd", + "n_q_head", + "n_kv_head", + "num_tensor_parallel_workers", + "batch_size", + "prefill_chunk_size", + "kv_cache_size", + "is_prefill", + "attention_backend", + "block_size", + "max_model_len", + } + ), + timing_columns=frozenset( + { + "time_stats.attn_prefill.median", + "time_stats.attn_decode.median", + "time_stats.attn_kv_cache_save.median", + } + ), + ), + compute_rows=_read_rows( + compute_path, + required=frozenset( + { + "n_embd", + "n_expanded_embd", + "n_head", + "n_kv_head", + "num_tensor_parallel_workers", + "num_tokens", + "use_gated_mlp", + } + ), + timing_columns=frozenset(_COMPUTE_COLUMNS.values()), + ), + model_name=model_name, + hardware_name=hardware_name, + attention_backend=attention_backend, + block_size=block_size, + datatype=datatype, + source_revision=source_revision, + data_digest=digester.hexdigest(), + ) + + @property + def revision(self) -> str: + return self._revision + + def lookup(self, query: InferenceEvidenceQuery) -> InferenceEvidenceResult | None: + if ( + query.model_name != self._model_name + or query.hardware_name != self._hardware_name + or query.datatype != self._datatype + ): + return None + # Vidur's block model contributes one ``add_time`` after the MLP. The + # attention residual in Blueprinting remains explicit work, but it has + # no independently comparable public Vidur component record. + if query.primitive == "residual_add" and query.source_layer != "mlp.residual": + return None + if query.primitive in {"attention_core", "attention_kv_cache_save"}: + values = self._attention_values(query) + else: + values = self._compute_values(query) + if not values: + return None + return InferenceEvidenceResult( + seconds=statistics.median(values) * 1e-3, + provider="vidur-profile-baseline", + revision=self.revision, + match=f"exact-median-{len(values)}", + ) + + def _attention_values(self, query: InferenceEvidenceQuery) -> tuple[float, ...]: + is_prefill = query.phase is InferencePhase.PREFILL + timing_column = ( + "time_stats.attn_kv_cache_save.median" + if query.primitive == "attention_kv_cache_save" + else ("time_stats.attn_prefill.median" if is_prefill else "time_stats.attn_decode.median") + ) + expected_prefill_chunk = query.query_tokens if is_prefill else 0 + # Vidur records the cache length before the current decode token is + # appended, while Blueprinting context_tokens is the number of keys + # visible to attention after that append. + expected_kv_cache = 0 if is_prefill else query.context_tokens - 1 + values = [] + for row in self._attention_rows: + matches = ( + _integer(row, "n_embd") == query.hidden_size + and _integer(row, "n_q_head") == query.attention_heads + and _integer(row, "n_kv_head") == query.attention_heads + and _integer(row, "num_tensor_parallel_workers") == query.tensor_parallel + and _integer(row, "block_size") == self._block_size + and _integer(row, "max_model_len") == query.model_sequence_length + and _integer(row, "batch_size") == query.batch_size + and _integer(row, "prefill_chunk_size") == expected_prefill_chunk + and _integer(row, "kv_cache_size") == expected_kv_cache + and _boolean(row, "is_prefill") is is_prefill + and row.get("attention_backend") == self._attention_backend + ) + value = _milliseconds(row, timing_column) + if matches and value is not None: + values.append(value) + return tuple(values) + + def _compute_values(self, query: InferenceEvidenceQuery) -> tuple[float, ...]: + timing_column = _COMPUTE_COLUMNS.get(query.primitive) + if timing_column is None: + return () + num_tokens = query.batch_size * query.query_tokens + values = [] + for row in self._compute_rows: + matches = ( + _integer(row, "n_embd") == query.hidden_size + and _integer(row, "n_expanded_embd") == query.feedforward_size + and _integer(row, "n_head") == query.attention_heads + and _integer(row, "n_kv_head") == query.attention_heads + and _integer(row, "num_tensor_parallel_workers") == query.tensor_parallel + and _integer(row, "num_tokens") == num_tokens + and _boolean(row, "use_gated_mlp") is False + ) + value = _milliseconds(row, timing_column) + if matches and value is not None: + values.append(value) + return tuple(values) diff --git a/src/blueprinting/compiler/experiments/__init__.py b/src/blueprinting/compiler/experiments/__init__.py index 36ccd96..793cf27 100644 --- a/src/blueprinting/compiler/experiments/__init__.py +++ b/src/blueprinting/compiler/experiments/__init__.py @@ -6,10 +6,36 @@ discover_seqsel_tab5_cases, run_calculon_experiment, ) +from .regression import ( + BaselineRegressionGate, + RegressionCheck, + run_inference_baseline_regression, + run_training_baseline_regression, +) +from .vidur import ( + VidurCaseReport, + VidurComponentComparison, + VidurExperimentCase, + VidurExperimentReport, + VidurPhaseComparison, + compare_inference_phase_to_vidur, + run_vidur_experiment, +) __all__ = [ "CalculonCase", "CalculonExperimentReport", + "BaselineRegressionGate", + "RegressionCheck", "discover_seqsel_tab5_cases", "run_calculon_experiment", + "run_inference_baseline_regression", + "run_training_baseline_regression", + "VidurCaseReport", + "VidurComponentComparison", + "VidurExperimentCase", + "VidurExperimentReport", + "VidurPhaseComparison", + "compare_inference_phase_to_vidur", + "run_vidur_experiment", ] diff --git a/src/blueprinting/compiler/experiments/regression.py b/src/blueprinting/compiler/experiments/regression.py new file mode 100644 index 0000000..75612ad --- /dev/null +++ b/src/blueprinting/compiler/experiments/regression.py @@ -0,0 +1,468 @@ +"""Repository quality gates for independent training and inference baselines.""" + +from __future__ import annotations + +import hashlib +import json +import math +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from ..analysis import HardwareProfile, VidurProfileBaseline +from ..bindings import InferencePhase +from ..models import TransformerInferenceExecutionSpec, TransformerModelSpec +from .calculon import CalculonExperimentReport, discover_seqsel_tab5_cases, run_calculon_experiment +from .vidur import VidurExperimentCase, VidurExperimentReport, run_vidur_experiment + +_CONTRACT_PATH = Path("data/validation/baseline_regression_contract.json") + + +@dataclass(frozen=True) +class RegressionCheck: + """One named, inspectable quality-gate predicate.""" + + name: str + passed: bool + expected: str + actual: Any + + def to_dict(self) -> dict[str, Any]: + return { + "name": self.name, + "passed": self.passed, + "expected": self.expected, + "actual": self.actual, + } + + +@dataclass(frozen=True) +class BaselineRegressionGate: + """A deterministic set of checks suitable for tests and CI artifacts.""" + + schema: str + domain: str + checks: tuple[RegressionCheck, ...] + + @property + def ok(self) -> bool: + return all(check.passed for check in self.checks) + + @property + def failures(self) -> tuple[RegressionCheck, ...]: + return tuple(check for check in self.checks if not check.passed) + + def require(self) -> None: + if self.ok: + return + details = "\n".join( + f"- {check.name}: expected {check.expected}; actual={check.actual!r}" for check in self.failures + ) + raise AssertionError(f"{self.domain} baseline regression failed:\n{details}") + + def to_dict(self) -> dict[str, Any]: + return { + "schema": self.schema, + "domain": self.domain, + "ok": self.ok, + "checks": [check.to_dict() for check in self.checks], + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), indent=2, sort_keys=True, ensure_ascii=False) + "\n" + + +def _exact(name: str, actual: Any, expected: Any) -> RegressionCheck: + return RegressionCheck(name, actual == expected, f"exactly {expected!r}", actual) + + +def _close(name: str, actual: float | None, expected: float, *, absolute_tolerance: float) -> RegressionCheck: + passed = actual is not None and math.isclose(actual, expected, rel_tol=0.0, abs_tol=absolute_tolerance) + return RegressionCheck(name, passed, f"{expected!r} ± {absolute_tolerance:g}", actual) + + +def _at_most(name: str, actual: float | None, maximum: float) -> RegressionCheck: + return RegressionCheck(name, actual is not None and actual <= maximum, f"<= {maximum!r}", actual) + + +def _at_least(name: str, actual: float | None, minimum: float) -> RegressionCheck: + return RegressionCheck(name, actual is not None and actual >= minimum, f">= {minimum!r}", actual) + + +def _read_json(path: Path) -> dict[str, Any]: + with path.open(encoding="utf-8") as stream: + value = json.load(stream) + if not isinstance(value, dict): + raise TypeError(f"{path} must contain a JSON object") + return value + + +def _contract(repository_root: Path) -> dict[str, Any]: + contract = _read_json(repository_root / _CONTRACT_PATH) + if contract.get("schema") != "blueprinting.baseline-regression-contract.v1": + raise ValueError("unsupported baseline regression contract schema") + return contract + + +def _sha256(path: Path) -> str: + digester = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digester.update(chunk) + return digester.hexdigest() + + +def _fixture_path(fixture_root: Path, relative_path: str) -> Path: + path = (fixture_root / relative_path).resolve() + if not path.is_relative_to(fixture_root): + raise ValueError(f"fixture path escapes validation slice: {relative_path!r}") + return path + + +def _training_checks(report: CalculonExperimentReport, contract: dict[str, Any]) -> tuple[RegressionCheck, ...]: + budgets = contract["budgets"] + golden = contract["golden"] + checks = [ + _exact("training.case_count", len(report.cases), contract["case_count"]), + _exact("training.evidence_revision", report.evidence_revision, contract["evidence_revision"]), + _at_most( + "training.workload_max_absolute_error_percent", + report.workload_max_absolute_error_percent, + budgets["workload_max_absolute_error_percent"], + ), + _at_most( + "training.calculon_mean_absolute_error_percent", + report.calibrated_mean_absolute_error_percent, + budgets["calculon_mean_absolute_error_percent"], + ), + _at_most( + "training.calculon_max_absolute_error_percent", + report.calibrated_max_absolute_error_percent, + budgets["calculon_max_absolute_error_percent"], + ), + _at_most( + "training.paper_mean_absolute_error_percent", + report.paper_mean_absolute_error_percent, + budgets["paper_mean_absolute_error_percent"], + ), + _at_most( + "training.paper_max_absolute_error_percent", + report.paper_max_absolute_error_percent, + budgets["paper_max_absolute_error_percent"], + ), + _close( + "training.golden.peak_mean_absolute_error_percent", + report.peak_mean_absolute_error_percent, + golden["peak_mean_absolute_error_percent"], + absolute_tolerance=1e-12, + ), + _close( + "training.golden.calculon_mean_absolute_error_percent", + report.calibrated_mean_absolute_error_percent, + golden["calculon_mean_absolute_error_percent"], + absolute_tolerance=1e-12, + ), + _close( + "training.golden.paper_mean_absolute_error_percent", + report.paper_mean_absolute_error_percent, + golden["paper_mean_absolute_error_percent"], + absolute_tolerance=1e-12, + ), + _close( + "training.golden.paper_max_absolute_error_percent", + report.paper_max_absolute_error_percent, + golden["paper_max_absolute_error_percent"], + absolute_tolerance=1e-12, + ), + _exact( + "training.policy.fit_against_case_outputs", report.calibration_policy["fit_against_case_outputs"], False + ), + ] + memory_error = max( + abs(case.calibrated.memory.total - case.calculon_stats["proc_mem_tier1_cap_req"]) for case in report.cases + ) + checks.append( + _at_most( + "training.memory_max_absolute_error_bytes", + memory_error, + budgets["memory_max_absolute_error_bytes"], + ) + ) + reports_by_name = {case.case: case for case in report.cases} + checks.append( + _exact( + "training.case_names", + tuple(reports_by_name), + tuple(golden["portable_digests"]), + ) + ) + for case_name, expected_digest in golden["portable_digests"].items(): + case = reports_by_name.get(case_name) + checks.append( + _exact( + f"training.{case_name}.portable_digest", + case.portable_digest if case is not None else None, + expected_digest, + ) + ) + return tuple(checks) + + +def run_training_baseline_regression(repository_root: str | Path) -> BaselineRegressionGate: + """Run all eight Calculon/SeqSel cases against their frozen contract.""" + + root = Path(repository_root).resolve() + contract = _contract(root)["training"] + report = run_calculon_experiment(discover_seqsel_tab5_cases(root / "data")) + return BaselineRegressionGate( + schema="blueprinting.baseline-regression-gate.v1", + domain="training/calculon", + checks=_training_checks(report, contract), + ) + + +def _load_vidur_report( + repository_root: Path, + contract: dict[str, Any], +) -> tuple[VidurExperimentReport, dict[str, Any], tuple[RegressionCheck, ...]]: + fixture_root = (repository_root / contract["fixture"]).resolve() + if not fixture_root.is_relative_to(repository_root): + raise ValueError("Vidur fixture path escapes the repository") + manifest_path = fixture_root / "manifest.json" + manifest = _read_json(manifest_path) + license_path = _fixture_path(fixture_root, manifest["source"]["license_file"]) + checks = [ + _exact( + "inference.fixture.manifest_sha256", + _sha256(manifest_path), + contract["fixture_manifest_sha256"], + ), + _exact("inference.fixture.schema", manifest.get("schema"), "blueprinting.vidur-validation-slice.v1"), + _exact( + "inference.fixture.source_repository", + manifest["source"].get("repository"), + contract["source_repository"], + ), + _exact( + "inference.fixture.source_revision", + manifest["source"].get("revision"), + contract["source_revision"], + ), + _exact("inference.fixture.license", manifest["source"].get("license"), "MIT"), + _exact( + "inference.fixture.license_file", + manifest["source"].get("license_file"), + contract["license_file"], + ), + _exact( + "inference.fixture.license_file_exists", + license_path.is_file(), + True, + ), + _exact( + "inference.fixture.license_sha256", + _sha256(license_path), + contract["license_sha256"], + ), + ] + for file_name, metadata in manifest["files"].items(): + fixture_path = _fixture_path(fixture_root, file_name) + checks.append( + _exact( + f"inference.fixture.{file_name}.sha256", + _sha256(fixture_path), + metadata["sha256"], + ) + ) + + model = TransformerModelSpec(**manifest["blueprinting"]["model"]) + execution = TransformerInferenceExecutionSpec(**manifest["blueprinting"]["execution"]) + hardware_manifest = manifest["blueprinting"]["hardware"] + hardware = HardwareProfile.from_mapping( + hardware_manifest["name"], + _read_json(repository_root / hardware_manifest["profile"]), + datatype=execution.datatype, + ) + baseline = VidurProfileBaseline.from_csv( + attention_csv=_fixture_path(fixture_root, "attention.csv"), + compute_csv=_fixture_path(fixture_root, "mlp.csv"), + model_name=model.name, + hardware_name=hardware.name, + attention_backend=manifest["selection"]["attention_backend"], + block_size=manifest["selection"]["block_size"], + source_revision=manifest["source"]["revision"], + datatype=execution.datatype, + ) + cases = tuple( + VidurExperimentCase( + name=(f"phi2-a100-tp1/{case_data['phase']}/b{case_data['batch_size']}-c{case_data['context_tokens']}"), + model=model, + execution=execution, + hardware=hardware, + phase=InferencePhase(case_data["phase"]), + batch_size=case_data["batch_size"], + context_tokens=case_data["context_tokens"], + ) + for case_data in manifest["selection"]["cases"] + ) + return run_vidur_experiment(cases, baseline), manifest, tuple(checks) + + +def _inference_checks( + report: VidurExperimentReport, + contract: dict[str, Any], + fixture_checks: tuple[RegressionCheck, ...], +) -> tuple[RegressionCheck, ...]: + budgets = contract["budgets"] + golden = contract["golden"] + comparable_subtotal_errors = tuple( + abs(case.system_evidence.comparable_subtotal_relative_error_percent) + for case in report.cases + if case.system_evidence.comparable_subtotal_relative_error_percent is not None + ) + maximum_comparable_subtotal_error = max(comparable_subtotal_errors) if comparable_subtotal_errors else None + peak_subtotal_mape = report.peak_comparable_subtotal_mean_absolute_error_percent + system_subtotal_mape = report.system_evidence_comparable_subtotal_mean_absolute_error_percent + system_component_mape = report.system_evidence_component_mean_absolute_error_percent + system_component_max = report.system_evidence_component_max_absolute_error_percent + improvement = ( + peak_subtotal_mape - system_subtotal_mape + if peak_subtotal_mape is not None and system_subtotal_mape is not None + else None + ) + checks = [ + *fixture_checks, + _exact("inference.case_count", len(report.cases), contract["case_count"]), + _exact("inference.baseline_revision", report.baseline_revision, contract["baseline_revision"]), + _exact("inference.policy.baseline_role", report.policy["baseline_role"], "post-hoc-comparison-only"), + _exact("inference.policy.oracle_read_during_lowering", report.policy["oracle_read_during_lowering"], False), + _exact("inference.policy.oracle_read_during_costing", report.policy["oracle_read_during_costing"], False), + _exact("inference.policy.fit_against_case_outputs", report.policy["fit_against_case_outputs"], False), + _exact( + "inference.policy.validation_claim", + report.policy["validation_claim"], + contract["validation_claim"], + ), + _exact( + "inference.policy.topology_equivalence", + report.policy["topology_equivalence"], + "not-claimed-by-raw-component-profile-alignment", + ), + _at_most( + "inference.system_evidence_comparable_subtotal_mean_absolute_error_percent", + system_subtotal_mape, + budgets["system_evidence_comparable_subtotal_mean_absolute_error_percent"], + ), + _at_most( + "inference.system_evidence_comparable_subtotal_max_absolute_error_percent", + maximum_comparable_subtotal_error, + budgets["system_evidence_comparable_subtotal_max_absolute_error_percent"], + ), + _at_least( + "inference.improvement_over_peak_percentage_points", + improvement, + budgets["minimum_improvement_over_peak_percentage_points"], + ), + _close( + "inference.golden.peak_comparable_subtotal_mean_absolute_error_percent", + peak_subtotal_mape, + golden["peak_comparable_subtotal_mean_absolute_error_percent"], + absolute_tolerance=1e-9, + ), + _close( + "inference.golden.system_evidence_comparable_subtotal_mean_absolute_error_percent", + system_subtotal_mape, + golden["system_evidence_comparable_subtotal_mean_absolute_error_percent"], + absolute_tolerance=1e-9, + ), + _close( + "inference.golden.system_evidence_component_mean_absolute_error_percent", + system_component_mape, + golden["system_evidence_component_mean_absolute_error_percent"], + absolute_tolerance=1e-9, + ), + _close( + "inference.golden.system_evidence_component_max_absolute_error_percent", + system_component_max, + golden["system_evidence_component_max_absolute_error_percent"], + absolute_tolerance=1e-9, + ), + ] + reports_by_name = {case.case: case for case in report.cases} + checks.append(_exact("inference.case_names", tuple(reports_by_name), tuple(golden["cases"]))) + for case_name, expected in golden["cases"].items(): + case = reports_by_name.get(case_name) + if case is None: + checks.append(_exact(f"inference.{case_name}.present", False, True)) + continue + comparison = case.system_evidence + checks.extend( + ( + _exact(f"inference.{case_name}.model_digest", case.model_digest, contract["model_digest"]), + _exact( + f"inference.{case_name}.distributed_digest", + case.distributed_digest, + expected["distributed_digest"], + ), + _exact( + f"inference.{case_name}.portable_digest", + case.portable_digest, + expected["portable_digest"], + ), + _at_least( + f"inference.{case_name}.component_coverage_budget", + comparison.component_coverage, + budgets["minimum_semantic_component_coverage"], + ), + _close( + f"inference.{case_name}.component_coverage_golden", + comparison.component_coverage, + expected["component_coverage"], + absolute_tolerance=1e-12, + ), + _close( + f"inference.{case_name}.baseline_comparable_block_seconds", + comparison.baseline_comparable_block_seconds, + expected["baseline_comparable_block_seconds"], + absolute_tolerance=1e-15, + ), + _close( + f"inference.{case_name}.system_evidence_comparable_block_seconds", + comparison.compiled_comparable_block_seconds, + expected["system_evidence_comparable_block_seconds"], + absolute_tolerance=1e-15, + ), + _close( + f"inference.{case_name}.system_evidence_comparable_subtotal_error_percent", + comparison.comparable_subtotal_relative_error_percent, + expected["system_evidence_comparable_subtotal_error_percent"], + absolute_tolerance=1e-9, + ), + _close( + f"inference.{case_name}.system_evidence_component_mean_absolute_error_percent", + comparison.component_mean_absolute_error_percent, + expected["system_evidence_component_mean_absolute_error_percent"], + absolute_tolerance=1e-9, + ), + _close( + f"inference.{case_name}.system_evidence_component_max_absolute_error_percent", + comparison.component_max_absolute_error_percent, + expected["system_evidence_component_max_absolute_error_percent"], + absolute_tolerance=1e-9, + ), + ) + ) + return tuple(checks) + + +def run_inference_baseline_regression(repository_root: str | Path) -> BaselineRegressionGate: + """Run the pinned offline Vidur slice against its accuracy and golden contract.""" + + root = Path(repository_root).resolve() + contract = _contract(root)["inference"] + report, _, fixture_checks = _load_vidur_report(root, contract) + return BaselineRegressionGate( + schema="blueprinting.baseline-regression-gate.v1", + domain="inference/vidur", + checks=_inference_checks(report, contract, fixture_checks), + ) diff --git a/src/blueprinting/compiler/experiments/vidur.py b/src/blueprinting/compiler/experiments/vidur.py new file mode 100644 index 0000000..ee4a6d4 --- /dev/null +++ b/src/blueprinting/compiler/experiments/vidur.py @@ -0,0 +1,417 @@ +"""Vidur baseline comparison for Blueprinting's independent inference path. + +Vidur data is read only after ModelIR -> DistributedTaskIR -> PortablePlanIR +lowering and Blueprinting cost evaluation have completed. The comparison +therefore measures agreement; it cannot make the compiled result agree by +construction. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Any + +from ..analysis import ( + CalibrationMode, + HardwareProfile, + InferenceBaseline, + InferencePhaseEstimate, + estimate_inference_phase, + inference_evidence_query_for, +) +from ..bindings import InferencePhase +from ..ir import PortablePlanIR +from ..lowering import DistributeTransformerInferencePass, PlanTransformerInferencePass +from ..models import ( + TransformerInferenceExecutionSpec, + TransformerModelSpec, + build_transformer_inference_model_ir, + inference_compilation_session_for, +) +from ..passes import PassManager, PassPipeline + + +@dataclass(frozen=True) +class VidurComponentComparison: + """One independently costed Blueprinting task and its optional Vidur peer.""" + + task_name: str + source_layer: str + primitive: str + compiled_seconds: float + baseline_seconds: float | None + baseline_match: str | None + + @property + def comparable(self) -> bool: + return self.baseline_seconds is not None + + @property + def absolute_error_seconds(self) -> float | None: + if self.baseline_seconds is None: + return None + return self.compiled_seconds - self.baseline_seconds + + @property + def relative_error_percent(self) -> float | None: + if self.baseline_seconds is None or self.baseline_seconds == 0: + return None + return (self.compiled_seconds - self.baseline_seconds) / self.baseline_seconds * 100 + + def to_dict(self) -> dict[str, Any]: + return { + "task": self.task_name, + "source_layer": self.source_layer, + "primitive": self.primitive, + "compiled_seconds": self.compiled_seconds, + "baseline_seconds": self.baseline_seconds, + "baseline_match": self.baseline_match, + "absolute_error_seconds": self.absolute_error_seconds, + "relative_error_percent": self.relative_error_percent, + "status": "comparable" if self.comparable else "not-covered", + } + + +@dataclass(frozen=True) +class VidurPhaseComparison: + """Comparison over the intersection of Blueprinting and Vidur semantics.""" + + mode: CalibrationMode + phase: InferencePhase + batch_size: int + query_tokens: int + context_tokens: int + plan_digest: str + hardware_revision: str + baseline_revision: str + compiled_block_seconds: float + components: tuple[VidurComponentComparison, ...] + + @property + def matched_components(self) -> tuple[VidurComponentComparison, ...]: + return tuple(component for component in self.components if component.comparable) + + @property + def compiled_comparable_block_seconds(self) -> float: + return sum(component.compiled_seconds for component in self.matched_components) + + @property + def baseline_comparable_block_seconds(self) -> float: + return sum(component.baseline_seconds or 0.0 for component in self.matched_components) + + @property + def excluded_compiled_block_seconds(self) -> float: + return self.compiled_block_seconds - self.compiled_comparable_block_seconds + + @property + def component_coverage(self) -> float: + return len(self.matched_components) / len(self.components) if self.components else 0.0 + + @property + def comparable_subtotal_relative_error_percent(self) -> float | None: + reference = self.baseline_comparable_block_seconds + if reference == 0: + return None + return (self.compiled_comparable_block_seconds - reference) / reference * 100 + + @property + def component_absolute_percentage_errors(self) -> tuple[float, ...]: + """Absolute component errors before aggregation, so cancellation is impossible.""" + + return tuple( + abs(error) + for component in self.matched_components + if (error := component.relative_error_percent) is not None + ) + + @property + def component_mean_absolute_error_percent(self) -> float | None: + errors = self.component_absolute_percentage_errors + return sum(errors) / len(errors) if errors else None + + @property + def component_max_absolute_error_percent(self) -> float | None: + errors = self.component_absolute_percentage_errors + return max(errors) if errors else None + + def to_dict(self) -> dict[str, Any]: + return { + "mode": self.mode.value, + "phase": self.phase.value, + "batch_size": self.batch_size, + "query_tokens": self.query_tokens, + "context_tokens": self.context_tokens, + "plan_digest": self.plan_digest, + "hardware_revision": self.hardware_revision, + "baseline_revision": self.baseline_revision, + "component_coverage": self.component_coverage, + "matched_component_count": len(self.matched_components), + "component_count": len(self.components), + "compiled_block_seconds": self.compiled_block_seconds, + "compiled_comparable_block_seconds": self.compiled_comparable_block_seconds, + "baseline_comparable_block_seconds": self.baseline_comparable_block_seconds, + "excluded_compiled_block_seconds": self.excluded_compiled_block_seconds, + "comparable_subtotal_relative_error_percent": self.comparable_subtotal_relative_error_percent, + "component_mean_absolute_error_percent": self.component_mean_absolute_error_percent, + "component_max_absolute_error_percent": self.component_max_absolute_error_percent, + "components": [component.to_dict() for component in self.components], + } + + +@dataclass(frozen=True) +class VidurExperimentCase: + """One static phase point compiled by Blueprinting before comparison.""" + + name: str + model: TransformerModelSpec + execution: TransformerInferenceExecutionSpec + hardware: HardwareProfile + phase: InferencePhase + batch_size: int + context_tokens: int + + def __post_init__(self) -> None: + if not isinstance(self.name, str) or not self.name: + raise ValueError("Vidur experiment case name must not be empty") + if not isinstance(self.phase, InferencePhase): + raise TypeError("phase must be InferencePhase") + for field_name in ("batch_size", "context_tokens"): + value = getattr(self, field_name) + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError(f"{field_name} must be a positive integer") + self.execution.validate_model(self.model) + if self.context_tokens > self.model.sequence_length: + raise ValueError("context_tokens cannot exceed model sequence_length") + if self.hardware.datatype != self.execution.datatype: + raise ValueError("hardware and execution datatype must match") + + +@dataclass(frozen=True) +class VidurCaseReport: + case: str + model_digest: str + distributed_digest: str + portable_digest: str + pass_checkpoints: tuple[dict[str, Any], ...] + peak_only: VidurPhaseComparison + system_evidence: VidurPhaseComparison + + def to_dict(self) -> dict[str, Any]: + return { + "case": self.case, + "ir": { + "model_digest": self.model_digest, + "distributed_digest": self.distributed_digest, + "portable_digest": self.portable_digest, + "pass_checkpoints": list(self.pass_checkpoints), + }, + "peak_only": self.peak_only.to_dict(), + "system_evidence": self.system_evidence.to_dict(), + } + + +@dataclass(frozen=True) +class VidurExperimentReport: + schema: str + baseline_revision: str + policy: dict[str, Any] + cases: tuple[VidurCaseReport, ...] + + @staticmethod + def _comparable_subtotal_mean_absolute_error( + comparisons: tuple[VidurPhaseComparison, ...], + ) -> float | None: + errors = tuple( + abs(item.comparable_subtotal_relative_error_percent) + for item in comparisons + if item.comparable_subtotal_relative_error_percent is not None + ) + return sum(errors) / len(errors) if errors else None + + @property + def peak_comparable_subtotal_mean_absolute_error_percent(self) -> float | None: + return self._comparable_subtotal_mean_absolute_error(tuple(case.peak_only for case in self.cases)) + + @property + def system_evidence_comparable_subtotal_mean_absolute_error_percent(self) -> float | None: + return self._comparable_subtotal_mean_absolute_error(tuple(case.system_evidence for case in self.cases)) + + @staticmethod + def _component_errors(comparisons: tuple[VidurPhaseComparison, ...]) -> tuple[float, ...]: + return tuple(error for comparison in comparisons for error in comparison.component_absolute_percentage_errors) + + @classmethod + def _component_mean_absolute_error(cls, comparisons: tuple[VidurPhaseComparison, ...]) -> float | None: + errors = cls._component_errors(comparisons) + return sum(errors) / len(errors) if errors else None + + @classmethod + def _component_max_absolute_error(cls, comparisons: tuple[VidurPhaseComparison, ...]) -> float | None: + errors = cls._component_errors(comparisons) + return max(errors) if errors else None + + @property + def peak_component_mean_absolute_error_percent(self) -> float | None: + return self._component_mean_absolute_error(tuple(case.peak_only for case in self.cases)) + + @property + def system_evidence_component_mean_absolute_error_percent(self) -> float | None: + return self._component_mean_absolute_error(tuple(case.system_evidence for case in self.cases)) + + @property + def system_evidence_component_max_absolute_error_percent(self) -> float | None: + return self._component_max_absolute_error(tuple(case.system_evidence for case in self.cases)) + + def to_dict(self) -> dict[str, Any]: + return { + "schema": self.schema, + "baseline_revision": self.baseline_revision, + "policy": self.policy, + "summary": { + "case_count": len(self.cases), + "peak_comparable_subtotal_mean_absolute_error_percent": ( + self.peak_comparable_subtotal_mean_absolute_error_percent + ), + "system_evidence_comparable_subtotal_mean_absolute_error_percent": ( + self.system_evidence_comparable_subtotal_mean_absolute_error_percent + ), + "peak_component_mean_absolute_error_percent": self.peak_component_mean_absolute_error_percent, + "system_evidence_component_mean_absolute_error_percent": ( + self.system_evidence_component_mean_absolute_error_percent + ), + "system_evidence_component_max_absolute_error_percent": ( + self.system_evidence_component_max_absolute_error_percent + ), + }, + "cases": [case.to_dict() for case in self.cases], + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), indent=2, sort_keys=True, ensure_ascii=False) + "\n" + + +def compare_inference_phase_to_vidur( + plan: PortablePlanIR, + estimate: InferencePhaseEstimate, + hardware: HardwareProfile, + baseline: InferenceBaseline, +) -> VidurPhaseComparison: + """Compare an already-lowered and already-costed phase with Vidur.""" + + model = plan.attributes.get("model_spec") + execution = plan.attributes.get("inference_execution_spec") + if not isinstance(model, TransformerModelSpec): + raise TypeError("portable inference plan is missing TransformerModelSpec") + if not isinstance(execution, TransformerInferenceExecutionSpec): + raise TypeError("portable inference plan is missing TransformerInferenceExecutionSpec") + if len(plan.tasks) != len(estimate.tasks): + raise ValueError("plan and estimate task counts differ") + + components = [] + for plan_task, task_estimate in zip(plan.tasks, estimate.tasks): + invocation = task_estimate.invocation + if plan_task.workload.attributes.get("name") != invocation.name: + raise ValueError("plan and estimate task order differs") + reference = baseline.lookup( + inference_evidence_query_for( + invocation, + hardware=hardware, + execution=execution, + model=model, + batch_size=estimate.batch_size, + query_tokens=estimate.query_tokens, + context_tokens=estimate.context_tokens, + ) + ) + components.append( + VidurComponentComparison( + task_name=invocation.name, + source_layer=invocation.source_layer, + primitive=invocation.primitive, + compiled_seconds=task_estimate.total_seconds, + baseline_seconds=reference.seconds if reference is not None else None, + baseline_match=reference.match if reference is not None else None, + ) + ) + return VidurPhaseComparison( + mode=estimate.mode, + phase=estimate.phase, + batch_size=estimate.batch_size, + query_tokens=estimate.query_tokens, + context_tokens=estimate.context_tokens, + plan_digest=plan.digest, + hardware_revision=hardware.evidence_revision, + baseline_revision=baseline.revision, + compiled_block_seconds=estimate.block_seconds, + components=tuple(components), + ) + + +def run_vidur_experiment( + cases: tuple[VidurExperimentCase, ...], + baseline: InferenceBaseline, +) -> VidurExperimentReport: + """Compile each phase independently, then compare both analytical modes.""" + + if not cases: + raise ValueError("Vidur experiment requires at least one case") + reports = [] + pipeline = PassPipeline.of(DistributeTransformerInferencePass(), PlanTransformerInferencePass()) + manager = PassManager() + for case in cases: + source = build_transformer_inference_model_ir(case.model, datatype=case.execution.datatype) + result = manager.run( + pipeline, + source, + session=inference_compilation_session_for( + case.model, + case.execution, + phase=case.phase, + batch_size=case.batch_size, + context_tokens=case.context_tokens, + ), + ) + plan = result.ir + if not isinstance(plan, PortablePlanIR): + raise TypeError(f"inference pipeline returned {type(plan).__name__}, expected PortablePlanIR") + peak = estimate_inference_phase(plan, case.hardware, CalibrationMode.PEAK_ONLY) + system = estimate_inference_phase(plan, case.hardware, CalibrationMode.SYSTEM_EVIDENCE) + checkpoints = tuple( + { + "pass": checkpoint.record.pass_name, + "schema": checkpoint.ir.header.schema_name, + "digest": checkpoint.ir.digest, + } + for checkpoint in result.checkpoints + ) + reports.append( + VidurCaseReport( + case=case.name, + model_digest=source.digest, + distributed_digest=result.checkpoints[0].ir.digest, + portable_digest=plan.digest, + pass_checkpoints=checkpoints, + peak_only=compare_inference_phase_to_vidur(plan, peak, case.hardware, baseline), + system_evidence=compare_inference_phase_to_vidur(plan, system, case.hardware, baseline), + ) + ) + return VidurExperimentReport( + schema="blueprinting.vidur-baseline-experiment.v1", + baseline_revision=baseline.revision, + policy={ + "baseline_role": "post-hoc-comparison-only", + "oracle_read_during_lowering": False, + "oracle_read_during_costing": False, + "comparison_domain": "exact component intersection", + "validation_claim": "drift-detection-not-accuracy-validation", + "topology_equivalence": "not-claimed-by-raw-component-profile-alignment", + "fit_against_case_outputs": False, + "forbidden_inputs": [ + "Vidur component duration during lowering", + "Vidur phase total during costing", + "per-case correction factor", + ], + }, + cases=tuple(reports), + ) diff --git a/src/blueprinting/compiler/ir/portable_plan.py b/src/blueprinting/compiler/ir/portable_plan.py index 6e3c371..3ed6347 100644 --- a/src/blueprinting/compiler/ir/portable_plan.py +++ b/src/blueprinting/compiler/ir/portable_plan.py @@ -390,6 +390,7 @@ def verify(self) -> VerificationReport: not in { PlanBufferRole.CONSTANT, PlanBufferRole.STATE, + PlanBufferRole.WORKSPACE, } ): bag.error("buffer.missing_producer", "non-input buffer requires a producer", *path, "producer") diff --git a/src/blueprinting/compiler/lowering/__init__.py b/src/blueprinting/compiler/lowering/__init__.py index b58bd3d..dcc0de8 100644 --- a/src/blueprinting/compiler/lowering/__init__.py +++ b/src/blueprinting/compiler/lowering/__init__.py @@ -1,5 +1,11 @@ """Production lowering passes for canonical IR dialects.""" from .transformer import DistributeTransformerTrainingPass, PlanTransformerTrainingPass +from .transformer_inference import DistributeTransformerInferencePass, PlanTransformerInferencePass -__all__ = ["DistributeTransformerTrainingPass", "PlanTransformerTrainingPass"] +__all__ = [ + "DistributeTransformerInferencePass", + "DistributeTransformerTrainingPass", + "PlanTransformerInferencePass", + "PlanTransformerTrainingPass", +] diff --git a/src/blueprinting/compiler/lowering/transformer_inference.py b/src/blueprinting/compiler/lowering/transformer_inference.py new file mode 100644 index 0000000..f8f6151 --- /dev/null +++ b/src/blueprinting/compiler/lowering/transformer_inference.py @@ -0,0 +1,449 @@ +"""Inference lowerings from a semantic decoder to a phase-local work plan.""" + +from __future__ import annotations + +from ..analysis.transformer_inference import ( + InferenceBlockMemoryFacts, + InferenceInvocation, + compile_transformer_inference_block, +) +from ..analysis.transformer_workload import EngineKind +from ..axes import BindingAxis +from ..bindings import InferencePhase, WorkloadMode +from ..frozen import FrozenDict +from ..ids import BufferId, Lineage, NodeId, ValueId +from ..ir import ( + AbstractStorageClass, + CollectiveKind, + CollectiveSpec, + DistributedTask, + DistributedTaskIR, + DistributedTaskKind, + DistributedValue, + Effect, + EffectKind, + ImplementationRequirement, + LogicalMesh, + MeshAxis, + ModelIR, + ObjectiveDirection, + ObjectiveKind, + OperationName, + PlanBuffer, + PlanBufferRole, + PlanObjective, + PlanTask, + PlanTaskKind, + PortablePlanIR, + ReductionKind, + ResourceKind, + ResourceRequirement, + ResourceScope, + ShardingSpec, + TensorType, + ValueRole, + WorkloadFacts, +) +from ..models.transformer import TransformerModelSpec +from ..models.transformer_inference import TransformerInferenceExecutionSpec +from ..passes import CompilerPass, PassContext, PassContract + + +def _semantic_specs( + ir: ModelIR, + context: PassContext, +) -> tuple[TransformerModelSpec, TransformerInferenceExecutionSpec, InferencePhase, int, int, int]: + if len(ir.operations) != 1 or ir.operations[0].operation != OperationName("transformer", "decoder_inference"): + raise ValueError("Transformer inference distribution expects one transformer.decoder_inference operation") + model = ir.operations[0].attributes.get("model_spec") + if not isinstance(model, TransformerModelSpec): + raise TypeError("model operation is missing a typed TransformerModelSpec") + workload = context.session.bindings.workload + strategy = context.session.bindings.strategy + if workload is None or strategy is None: + raise ValueError("Transformer inference distribution requires workload and strategy bindings") + if workload.mode is not WorkloadMode.INFERENCE or workload.inference_phase is None: + raise ValueError("Transformer inference requires an explicit inference phase") + execution = strategy.attributes.get("inference_execution_spec") + if not isinstance(execution, TransformerInferenceExecutionSpec): + raise TypeError("strategy binding is missing a typed TransformerInferenceExecutionSpec") + if ( + strategy.tensor_parallel != execution.tensor_parallel + or strategy.pipeline_parallel != execution.pipeline_parallel + or strategy.data_parallel != execution.replicas + ): + raise ValueError("strategy binding is inconsistent with inference execution facts") + if any( + isinstance(value, bool) or not isinstance(value, int) + for value in (workload.batch_size, workload.sequence_length) + ): + raise TypeError("static Transformer inference requires concrete integer workload bindings") + batch_size = workload.batch_size + context_tokens = workload.sequence_length + query_tokens = context_tokens if workload.inference_phase is InferencePhase.PREFILL else 1 + if workload.attributes.get("query_tokens") != query_tokens: + raise ValueError("workload query_tokens attribute is inconsistent with the inference phase") + if workload.attributes.get("context_tokens") != context_tokens: + raise ValueError("workload context_tokens attribute is inconsistent with sequence_length") + execution.validate_model(model) + return model, execution, workload.inference_phase, batch_size, query_tokens, context_tokens + + +class DistributeTransformerInferencePass(CompilerPass[ModelIR, DistributedTaskIR]): + """Expand one phase into observable local and collective components.""" + + contract = PassContract.create( + "transformer-inference-distribute-v1", + ModelIR, + DistributedTaskIR, + required_bindings=frozenset({BindingAxis.WORKLOAD, BindingAxis.STRATEGY}), + ) + + def run(self, ir: ModelIR, context: PassContext) -> DistributedTaskIR: + model, execution, phase, batch_size, query_tokens, context_tokens = _semantic_specs(ir, context) + invocations, block_memory = compile_transformer_inference_block( + model, + execution, + phase=phase, + batch_size=batch_size, + context_tokens=context_tokens, + ) + ranks = tuple(range(execution.tensor_parallel)) + mesh = LogicalMesh("local-tensor-parallel-group", (MeshAxis("tp", execution.tensor_parallel),)) + source_input = ir.inputs[0] + source_output = ir.outputs[0] + source_cache = next(value.id for value in ir.values if value.role is ValueRole.KV_CACHE) + input_id = ValueId.derive(ir.digest, phase.value, "transformer-distributed", "input") + cache_id = ValueId.derive(ir.digest, phase.value, "transformer-distributed", "kv-cache") + output_id = ValueId.derive(ir.digest, phase.value, "transformer-distributed", "output") + boundary_type = TensorType((batch_size, query_tokens, model.hidden_size), execution.datatype) + cache_type = TensorType((2, batch_size, context_tokens, model.hidden_size), execution.datatype) + boundary_sharding = ShardingSpec.replicated(boundary_type.rank, ("tp",)) + cache_sharding = ShardingSpec(((), (), (), ("tp",))) + + task_ids = tuple( + NodeId.derive(ir.digest, phase.value, "transformer-distributed", index, invocation.name) + for index, invocation in enumerate(invocations) + ) + tasks = [] + cache_primitives = frozenset({"attention_kv_cache_save", "attention_core"}) + for index, (task_id, invocation) in enumerate(zip(task_ids, invocations)): + collective = None + kind = DistributedTaskKind.LOCAL_COMPUTE + if invocation.engine is EngineKind.COLLECTIVE: + kind = DistributedTaskKind.COLLECTIVE + collective = CollectiveSpec( + kind=invocation.collective, + participants=ranks, + message_bytes=invocation.work.message_bytes, + reduction=( + ReductionKind.SUM + if invocation.collective in {CollectiveKind.ALL_REDUCE, CollectiveKind.REDUCE_SCATTER} + else None + ), + ) + operation = ( + OperationName("collective", invocation.collective.value) + if invocation.collective is not None + else OperationName("transformer", f"{invocation.primitive}_{phase.value}") + ) + inputs = [] + if index == 0: + inputs.append(input_id) + if invocation.primitive in cache_primitives: + inputs.append(cache_id) + effects = () + if invocation.primitive == "attention_kv_cache_save": + effects = (Effect(EffectKind.WRITE, "kv_cache"),) + elif invocation.primitive == "attention_core": + effects = (Effect(EffectKind.READ, "kv_cache"),) + tasks.append( + DistributedTask( + id=task_id, + kind=kind, + operation=operation, + ranks=ranks, + inputs=tuple(inputs), + outputs=(output_id,) if index == len(invocations) - 1 else (), + dependencies=(task_ids[index - 1],) if index else (), + lineage=Lineage.lowered("transformer-inference-decompose", (ir.operations[0].id,)), + collective=collective, + effects=effects, + attributes=FrozenDict({"invocation": invocation}), + ) + ) + + return DistributedTaskIR( + name=f"{model.name}-{phase.value}-local-tp-block", + source_model_digest=ir.digest, + mesh=mesh, + values=( + DistributedValue( + input_id, + boundary_type, + ValueRole.INPUT, + boundary_sharding, + ranks, + Lineage.lowered("transformer-inference-distribute", (source_input,)), + source_value=source_input, + ), + DistributedValue( + cache_id, + cache_type, + ValueRole.KV_CACHE, + cache_sharding, + ranks, + Lineage.lowered("transformer-inference-distribute", (source_cache,)), + source_value=source_cache, + ), + DistributedValue( + output_id, + boundary_type, + ValueRole.OUTPUT, + boundary_sharding, + ranks, + Lineage.lowered("transformer-inference-distribute", (source_output,)), + source_value=source_output, + ), + ), + tasks=tuple(tasks), + inputs=(input_id, cache_id), + outputs=(output_id,), + attributes=FrozenDict( + { + "model_spec": model, + "inference_execution_spec": execution, + "inference_phase": phase, + "batch_size": batch_size, + "query_tokens": query_tokens, + "context_tokens": context_tokens, + "block_memory": block_memory, + "scope": "one-local-tensor-parallel-block-phase", + } + ), + ) + + +def _plan_resources(invocation: InferenceInvocation) -> tuple[ResourceRequirement, ...]: + resources = [] + if invocation.work.operations: + resources.append( + ResourceRequirement( + ResourceKind.COMPUTE, + invocation.work.operations, + ResourceScope.PER_RANK, + FrozenDict({"engine": invocation.engine.value}), + ) + ) + if invocation.work.memory_bytes: + resources.append( + ResourceRequirement(ResourceKind.MEMORY_BANDWIDTH, invocation.work.memory_bytes, ResourceScope.PER_RANK) + ) + if invocation.work.message_bytes: + resources.append( + ResourceRequirement( + ResourceKind.NETWORK, + invocation.work.message_bytes, + ResourceScope.PER_RANK, + FrozenDict({"network_tier": invocation.network_tier}), + ) + ) + return tuple(resources) + + +def _implementation(invocation: InferenceInvocation) -> ImplementationRequirement: + if invocation.engine is EngineKind.COLLECTIVE: + return ImplementationRequirement( + invocation.collective.value, alternatives=("collective-library", "network-engine") + ) + alternatives = { + "attention_core": ("flash-attention", "paged-attention", "dense-attention"), + "attention_kv_cache_save": ("fused-kv-write", "vector-engine"), + "attention_pre_projection": ("tensor-core", "matrix-engine"), + "attention_post_projection": ("tensor-core", "matrix-engine"), + "mlp_up_projection": ("tensor-core", "matrix-engine"), + "mlp_down_projection": ("tensor-core", "matrix-engine"), + }.get(invocation.primitive) + if alternatives is not None: + return ImplementationRequirement(invocation.primitive, alternatives=alternatives) + return ImplementationRequirement(invocation.primitive, alternatives=("vector-engine",)) + + +class PlanTransformerInferencePass(CompilerPass[DistributedTaskIR, PortablePlanIR]): + """Materialize a phase plan without target placement or measured time.""" + + contract = PassContract.create( + "transformer-inference-plan-work-v1", + DistributedTaskIR, + PortablePlanIR, + required_bindings=frozenset({BindingAxis.WORKLOAD, BindingAxis.STRATEGY}), + ) + + def run(self, ir: DistributedTaskIR, context: PassContext) -> PortablePlanIR: + model = ir.attributes.get("model_spec") + execution = ir.attributes.get("inference_execution_spec") + phase = ir.attributes.get("inference_phase") + block_memory = ir.attributes.get("block_memory") + if not isinstance(model, TransformerModelSpec): + raise TypeError("distributed inference IR is missing TransformerModelSpec") + if not isinstance(execution, TransformerInferenceExecutionSpec): + raise TypeError("distributed inference IR is missing TransformerInferenceExecutionSpec") + if not isinstance(phase, InferencePhase): + raise TypeError("distributed inference IR is missing InferencePhase") + if not isinstance(block_memory, InferenceBlockMemoryFacts): + raise TypeError("distributed inference IR is missing InferenceBlockMemoryFacts") + strategy = context.session.bindings.strategy + if strategy is None: + raise ValueError("portable inference planning requires a strategy binding") + + input_id = BufferId.derive(ir.digest, "transformer-inference-portable", "input") + output_id = BufferId.derive(ir.digest, "transformer-inference-portable", "output") + weight_id = BufferId.derive(ir.digest, "transformer-inference-portable", "weights") + cache_id = BufferId.derive(ir.digest, "transformer-inference-portable", "kv-cache") + workspace_id = BufferId.derive(ir.digest, "transformer-inference-portable", "workspace-upper-bound") + task_ids = tuple( + NodeId.derive(ir.digest, "transformer-inference-portable", index, task.id) + for index, task in enumerate(ir.tasks) + ) + invocations = tuple(task.attributes.get("invocation") for task in ir.tasks) + if any(not isinstance(item, InferenceInvocation) for item in invocations): + raise TypeError("distributed inference task is missing InferenceInvocation") + weight_consumers = tuple( + task_id for task_id, invocation in zip(task_ids, invocations) if invocation.engine is EngineKind.MATRIX + ) + cache_consumers = tuple( + task_id + for task_id, invocation in zip(task_ids, invocations) + if invocation.primitive in {"attention_kv_cache_save", "attention_core"} + ) + + tasks = [] + for index, (source_task, task_id, invocation) in enumerate(zip(ir.tasks, task_ids, invocations)): + inputs = [workspace_id] + if index == 0: + inputs.append(input_id) + if invocation.engine is EngineKind.MATRIX: + inputs.append(weight_id) + if invocation.primitive in {"attention_kv_cache_save", "attention_core"}: + inputs.append(cache_id) + tasks.append( + PlanTask( + id=task_id, + kind=( + PlanTaskKind.COLLECTIVE if invocation.engine is EngineKind.COLLECTIVE else PlanTaskKind.COMPUTE + ), + operation=source_task.operation, + dependencies=(task_ids[index - 1],) if index else (), + inputs=tuple(inputs), + outputs=(output_id,) if index == len(ir.tasks) - 1 else (), + logical_ranks=source_task.ranks, + workload=WorkloadFacts( + operations=invocation.work.operations, + read_bytes=invocation.work.read_bytes, + write_bytes=invocation.work.write_bytes, + message_bytes=invocation.work.message_bytes, + attributes=FrozenDict( + { + "name": invocation.name, + "engine": invocation.engine.value, + "phase": invocation.phase.value, + "primitive": invocation.primitive, + "source_layer": invocation.source_layer, + "query_tokens": ir.attributes["query_tokens"], + "context_tokens": ir.attributes["context_tokens"], + "collective": ( + invocation.collective.value if invocation.collective is not None else "" + ), + "network_tier": ( + invocation.network_tier if invocation.network_tier is not None else -1 + ), + } + ), + ), + lineage=Lineage.lowered("transformer-inference-plan-work", (source_task.id,)), + resources=_plan_resources(invocation), + implementations=(_implementation(invocation),), + concurrency_group=("network" if invocation.engine is EngineKind.COLLECTIVE else "compute"), + effects=source_task.effects, + ) + ) + + return PortablePlanIR( + name=f"{model.name}-{phase.value}-local-tp-block-plan", + source_distributed_digest=ir.digest, + strategy_fingerprint=strategy.fingerprint, + planner_revision="transformer-inference-work-analysis-v1", + tasks=tuple(tasks), + buffers=( + PlanBuffer( + input_id, + block_memory.boundary, + PlanBufferRole.INPUT, + AbstractStorageClass.DEVICE_LOCAL, + Lineage.lowered("transformer-inference-plan-buffer", (ir.inputs[0],)), + consumers=(task_ids[0],), + alignment_bytes=16, + ), + PlanBuffer( + output_id, + block_memory.boundary, + PlanBufferRole.OUTPUT, + AbstractStorageClass.DEVICE_LOCAL, + Lineage.lowered("transformer-inference-plan-buffer", (ir.outputs[0],)), + producer=task_ids[-1], + alignment_bytes=16, + ), + PlanBuffer( + weight_id, + block_memory.weights, + PlanBufferRole.CONSTANT, + AbstractStorageClass.PERSISTENT, + Lineage.lowered("transformer-inference-plan-weights", tuple(task.id for task in ir.tasks)), + consumers=weight_consumers, + alignment_bytes=16, + attributes=FrozenDict({"semantic": "block_weights"}), + ), + PlanBuffer( + cache_id, + block_memory.kv_cache, + PlanBufferRole.STATE, + AbstractStorageClass.PERSISTENT, + Lineage.lowered("transformer-inference-plan-cache", (ir.inputs[1],)), + consumers=cache_consumers, + alignment_bytes=16, + attributes=FrozenDict({"semantic": "kv_cache", "phase": phase.value}), + ), + PlanBuffer( + workspace_id, + block_memory.working_upper_bound, + PlanBufferRole.WORKSPACE, + AbstractStorageClass.TRANSIENT, + Lineage.lowered("transformer-inference-plan-workspace", tuple(task.id for task in ir.tasks)), + consumers=task_ids, + alignment_bytes=16, + attributes=FrozenDict( + { + "semantic": "block_working_upper_bound", + "bound": "unfused-score-materialization", + } + ), + ), + ), + inputs=(input_id,), + outputs=(output_id,), + objectives=( + PlanObjective(ObjectiveKind.LATENCY, ObjectiveDirection.MINIMIZE), + PlanObjective(ObjectiveKind.PEAK_MEMORY, ObjectiveDirection.MINIMIZE), + ), + attributes=FrozenDict( + { + "model_spec": model, + "inference_execution_spec": execution, + "inference_phase": phase, + "batch_size": ir.attributes["batch_size"], + "query_tokens": ir.attributes["query_tokens"], + "context_tokens": ir.attributes["context_tokens"], + "scope": ir.attributes["scope"], + } + ), + ) diff --git a/src/blueprinting/compiler/models/__init__.py b/src/blueprinting/compiler/models/__init__.py index 4d4aa34..8ff0c81 100644 --- a/src/blueprinting/compiler/models/__init__.py +++ b/src/blueprinting/compiler/models/__init__.py @@ -8,12 +8,22 @@ build_transformer_model_ir, compilation_session_for, ) +from .transformer_inference import ( + TransformerInferenceExecutionSpec, + TransformerInferenceRequestSpec, + build_transformer_inference_model_ir, + inference_compilation_session_for, +) __all__ = [ "RecomputePolicy", "TensorParallelCommunication", "TransformerExecutionSpec", + "TransformerInferenceExecutionSpec", + "TransformerInferenceRequestSpec", "TransformerModelSpec", + "build_transformer_inference_model_ir", "build_transformer_model_ir", "compilation_session_for", + "inference_compilation_session_for", ] diff --git a/src/blueprinting/compiler/models/transformer_inference.py b/src/blueprinting/compiler/models/transformer_inference.py new file mode 100644 index 0000000..e85ec44 --- /dev/null +++ b/src/blueprinting/compiler/models/transformer_inference.py @@ -0,0 +1,247 @@ +"""Typed frontend facts for decoder-only Transformer inference. + +Inference keeps three concerns separate: + +* :class:`TransformerModelSpec` describes model structure; +* :class:`TransformerInferenceExecutionSpec` describes a logical mapping; +* :class:`TransformerInferenceRequestSpec` describes one request cohort. + +The frontend emits a phase-neutral semantic operation. ``PREFILL`` and +``DECODE`` become explicit workload bindings, so the same model snapshot can +be specialized independently for request-level simulation later on. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any + +from ..axes import BindingAxis +from ..bindings import ( + BindingSet, + InferencePhase, + StrategyBinding, + WorkloadBinding, + WorkloadMode, +) +from ..codec import record_type +from ..expr import Symbol +from ..frozen import FrozenDict +from ..ids import Lineage, NodeId, ValueId +from ..ir import Effect, EffectKind, ModelIR, ModelOperation, ModelValue, OperationName, TensorType, ValueRole +from ..session import CompilationSession +from .transformer import TransformerModelSpec + +_SUPPORTED_DATATYPES = frozenset({"float8", "float16", "bfloat16", "float32"}) + + +def _positive_integer(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError(f"{name} must be a positive integer") + return value + + +@record_type("compiler.transformer.inference_execution_spec.v1") +@dataclass(frozen=True) +class TransformerInferenceExecutionSpec: + """Target-neutral logical mapping for an inference replica.""" + + world_size: int + tensor_parallel: int + pipeline_parallel: int + replicas: int + datatype: str + tensor_parallel_network: int + pipeline_parallel_network: int + + def __post_init__(self) -> None: + for field_name in ("world_size", "tensor_parallel", "pipeline_parallel", "replicas"): + _positive_integer(getattr(self, field_name), field_name) + for field_name in ("tensor_parallel_network", "pipeline_parallel_network"): + value = getattr(self, field_name) + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError(f"{field_name} must be a non-negative integer") + if self.world_size != self.tensor_parallel * self.pipeline_parallel * self.replicas: + raise ValueError("world_size must equal tensor_parallel * pipeline_parallel * replicas") + if self.datatype not in _SUPPORTED_DATATYPES: + raise ValueError(f"unsupported datatype: {self.datatype!r}") + + @property + def bytes_per_element(self) -> int: + return {"float8": 1, "float16": 2, "bfloat16": 2, "float32": 4}[self.datatype] + + def validate_model(self, model: TransformerModelSpec) -> None: + """Reject mappings whose local tensor shapes are not integral.""" + + divisibility = { + "hidden_size": model.hidden_size, + "feedforward_size": model.feedforward_size, + "attention_heads": model.attention_heads, + } + for name, value in divisibility.items(): + if value % self.tensor_parallel: + raise ValueError(f"{name} must be divisible by tensor_parallel") + if self.pipeline_parallel > model.block_count: + raise ValueError("pipeline_parallel cannot exceed block_count") + if model.block_count % self.pipeline_parallel: + raise ValueError("pipeline_parallel must divide block_count for static inference planning") + + @classmethod + def from_mapping(cls, data: Mapping[str, Any]) -> TransformerInferenceExecutionSpec: + replicas = data.get("replicas", data.get("data_par", 1)) + tensor_parallel = data["tensor_par"] + pipeline_parallel = data["pipeline_par"] + return cls( + world_size=data.get("num_procs", tensor_parallel * pipeline_parallel * replicas), + tensor_parallel=tensor_parallel, + pipeline_parallel=pipeline_parallel, + replicas=replicas, + datatype=data["datatype"], + tensor_parallel_network=data.get("tensor_par_net", 0), + pipeline_parallel_network=data.get("pipeline_par_net", 0), + ) + + +@record_type("compiler.transformer.inference_request_spec.v1") +@dataclass(frozen=True) +class TransformerInferenceRequestSpec: + """A homogeneous request cohort before online scheduling is applied.""" + + batch_size: int + prompt_tokens: int + generated_tokens: int + + def __post_init__(self) -> None: + for field_name in ("batch_size", "prompt_tokens", "generated_tokens"): + _positive_integer(getattr(self, field_name), field_name) + + @property + def decode_iterations(self) -> int: + """Iterations after prefill; prefill itself emits the first token.""" + + return self.generated_tokens - 1 + + @property + def final_context_tokens(self) -> int: + """Largest KV context consumed while producing this cohort.""" + + return self.prompt_tokens + self.decode_iterations + + def decode_contexts(self) -> range: + return range(self.prompt_tokens + 1, self.final_context_tokens + 1) + + def validate_model(self, model: TransformerModelSpec) -> None: + if self.final_context_tokens > model.sequence_length: + raise ValueError("prompt_tokens + generated_tokens - 1 cannot exceed the model sequence_length") + + @classmethod + def from_mapping(cls, data: Mapping[str, Any]) -> TransformerInferenceRequestSpec: + return cls( + batch_size=data["batch_size"], + prompt_tokens=data["prompt_tokens"], + generated_tokens=data["generated_tokens"], + ) + + +def build_transformer_inference_model_ir( + model: TransformerModelSpec, + *, + datatype: str = "float16", +) -> ModelIR: + """Import a phase-neutral decoder inference operation. + + KV cache is an explicit mutable semantic resource. Its concrete context + extent is supplied by a phase workload binding, not embedded in the model. + """ + + if datatype not in _SUPPORTED_DATATYPES: + raise ValueError(f"unsupported datatype: {datatype!r}") + batch = Symbol("batch_size", BindingAxis.WORKLOAD, positive=True) + context = Symbol("sequence_length", BindingAxis.WORKLOAD, positive=True) + query = Symbol("query_tokens", BindingAxis.WORKLOAD, positive=True) + hidden_type = TensorType((batch, query, model.hidden_size), datatype) + cache_type = TensorType((model.block_count, 2, batch, context, model.hidden_size), datatype) + input_id = ValueId.derive("transformer-inference", model, "hidden-input") + cache_id = ValueId.derive("transformer-inference", model, "kv-cache") + output_id = ValueId.derive("transformer-inference", model, "hidden-output") + operation_id = NodeId.derive("transformer-inference", model, "decoder") + return ModelIR( + name=f"{model.name}-inference", + values=( + ModelValue( + input_id, hidden_type, ValueRole.INPUT, Lineage.root("transformer-inference-import"), "hidden_input" + ), + ModelValue( + cache_id, cache_type, ValueRole.KV_CACHE, Lineage.root("transformer-inference-import"), "kv_cache" + ), + ModelValue( + output_id, + hidden_type, + ValueRole.OUTPUT, + Lineage.lowered("transformer-inference-semantic-op", (input_id,)), + "hidden_output", + ), + ), + operations=( + ModelOperation( + operation_id, + OperationName("transformer", "decoder_inference"), + (input_id, cache_id), + (output_id,), + Lineage.root("transformer-inference-import"), + effects=(Effect(EffectKind.STATE, "kv_cache"),), + attributes=FrozenDict({"model_spec": model}), + ), + ), + inputs=(input_id,), + outputs=(output_id,), + attributes=FrozenDict( + { + "model_family": "decoder-only-transformer", + "workload_mode": WorkloadMode.INFERENCE.value, + } + ), + ) + + +def inference_compilation_session_for( + model: TransformerModelSpec, + execution: TransformerInferenceExecutionSpec, + *, + phase: InferencePhase, + batch_size: int, + context_tokens: int, +) -> CompilationSession: + """Create an explicit phase binding for static inference specialization.""" + + execution.validate_model(model) + _positive_integer(batch_size, "batch_size") + _positive_integer(context_tokens, "context_tokens") + if not isinstance(phase, InferencePhase): + raise TypeError("phase must be InferencePhase") + query_tokens = context_tokens if phase is InferencePhase.PREFILL else 1 + workload = WorkloadBinding( + WorkloadMode.INFERENCE, + batch_size=batch_size, + sequence_length=context_tokens, + inference_phase=phase, + attributes=FrozenDict( + { + "query_tokens": query_tokens, + "context_tokens": context_tokens, + } + ), + ) + strategy = StrategyBinding( + tensor_parallel=execution.tensor_parallel, + pipeline_parallel=execution.pipeline_parallel, + data_parallel=execution.replicas, + recompute_policy="none", + pipeline_policy="static-inference", + attributes=FrozenDict({"inference_execution_spec": execution}), + ) + return CompilationSession( + bindings=BindingSet(workload=workload, strategy=strategy), + features=frozenset({"transformer-inference-analysis-v1", f"inference-{phase.value}"}), + ) diff --git a/tests/application/test_inference_analysis_service.py b/tests/application/test_inference_analysis_service.py new file mode 100644 index 0000000..21970d0 --- /dev/null +++ b/tests/application/test_inference_analysis_service.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from blueprinting.application import BlueprintingService, InferenceAnalysisDraft + +ROOT = Path(__file__).resolve().parents[2] + + +def _json(path: Path): + with path.open(encoding="utf-8") as stream: + return json.load(stream) + + +def _draft(*, generated_tokens: int = 4) -> InferenceAnalysisDraft: + return InferenceAnalysisDraft.from_mappings( + model_name="gpt3-175B", + model_data=_json(ROOT / "data" / "models" / "gpt3-175B.json"), + execution_name="static-inference", + execution_data={ + "tensor_par": 8, + "pipeline_par": 1, + "replicas": 1, + "datatype": "float16", + "tensor_par_net": 0, + "pipeline_par_net": 0, + }, + request_data={ + "batch_size": 1, + "prompt_tokens": 128, + "generated_tokens": generated_tokens, + }, + hardware_name="a100_80g", + hardware_data=_json(ROOT / "data" / "systems" / "a100_80g.json"), + ) + + +def test_service_composes_prefill_and_each_decode_context_without_a_scheduler(): + outcome = BlueprintingService().analyze_inference(_draft(generated_tokens=4)) + + assert outcome.ok + assert outcome.report is not None + report = outcome.report + assert report.prefill_seconds > 0 + assert report.mean_decode_step_seconds > 0 + assert report.model_execution_seconds == pytest.approx( + report.prefill_seconds + sum(step.seconds for step in report.decode_steps) + ) + assert tuple(step.context_tokens for step in report.decode_steps) == (129, 130, 131) + assert report.memory["kv_cache"] > 0 + assert report.memory["max_kv_cache"] >= report.memory["kv_cache"] + assert report.stages[-1].stage == "decode.portable" + assert all(stage.valid for stage in report.stages) + + +def test_single_generated_token_stops_after_prefill(): + outcome = BlueprintingService().analyze_inference(_draft(generated_tokens=1)) + + assert outcome.ok + assert outcome.report is not None + assert outcome.report.decode_steps == () + assert outcome.report.mean_decode_step_seconds == 0 + assert outcome.report.model_execution_seconds == outcome.report.prefill_seconds + + +def test_request_past_model_context_returns_a_structured_diagnostic(): + draft = _draft(generated_tokens=4) + request = dict(draft.request_data.items()) + request["prompt_tokens"] = 2048 + too_long = InferenceAnalysisDraft.from_mappings( + model_name=draft.model_name, + model_data=dict(draft.model_data.items()), + execution_name=draft.execution_name, + execution_data=dict(draft.execution_data.items()), + request_data=request, + hardware_name=draft.hardware_name, + hardware_data=dict(draft.hardware_data.items()), + ) + + outcome = BlueprintingService().analyze_inference(too_long) + + assert not outcome.ok + assert outcome.diagnostics[0].code == "inference.configuration.invalid" + assert "cannot exceed" in outcome.diagnostics[0].message diff --git a/tests/compiler/test_transformer_inference.py b/tests/compiler/test_transformer_inference.py new file mode 100644 index 0000000..82c2388 --- /dev/null +++ b/tests/compiler/test_transformer_inference.py @@ -0,0 +1,257 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from blueprinting.compiler.analysis import ( + HardwareProfile, + InferenceCostProvider, + InferenceEvidenceQuery, + VidurProfileBaseline, + estimate_inference_phase, +) +from blueprinting.compiler.bindings import InferencePhase +from blueprinting.compiler.experiments import ( + VidurExperimentCase, + compare_inference_phase_to_vidur, + run_vidur_experiment, +) +from blueprinting.compiler.lowering import DistributeTransformerInferencePass, PlanTransformerInferencePass +from blueprinting.compiler.models import ( + TransformerInferenceExecutionSpec, + TransformerInferenceRequestSpec, + TransformerModelSpec, + build_transformer_inference_model_ir, + inference_compilation_session_for, +) +from blueprinting.compiler.passes import PassManager, PassPipeline + +ROOT = Path(__file__).resolve().parents[2] + + +def _model() -> TransformerModelSpec: + return TransformerModelSpec( + name="fixture-inference", + hidden_size=64, + feedforward_size=256, + sequence_length=512, + attention_heads=8, + attention_head_size=8, + block_count=8, + ) + + +def _execution() -> TransformerInferenceExecutionSpec: + return TransformerInferenceExecutionSpec( + world_size=4, + tensor_parallel=2, + pipeline_parallel=2, + replicas=1, + datatype="float16", + tensor_parallel_network=0, + pipeline_parallel_network=0, + ) + + +def _compile(phase: InferencePhase, context_tokens: int): + model = _model() + execution = _execution() + source = build_transformer_inference_model_ir(model) + result = PassManager().run( + PassPipeline.of(DistributeTransformerInferencePass(), PlanTransformerInferencePass()), + source, + session=inference_compilation_session_for( + model, + execution, + phase=phase, + batch_size=3, + context_tokens=context_tokens, + ), + ) + return source, result.ir + + +def _task(plan, primitive: str): + return next(task for task in plan.tasks if task.workload.attributes.get("primitive") == primitive) + + +@pytest.mark.parametrize("phase", tuple(InferencePhase)) +def test_inference_lowering_has_valid_auditable_phase_plans(phase: InferencePhase): + source, plan = _compile(phase, 64) + + assert source.verify().ok + assert plan.verify().ok + assert plan.attributes["inference_phase"] is phase + assert all("invocation" not in task.attributes for task in plan.tasks) + assert all(task.workload.attributes.get("phase") == phase.value for task in plan.tasks) + assert {buffer.attributes.get("semantic") for buffer in plan.buffers} >= { + "block_weights", + "block_working_upper_bound", + "kv_cache", + } + assert sum(task.workload.message_bytes > 0 for task in plan.tasks) == 2 + + +def test_prefill_attention_is_quadratic_and_decode_attention_is_linear_in_context(): + _, prefill_32 = _compile(InferencePhase.PREFILL, 32) + _, prefill_64 = _compile(InferencePhase.PREFILL, 64) + _, decode_32 = _compile(InferencePhase.DECODE, 32) + _, decode_64 = _compile(InferencePhase.DECODE, 64) + + assert ( + _task(prefill_64, "attention_core").workload.operations + == 4 * _task(prefill_32, "attention_core").workload.operations + ) + assert ( + _task(decode_64, "attention_core").workload.operations + == 2 * _task(decode_32, "attention_core").workload.operations + ) + + +def test_kv_cache_capacity_is_derived_from_shape_not_a_correction_factor(): + _, plan = _compile(InferencePhase.DECODE, 96) + kv_buffer = next(buffer for buffer in plan.buffers if buffer.attributes.get("semantic") == "kv_cache") + workspace = next( + buffer for buffer in plan.buffers if buffer.attributes.get("semantic") == "block_working_upper_bound" + ) + + assert kv_buffer.size_bytes == 2 * 3 * 96 * (64 // 2) * 2 + assert workspace.size_bytes > 0 + + +def test_request_semantics_count_prefill_as_the_first_output_token(): + one = TransformerInferenceRequestSpec(batch_size=1, prompt_tokens=64, generated_tokens=1) + four = TransformerInferenceRequestSpec(batch_size=1, prompt_tokens=64, generated_tokens=4) + + assert one.decode_iterations == 0 + assert tuple(one.decode_contexts()) == () + assert four.decode_iterations == 3 + assert tuple(four.decode_contexts()) == (65, 66, 67) + + +def test_invalid_mapping_is_rejected_before_lowering(): + model = _model() + invalid = TransformerInferenceExecutionSpec( + world_size=3, + tensor_parallel=3, + pipeline_parallel=1, + replicas=1, + datatype="float16", + tensor_parallel_network=0, + pipeline_parallel_network=0, + ) + + with pytest.raises(ValueError, match="hidden_size must be divisible"): + invalid.validate_model(model) + + +def test_vidur_adapter_uses_only_exact_shape_matches(tmp_path: Path): + attention = tmp_path / "attention.csv" + compute = tmp_path / "mlp.csv" + attention.write_text( + "n_embd,n_q_head,n_kv_head,num_tensor_parallel_workers,block_size,max_model_len," + "batch_size,prefill_chunk_size," + "kv_cache_size,is_prefill,attention_backend,time_stats.attn_prefill.median," + "time_stats.attn_decode.median\n" + "64,8,8,2,16,512,3,64,0,True,AttentionBackend.FLASH_ATTENTION,1.5,\n" + "64,8,8,2,16,512,3.5,0,95,False,AttentionBackend.FLASH_ATTENTION,,99.0\n" + "64,8,8,2,16,512,3,0,95,False,AttentionBackend.FLASH_ATTENTION,,0.25\n", + encoding="utf-8", + ) + compute.write_text( + "n_embd,n_expanded_embd,n_head,n_kv_head,num_tensor_parallel_workers,num_tokens,use_gated_mlp," + "time_stats.attn_pre_proj.median\n" + "64,256,8,8,2,192,False,0.75\n" + "64,256,8,2,2,192,False,99.0\n", + encoding="utf-8", + ) + baseline = VidurProfileBaseline.from_csv( + attention_csv=attention, + compute_csv=compute, + model_name="fixture-inference", + hardware_name="fixture-hardware", + attention_backend="AttentionBackend.FLASH_ATTENTION", + block_size=16, + source_revision="vidur-test-commit", + ) + query = InferenceEvidenceQuery( + phase=InferencePhase.DECODE, + primitive="attention_core", + source_layer="attention.core", + model_name="fixture-inference", + hardware_name="fixture-hardware", + model_sequence_length=512, + hidden_size=64, + feedforward_size=256, + attention_heads=8, + batch_size=3, + query_tokens=1, + context_tokens=96, + tensor_parallel=2, + datatype="float16", + ) + + exact = baseline.lookup(query) + assert exact is not None + assert exact.seconds == pytest.approx(0.00025) + assert exact.provider == "vidur-profile-baseline" + assert baseline.lookup(InferenceEvidenceQuery(**{**query.__dict__, "context_tokens": 95})) is None + compute_exact = baseline.lookup( + InferenceEvidenceQuery( + **{ + **query.__dict__, + "phase": InferencePhase.PREFILL, + "primitive": "attention_pre_projection", + "query_tokens": 64, + "context_tokens": 64, + } + ) + ) + assert compute_exact is not None + assert compute_exact.seconds == pytest.approx(0.00075) + + _, plan = _compile(InferencePhase.DECODE, 96) + hardware = HardwareProfile.from_mapping( + "fixture-hardware", + json.loads((ROOT / "data" / "systems" / "a100_80g.json").read_text(encoding="utf-8")), + datatype="float16", + ) + digest_before = plan.digest + estimate = estimate_inference_phase(plan, hardware) + attention_estimate = next(item for item in estimate.tasks if item.invocation.primitive == "attention_core") + comparison = compare_inference_phase_to_vidur(plan, estimate, hardware, baseline) + attention_comparison = next(item for item in comparison.components if item.primitive == "attention_core") + + assert attention_estimate.invocation.work.operations == _task(plan, "attention_core").workload.operations + assert attention_estimate.total_seconds == attention_estimate.analytical_seconds + assert attention_estimate.evidence_provider == "analytical-system-profile" + assert attention_comparison.baseline_seconds == pytest.approx(0.00025) + assert attention_comparison.compiled_seconds == attention_estimate.total_seconds + assert not isinstance(baseline, InferenceCostProvider) + assert plan.digest == digest_before + + experiment = run_vidur_experiment( + ( + VidurExperimentCase( + name="fixture/decode-96", + model=_model(), + execution=_execution(), + hardware=hardware, + phase=InferencePhase.DECODE, + batch_size=3, + context_tokens=96, + ), + ), + baseline, + ) + assert experiment.policy["baseline_role"] == "post-hoc-comparison-only" + assert experiment.policy["oracle_read_during_lowering"] is False + assert experiment.policy["oracle_read_during_costing"] is False + assert experiment.policy["validation_claim"] == "drift-detection-not-accuracy-validation" + assert experiment.cases[0].portable_digest == plan.digest + assert experiment.cases[0].system_evidence.baseline_comparable_block_seconds > 0 + assert experiment.cases[0].system_evidence.component_mean_absolute_error_percent >= abs( + experiment.cases[0].system_evidence.comparable_subtotal_relative_error_percent + ) diff --git a/tests/regression/test_baseline_quality_gate.py b/tests/regression/test_baseline_quality_gate.py new file mode 100644 index 0000000..56f4888 --- /dev/null +++ b/tests/regression/test_baseline_quality_gate.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from blueprinting.compiler.experiments import ( + BaselineRegressionGate, + RegressionCheck, + run_inference_baseline_regression, + run_training_baseline_regression, +) + +ROOT = Path(__file__).resolve().parents[2] + +pytestmark = pytest.mark.baseline_regression + + +def test_training_calculon_baseline_regression_gate(): + gate = run_training_baseline_regression(ROOT) + + gate.require() + assert gate.ok + assert len(gate.checks) >= 20 + + +def test_inference_vidur_baseline_regression_gate(): + gate = run_inference_baseline_regression(ROOT) + + gate.require() + assert gate.ok + assert len(gate.checks) >= 30 + + +def test_regression_gate_reports_every_failed_predicate(): + gate = BaselineRegressionGate( + schema="blueprinting.baseline-regression-gate.v1", + domain="test/baseline", + checks=( + RegressionCheck("first", False, "<= 1", 2), + RegressionCheck("second", False, "exactly 'stable'", "drifted"), + ), + ) + + with pytest.raises(AssertionError) as captured: + gate.require() + + assert "first: expected <= 1; actual=2" in str(captured.value) + assert "second: expected exactly 'stable'; actual='drifted'" in str(captured.value)