Add static inference planning and Vidur drift gates

.github/workflows/quality.yml

@@ -0,0 +1,62 @@
+name: Quality
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: quality-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  baseline-regression:
+    name: Training calibration and inference drift regressions
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+        with:
+          enable-cache: true
+
+      - name: Install Python
+        run: uv python install 3.12
+
+      - name: Install locked dependencies
+        run: uv sync --locked
+
+      - name: Run pinned baseline regression gates
+        run: uv run pytest -m baseline_regression tests/regression
+
+  test-suite:
+    name: Full test suite
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+        with:
+          enable-cache: true
+
+      - name: Install Python
+        run: uv python install 3.12
+
+      - name: Install locked dependencies
+        run: uv sync --locked --extra legacy-ui
+
+      - name: Run tests
+        run: uv run pytest

data/validation/baseline_regression_contract.json

@@ -0,0 +1,89 @@
+{
+  "schema": "blueprinting.baseline-regression-contract.v1",
+  "training": {
+    "baseline": "Calculon / SeqSel Table 5",
+    "evidence_revision": "eb1eb9fcc4a6e414e85b0252c23ea9ad2730aae2",
+    "case_count": 8,
+    "budgets": {
+      "workload_max_absolute_error_percent": 1e-9,
+      "calculon_mean_absolute_error_percent": 1e-9,
+      "calculon_max_absolute_error_percent": 1e-9,
+      "paper_mean_absolute_error_percent": 3.7,
+      "paper_max_absolute_error_percent": 8.9,
+      "memory_max_absolute_error_bytes": 1.0
+    },
+    "golden": {
+      "peak_mean_absolute_error_percent": 12.993831306582473,
+      "calculon_mean_absolute_error_percent": 6.17654222892984e-15,
+      "paper_mean_absolute_error_percent": 3.654361437637248,
+      "paper_max_absolute_error_percent": 8.874452316395821,
+      "portable_digests": {
+        "seqsel-tab5/megatron-22B/full": "9ce66dd1c115a1efbb0cad1cc3921c3f28ffc58f",
+        "seqsel-tab5/megatron-22B/seqsel": "873c82dde8f06f97a619855bf7fa74d5730c7d55",
+        "seqsel-tab5/gpt3-175B/full": "2c9566d969073f2983b2ca3e35d840c986063058",
+        "seqsel-tab5/gpt3-175B/seqsel": "d3f1826b4e0b24ee91fc5920c95c1dd3b3ca6782",
+        "seqsel-tab5/turing-530B/full": "1eca776c1c27807964278cf5e768193a6f06c4cc",
+        "seqsel-tab5/turing-530B/seqsel": "8095358ace4a3a62466c13b0895e03545c6f5621",
+        "seqsel-tab5/megatron-1T/full": "5ffc23392d5a374f442ef2f448366e3d9f6118ce",
+        "seqsel-tab5/megatron-1T/seqsel": "377375178daedbd9655bdccb618eab11ba8b68f3"
+      }
+    }
+  },
+  "inference": {
+    "baseline": "Vidur Phi-2 / A100 / TP1 raw component-profile alignment",
+    "validation_claim": "drift-detection-not-accuracy-validation",
+    "fixture": "data/validation/vidur/phi2_a100_tp1",
+    "fixture_manifest_sha256": "24511af4979e32fdc0d1c610960f93e92ac816a2a5b30f30cf6015d2492b2c33",
+    "source_repository": "https://github.com/microsoft/vidur",
+    "source_revision": "8383d2935bc62723a212090baa9f98ada206fc14",
+    "license_file": "LICENSE.vidur",
+    "license_sha256": "7df20dcdf9197e9945c14858d41c60f11b52b93e5b69e2b63416b874d598d322",
+    "baseline_revision": "6a583f1c90969bd82ef82b8cf6dd26bcef914909",
+    "model_digest": "d67e201b24cc428ac0d3330d7fd597cfe90dcdf6",
+    "case_count": 3,
+    "budgets": {
+      "minimum_semantic_component_coverage": 0.75,
+      "system_evidence_comparable_subtotal_mean_absolute_error_percent": 24.0,
+      "system_evidence_comparable_subtotal_max_absolute_error_percent": 28.0,
+      "minimum_improvement_over_peak_percentage_points": 10.0
+    },
+    "golden": {
+      "peak_comparable_subtotal_mean_absolute_error_percent": 39.60396144129218,
+      "system_evidence_comparable_subtotal_mean_absolute_error_percent": 23.57567170288191,
+      "system_evidence_component_mean_absolute_error_percent": 54.07702487970536,
+      "system_evidence_component_max_absolute_error_percent": 99.50036630036631,
+      "cases": {
+        "phi2-a100-tp1/decode/b1-c33": {
+          "distributed_digest": "1d6a4598cff8907554b3c246d3993143d17b3d5a",
+          "portable_digest": "5bbe78b2017f21215b6e66904ba95fa1ccda5e49",
+          "component_coverage": 0.75,
+          "baseline_comparable_block_seconds": 0.000147,
+          "system_evidence_comparable_block_seconds": 0.00010824697435897437,
+          "system_evidence_comparable_subtotal_error_percent": -26.36260247688818,
+          "system_evidence_component_mean_absolute_error_percent": 60.87949445624435,
+          "system_evidence_component_max_absolute_error_percent": 99.50036630036631
+        },
+        "phi2-a100-tp1/decode/b1-c129": {
+          "distributed_digest": "4ea3575ed2cd51d69d789e6addc7197a45d32efb",
+          "portable_digest": "74335a2d5a3ed2f9da31b334a93519c559a0c6a3",
+          "component_coverage": 0.75,
+          "baseline_comparable_block_seconds": 0.00015000000000000001,
+          "system_evidence_comparable_block_seconds": 0.00010879564102564103,
+          "system_evidence_comparable_subtotal_error_percent": -27.469572649572648,
+          "system_evidence_component_mean_absolute_error_percent": 60.549761945956284,
+          "system_evidence_component_max_absolute_error_percent": 99.50036630036631
+        },
+        "phi2-a100-tp1/prefill/b1-c128": {
+          "distributed_digest": "a91b167c1f4644bc7f16b38184f61a16d42d004c",
+          "portable_digest": "7fe68d256445d118ae842cef0d23e123741a7b62",
+          "component_coverage": 0.75,
+          "baseline_comparable_block_seconds": 0.0002015,
+          "system_evidence_comparable_block_seconds": 0.00023554310256410256,
+          "system_evidence_comparable_subtotal_error_percent": 16.894839982184898,
+          "system_evidence_component_mean_absolute_error_percent": 40.80181823691545,
+          "system_evidence_component_max_absolute_error_percent": 83.93685897435897
+        }
+      }
+    }
+  }
+}

data/validation/vidur/phi2_a100_tp1/LICENSE.vidur

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) Microsoft Corporation.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/validation/vidur/phi2_a100_tp1/attention.csv

@@ -0,0 +1,5 @@
+n_embd,n_q_head,n_kv_head,block_size,num_tensor_parallel_workers,max_model_len,batch_size,prefill_chunk_size,kv_cache_size,is_prefill,attention_backend,time_stats.attn_prefill.median,time_stats.attn_decode.median
+2560,32,32,16,1,4096,1,0,32,False,AttentionBackend.FLASH_ATTENTION,,0.009000000000000001
+2560,32,32,16,1,4096,1,0,128,False,AttentionBackend.FLASH_ATTENTION,,0.012
+2560,32,32,16,1,4096,1,128,0,True,AttentionBackend.FLASH_ATTENTION,0.014,
+2560,32,32,16,1,4096,1,128,0,True,AttentionBackend.FLASH_ATTENTION,0.014,

data/validation/vidur/phi2_a100_tp1/manifest.json

@@ -0,0 +1,59 @@
+{
+  "schema": "blueprinting.vidur-validation-slice.v1",
+  "source": {
+    "repository": "https://github.com/microsoft/vidur",
+    "revision": "8383d2935bc62723a212090baa9f98ada206fc14",
+    "license": "MIT",
+    "license_file": "LICENSE.vidur"
+  },
+  "selection": {
+    "model": "microsoft/phi-2",
+    "hardware": "a100",
+    "datatype": "float16",
+    "tensor_parallel": 1,
+    "attention_backend": "AttentionBackend.FLASH_ATTENTION",
+    "block_size": 16,
+    "cases": [
+      {"phase": "decode", "batch_size": 1, "context_tokens": 33},
+      {"phase": "decode", "batch_size": 1, "context_tokens": 129},
+      {"phase": "prefill", "batch_size": 1, "context_tokens": 128}
+    ],
+    "transformation": "Exact row filtering and column projection only; numeric values are unchanged. Duplicate selected upstream rows are retained."
+  },
+  "blueprinting": {
+    "model": {
+      "name": "microsoft/phi-2",
+      "hidden_size": 2560,
+      "feedforward_size": 10240,
+      "sequence_length": 4096,
+      "attention_heads": 32,
+      "attention_head_size": 80,
+      "block_count": 32
+    },
+    "execution": {
+      "world_size": 1,
+      "tensor_parallel": 1,
+      "pipeline_parallel": 1,
+      "replicas": 1,
+      "datatype": "float16",
+      "tensor_parallel_network": 0,
+      "pipeline_parallel_network": 0
+    },
+    "hardware": {
+      "name": "a100_80g",
+      "profile": "data/systems/a100_80g.json"
+    }
+  },
+  "files": {
+    "attention.csv": {
+      "upstream_path": "data/profiling/compute/a100/microsoft/phi-2/attention.csv",
+      "upstream_git_blob": "56c1d8c06ce1f08336fe4e82f445ec1249f5ca71",
+      "sha256": "3c1b7a28425484f9008307a5e63e9a92e52156d3d6734f2ce8f4a30f83fefb3f"
+    },
+    "mlp.csv": {
+      "upstream_path": "data/profiling/compute/a100/microsoft/phi-2/mlp.csv",
+      "upstream_git_blob": "5db2fe4390c7fe2a46e545ab8703f3fc37617a7e",
+      "sha256": "9271dd3a390d1d1e27489f8ea207ba41ee28e42e4da9df14c7924066f4373256"
+    }
+  }
+}

data/validation/vidur/phi2_a100_tp1/mlp.csv

@@ -0,0 +1,3 @@
+n_embd,n_expanded_embd,n_head,n_kv_head,vocab_size,use_gated_mlp,num_tokens,num_tensor_parallel_workers,time_stats.input_layernorm.median,time_stats.attn_pre_proj.median,time_stats.attn_rope.median,time_stats.attn_post_proj.median,time_stats.mlp_up_proj.median,time_stats.mlp_act.median,time_stats.mlp_down_proj.median,time_stats.add.median
+2560,10240,32,32,51200,False,128,1,0.008,0.043000000000000003,0.006,0.017,0.0505,0.005,0.054,0.004
+2560,10240,32,32,51200,False,1,1,0.007,0.03,0.004,0.014,0.038,0.002,0.039,0.004

docs/design/performance/database.en.md

@@ -3,7 +3,7 @@
 The performance database is a revisioned evidence store behind a normalized query protocol. It answers a precise question—how an architecture component or legal implementation is expected to behave in a declared context—without hiding architecture choices or calibration knobs inside a lookup table.
 
 !!! note "Design status"
-    The repository currently loads `HardwareProfile` directly. The request/result/store design on this page is the accepted migration target.
+    The general estimator still loads `HardwareProfile` directly. Static inference now distinguishes an admissible cost-provider contract from a read-only baseline contract; the Vidur CSV adapter is baseline-only. The request/result/store design on this page remains the accepted generalization target.
 
 ## Request contract
 

docs/design/performance/database.zh.md

@@ -3,7 +3,7 @@
 性能数据库是 normalized query protocol 背后的版本化 evidence store。它回答一个精确问题——某个 architecture component 或合法 implementation 在明确 context 中预计如何表现——但不会把 architecture choice 或 calibration knob 隐藏在 lookup table 中。
 
 !!! note "设计状态"
-    当前仓库仍直接加载 `HardwareProfile`。本页定义的 request/result/store 是已接受的迁移目标。
+    通用 estimator 仍直接加载 `HardwareProfile`。Static inference 已区分可参与估算的 cost-provider contract 与只读 baseline contract；Vidur CSV adapter 仅属于 baseline。本页的 request/result/store 仍是已接受的通用化目标。
 
 ## Request Contract