diff --git a/apps/portal/src/pages/Statistics.stories.tsx b/apps/portal/src/pages/Statistics.stories.tsx new file mode 100644 index 000000000..11297be81 --- /dev/null +++ b/apps/portal/src/pages/Statistics.stories.tsx @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { expect } from "storybook/test"; + +import { PerformanceTable } from "./Statistics"; +import type { AnalysisResponse } from "@/types"; + +const analysis: AnalysisResponse = { + groups: [{ + task: "Build a small API", + taskPromptId: "task-1", + workerType: "coder-acp-copilot", + total: 4, + completed: 4, + passed: 3, + rejected: 0, + passAtK: { 1: 0.75 }, + successAtT: [0.5, 0.75], + iterationStats: { mean: 2, stdDev: 0.8, min: 1, max: 3 }, + durationStats: { mean: 90_000, stdDev: 30_000, min: 60_000, max: 120_000 }, + }], + kValues: [1], + maxT: 3, + summary: { totalRuns: 4, completedRuns: 4, passedRuns: 3, overallPassRate: 0.75, avgIterationsToPass: 2 }, + availableCriteria: [], + selectedCriteria: [], + availableFeatures: [], + selectedFeatures: [], +}; + +const meta = { + component: PerformanceTable, + args: { data: analysis }, +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +export const WithVariation: Story = { + play: async ({ canvas }) => { + await expect(canvas.getByText("Mean 2.0")).toBeVisible(); + await expect(canvas.getByText("Std dev 0.8")).toBeVisible(); + await expect(canvas.getByText("Std dev 30s")).toBeVisible(); + }, +}; diff --git a/apps/portal/src/pages/Statistics.tsx b/apps/portal/src/pages/Statistics.tsx index 2226adea8..0ffdded77 100644 --- a/apps/portal/src/pages/Statistics.tsx +++ b/apps/portal/src/pages/Statistics.tsx @@ -428,7 +428,7 @@ function InsightsRow({ insights }: { insights: DerivedInsights }) { // ─── Performance breakdown (merged iteration + duration) ──────────────────── -function PerformanceTable({ data }: { data: AnalysisResponse }) { +export function PerformanceTable({ data }: { data: AnalysisResponse }) { const { groups } = data; if (groups.length === 0) { @@ -472,8 +472,9 @@ function PerformanceTable({ data }: { data: AnalysisResponse }) { Performance by task & worker - Pass rate, iteration distribution, and run duration — sorted by pass - rate. + Pass rate and run variation, sorted by pass rate. Mean and standard + deviation use passed runs only; standard deviation shows how much + individual runs vary from the mean. @@ -484,8 +485,8 @@ function PerformanceTable({ data }: { data: AnalysisResponse }) { Worker Runs Pass rate - Avg iter - Avg duration + Iterations to pass + Run duration @@ -551,22 +552,20 @@ function PerformanceTable({ data }: { data: AnalysisResponse }) { {group.iterationStats ? ( - - {formatNumber(group.iterationStats.mean)} - +
+ Mean {formatNumber(group.iterationStats.mean)} + Std dev {formatNumber(group.iterationStats.stdDev)} +
) : ( — )}
{group.durationStats ? ( - - {formatDuration(group.durationStats.mean)} - +
+ Mean {formatDuration(group.durationStats.mean)} + Std dev {formatDuration(group.durationStats.stdDev)} +
) : ( — )} diff --git a/docs/architecture/app-design.md b/docs/architecture/app-design.md index f1a243262..9366dbef5 100644 --- a/docs/architecture/app-design.md +++ b/docs/architecture/app-design.md @@ -702,6 +702,11 @@ The Statistics page (`apps/portal/src/pages/Statistics.tsx`) is backed by `GET / | `criteria` | Success criteria (comma-separated `criterionId`s) | A run counts as a pass only if **all** selected criteria pass; runs lacking a selected criterion are excluded | | `features` | Task prompt features (comma-separated `featureId`s) | Keep a run iff its task prompt was **detected** to have **every** selected feature (AND) | +The performance table labels each passed-run distribution with its mean and +standard deviation. Iteration statistics describe iterations to pass; duration +statistics sum the measured turn durations for each passed run. Missing values +appear when no passed run has the required measurement. + The response exposes the option lists and current selection for each filter so the Portal can render filter bars: `availableCriteria` / `selectedCriteria` and `availableFeatures` / `selectedFeatures`. Both `available*` lists are computed over the full valid-run set **before** filtering, so the bars stay populated even when a filter combination matches zero runs (the Portal then shows a "no runs match" empty state instead of the "no data yet" state). **Detected-based feature semantics (intentional divergence).** A task prompt's `features[]` stores a `{featureId, detected}` row for *every* evaluated feature, so presence is near-universal and meaningless as a filter. `availableFeatures` is therefore the sorted union of featureIds with `detected === true` across valid runs, and the feature filter matches on `detected === true`. This is deliberately stricter than the MDP route (`GET /api/v1/criteria/mdp`), whose feature filter is presence-based. Runs join to task prompts by effective id (`taskPromptId || computeTaskPromptId(scenario.task)`) so legacy runs without a stored `taskPromptId` still resolve their features.