synth: a data synthesizer — training data for traffic that doesn't exist yet

CLAUDE.md

@@ -87,21 +87,57 @@ touches one file and no others.
 
 ### The shadowing / agent-tuning loop
 
+There are four ways data gets in, and they all end at a `Trajectory`:
+`capture.py` (live), `traces.py` (already ran), `synth/` (doesn't exist yet), and
+`Dataset.from_*` (you have a file).
+
 - `capture.py` — `slm.capture(model)` is a drop-in OpenAI-compatible proxy that
   records an unmodified agent's traffic, reconstructing message-level
   trajectories (calls that extend a prior call's message prefix merge into one
   episode; use an `x-session-id` header to disambiguate interleaved conversations).
 - `traces.py` — the offline sibling of `capture.py`: ingests OpenTelemetry GenAI
-  spans (OTLP JSON, event/spec/OpenInference/raw-wire shapes), groups them into
-  conversations, and `to_dataset()`s them. For agents already instrumented —
-  no proxy in the path.
+  spans, groups them into conversations, and `to_dataset()`s them. Four dialects
+  are read (OTel spec `{role,parts}`, OpenInference, indexed OpenLLMetry,
+  OpenAI-wire blobs). For agents already instrumented — no proxy in the path.
 - `rl.py` — `Trajectory` / `TrajectoryGroup` / `judge_group` (LLM-judge scoring),
   fed into `method="grpo"`.
 - `apo.py` — `optimize_prompt()`: optimize the prompt instead of weights, same
   capture/judge front end, no GPU.
 - `eval.py` — `slm.evaluate()` / `shadowlm eval`: score a model on a held-out set
   (exact / contains / numeric / JSON / LLM-judge scorers).
 
+### The synthesizer (`synth/`)
+
+The fourth inlet — for traffic that doesn't exist yet (cold start, amplifying a
+handful of episodes, covering cases production never hit). Two orthogonal axes
+again, mirroring backends × methods:
+
+- **seeds** (`seeds.py`) — where scenarios come from: a plain-English `task`, a
+  `document` (chunked, facts extracted, answers judged against the passage), or
+  real `episodes` to vary. Any seed composes with any mode.
+- **modes** (`generate.py`) — what gets written per scenario: a `conversation`,
+  a `preference` pair, or `paraphrases`. Generation is always **taxonomy first,
+  instances second** — that structure, not prompt wording, is what stops mode
+  collapse.
+
+Everything converges on `Trajectory` (the same type capture and traces produce),
+then `emit.py` renders it into the shape the consumer takes. **Output shape is
+chosen from the method's spec, never its name** (`resolve_output`). `to_otlp` is
+the exact inverse of `traces._spec_message` — change one and you must change the
+other; `tests/test_synth_otlp_roundtrip.py` is what holds them together.
+
+`quality.py` validates (the "must end on an assistant turn" rule is load-bearing
+— see `torch.py:_train_dataset`), deduplicates, and gates on a judge score.
+Nothing is dropped silently: `SynthReport` reconciles exactly, and
+`report.balanced` asserts it.
+
+A run costs money, so it is meterable and stoppable. Tokens come from the
+provider's own `usage` block — never estimated, and there is deliberately no
+price table to go stale. `token_budget=` and `should_stop=` end a run early
+while **keeping** what it produced; both gate *generation* only, because
+leaving already-generated rows unscored fails them at the gate and wastes the
+whole spend. Studio runs persist under `work_root/synth/` and are cancellable.
+
 ### Signature methods (MoRE)
 
 `more.py` / `more_plus.py` implement "mixture of retrieval experts" — facts fused

Makefile

@@ -30,6 +30,18 @@ help:  ## list the available targets
 $(PY):
 	python3 -m venv $(VENV)
 
+# Targets that run the CLI need the package *installed*, not just a venv. Guard
+# them so a fresh clone gets told what to do instead of the bare
+# "make: .venv/bin/shadowlm: No such file or directory".
+$(SHADOWLM):
+	@echo "shadowlm isn't installed in $(VENV) yet. Install it:"
+	@echo "    make install         # Apple Silicon (adds the mlx backend)"
+	@echo "    make install-torch   # CUDA or CPU"
+	@echo ""
+	@echo "Or run it from an environment you already have:"
+	@echo "    python3 -m shadowlm.serve --port $(PORT)"
+	@exit 1
+
 .PHONY: install
 install: $(PY)  ## editable install with the CLI + a training backend (mlx)
 	$(PIP) install -q -e '.[mlx,cli]'
@@ -44,21 +56,21 @@ frontend:  ## install + build the React studio into shadowlm/_static
 
 # ---- run --------------------------------------------------------------------
 .PHONY: serve
-serve:  ## run the studio + API on one port (make serve PORT=8329)
+serve: | $(SHADOWLM)  ## run the studio + API on one port (make serve PORT=8329)
 	$(SHADOWLM) serve --port $(PORT)
 
 .PHONY: dev
-dev:  ## serve with Vite hot-reload UI alongside the backend
+dev: | $(SHADOWLM)  ## serve with Vite hot-reload UI alongside the backend
 	$(SHADOWLM) serve --port $(PORT) --dev
 
 .PHONY: demo
-demo:  ## end-to-end smoke: a tiny finetune through the CLI
+demo: | $(SHADOWLM)  ## end-to-end smoke: a tiny finetune through the CLI
 	$(SHADOWLM) finetune examples/sample_dataset.jsonl \
 	  --model mlx-community/Qwen2.5-0.5B-Instruct-4bit --method lora --max-steps 8
 
 # ---- checks -----------------------------------------------------------------
 .PHONY: check
-check:  ## compile the package + typecheck the frontend
+check: | $(PY)  ## compile the package + typecheck the frontend
 	$(PY) -m compileall -q shadowlm
 	cd frontend && npx tsc -b
 
@@ -68,7 +80,7 @@ test: $(PY)  ## the CPU test suite (what CI runs; tests/gpu needs a GPU box)
 	$(PY) -m pytest tests/ --ignore=tests/gpu -q
 
 .PHONY: gpu-test
-gpu-test:  ## the CUDA verification suite (run on a GPU box)
+gpu-test: | $(PY)  ## the CUDA verification suite (run on a GPU box)
 	$(PY) tests/gpu/test_cuda.py
 
 # ---- gpu (cloud demo box) ---------------------------------------------------

README.md

@@ -65,6 +65,36 @@ run = model.finetune([group], method="grpo") # 3. train the shadowLM on them
 No reward math, no rewriting the agent into an RL framework — the model API is
 the one boundary every agent already has, so ShadowLM trains from it.
 
+## No data yet? Synthesize it
+
+Capture needs a running agent and traces need one that already ran. When you
+have neither, describe the task — a teacher model writes the training set:
+
+```python
+run = slm.synthesize(                          # or document="handbook.md",
+    task="Triage billing emails: classify urgency, draft a reply, "  # or episodes=[...]
+         "escalate refunds over $200.",
+    teacher=slm.synth.frontier("gpt-4o"),      # or any slm.load(...) model
+    n=200, method="lora")                      # the method picks the shape
+print(run.report.summary())                    # kept 200/243 · 18 invalid · 19 dup · 6 low-scoring
+model.finetune(run.dataset, method="lora")
+```
+
+The teacher expands your task into distinct scenarios before writing anything,
+so you get coverage instead of one example rewritten 200 times. Every row is
+validated, deduplicated and judged, and **every rejection is counted** — the
+report reconciles exactly.
+
+`method=` picks the output shape from that method's spec: `dpo` gives preference
+pairs, `grpo` gives scored trajectory groups, `more_plus` gives query-diverse
+paraphrase units. `format="otlp"` emits OpenTelemetry GenAI spans that
+round-trip through `traces.from_otlp` — so the output injects into any stack
+that speaks OTel, not just this one.
+
+A run calls a paid API in a loop, so it reports the tokens the provider actually
+billed (never an estimate), and takes a `token_budget=` throttle and a
+`should_stop=` cancel. Both keep whatever the run has already produced.
+
 ## What you get today
 
 The whole **capture → judge → train → own a shadowLM** loop runs on these:
@@ -73,6 +103,7 @@ The whole **capture → judge → train → own a shadowLM** loop runs on these:
 |-------|--------------|-----|
 | **Capture proxy** | drop-in OpenAI endpoint that records your agent's traffic into trajectories — agent unchanged | `slm.capture()` |
 | **Trace ingestion** | already have OpenTelemetry GenAI spans? turn an OTLP dump into a training set, no proxy | `slm.traces.to_dataset()` |
+| **Data synthesizer** | no traffic yet? describe the task, point at a document, or amplify a few real episodes — emitted in the shape your method takes, or as OTel spans | `slm.synthesize()` |
 | **13 methods** | LoRA · QLoRA · DoRA · full · CPT · DPO · GRPO · MoRE · MoRE+ · BitFit · prompt · p-tuning · adapter | `method=` |
 | **Judge → train** | score episodes with an LLM judge, train with trajectory-GRPO or DPO | `judge_group` |
 | **APO** | optimize the *prompt* instead of weights — same capture/judge front end, no GPU | `slm.optimize_prompt()` |
@@ -178,6 +209,7 @@ Run output (mlx, a 0.5B model, ~3.5s):
 ## CLI & studio
 
 ```bash
+shadowlm synth --task "triage billing email" --teacher gpt-4o -n 200 -o data.jsonl
 shadowlm finetune data.jsonl --model Qwen/Qwen2.5-0.5B-Instruct --method lora
 shadowlm finetune --config run.yaml --dry-run   # reproducible runs, preview first
 shadowlm chat out/adapter/                       # talk to what you trained
@@ -187,8 +219,8 @@ shadowlm serve                                   # studio UI + API on one port
 Headline hyperparameters are typed flags; every other `TrainConfig` field is
 reachable via `--set field=value` or a `--config` file (flags override config
 override defaults). `shadowlm serve` opens the **studio** at `http://127.0.0.1:8329`
-— Datasets (upload + HuggingFace) → Models → guided Train → live Runs (loss
-charts + training console) → Playground (compare base ↔ finetuned). It's the
+— Datasets (upload + HuggingFace + synthesize) → Models → guided Train → live
+Runs (loss charts + training console) → Playground (compare base ↔ finetuned). It's the
 built React app, shipped in the wheel; the same JSON protocol powers
 `backend="remote"`.
 
@@ -215,7 +247,7 @@ API — nothing reimplemented — to turn the blocks into a one-click migration:
 ```
 [x] SDK — datasets → finetune → inference on mlx / torch / remote
 [x] 13 methods incl. MoRE, MoRE+ (decoupled MoE), trajectory GRPO, judge rewards
-[x] Capture proxy · OTLP trace ingestion · shadow accelerator · any-hardware
+[x] Capture proxy · OTLP trace ingestion · data synthesizer · shadow accelerator
 [x] Remote backend + reference server + the studio dashboard + CLI
 [x] Eval scorers (`slm.evaluate`, `shadowlm eval`) · worker fleet (`shadowlm worker`)
 [ ] Studio orchestration — decision inbox · cost gates · shadow router · switch

examples/README.md

@@ -46,6 +46,22 @@ Notes:
   bitfit has nothing to train there — the examples note this and point you to a
   base that has biases (e.g. `Qwen/Qwen2.5-7B-Instruct`).
 
+## Getting the data in
+
+The method examples above start from a file. These four start from wherever your
+data actually is — or from nothing at all:
+
+| script | starts from | ends at |
+|--------|-------------|---------|
+| `synthesize_from_task.py` | a plain-English task description | chat rows → `lora` |
+| `synthesize_from_doc.py`  | a reference document | grounded paraphrase units → `more_plus` |
+| `shadow_from_traces.py`   | an OTLP export of production spans | chat rows → `lora` |
+| `evaluate.py`             | a trained model | a task-quality score |
+
+The two synthesis scripts call a frontier teacher, so they need
+`OPENAI_API_KEY` — or swap in `slm.synth.as_teacher(slm.load(...))` to keep the
+whole loop local.
+
 ## Shared data
 
 The `data/` folder holds tiny sample datasets so the examples are self-contained:
@@ -56,6 +72,8 @@ The `data/` folder holds tiny sample datasets so the examples are self-contained
 | `data/preference.jsonl` | preference (`prompt/chosen/rejected`) | dpo |
 | `data/domain.jsonl`     | raw text (`text`)              | cpt |
 | `data/facts.jsonl`      | instruction (`instruction/output`) | more, more_plus |
+| `data/handbook.md`      | prose                          | synthesize_from_doc |
+| `data/agent_traces.otlp.json` | OTel GenAI spans         | shadow_from_traces |
 
 `grpo` defines its prompts and reward function inline in each script.
 

examples/data/handbook.md

@@ -0,0 +1,38 @@
+# Northwind Labs — Employee Handbook (excerpt)
+
+## Expenses
+
+Expenses are submitted through the Expensify workspace within 30 days of the
+purchase. Anything at or under $75 is auto-approved. Above $75 it routes to your
+manager, and above $2,000 it also needs a director sign-off. Receipts are
+required for every line item over $25. Reimbursements land in the payroll run
+following approval, which is the 15th and the last day of each month.
+
+## Travel
+
+Book flights through the Navan portal. Economy is the default; premium economy
+is allowed on flights over six hours, and business class needs VP approval
+regardless of duration. The nightly hotel cap is $280 in New York, London and
+San Francisco, and $190 everywhere else. Rental cars are reimbursed only when
+they are cheaper than the equivalent rideshare trips.
+
+## Time off
+
+Full-time staff accrue 1.75 vacation days per month, capped at 30 accrued days.
+Unused days above the cap stop accruing rather than being forfeited. Sick leave
+is separate and untracked. Requests of five consecutive days or more should be
+filed at least three weeks ahead so the team can plan around them.
+
+## Equipment
+
+Every engineer gets a laptop refresh every three years, or sooner if the machine
+fails a diagnostic check from IT. Monitors, keyboards and chairs come out of a
+$1,200 home-office budget that resets every two years. Personal phone plans are
+not reimbursed; a company line is available on request for on-call staff.
+
+## Security
+
+Production access requires hardware two-factor authentication — TOTP apps are
+not accepted for production. Access reviews run quarterly and anything unused
+for 90 days is revoked automatically. Customer data may never be copied to a
+personal device, including for debugging.

examples/synthesize_from_doc.py

@@ -0,0 +1,47 @@
+"""turn a document into a model that knows it — grounded, with MoRE+ routing
+
+Point the synthesizer at reference material and it pulls out the facts, then
+writes several differently-worded questions for each one. That phrasing variety
+is the point: MoRE+ trains one expert per fact and routes to it with BM25 over
+the *question* side, so a fact asked about only one way is an expert nobody can
+reach. Answers are judged against the source passage, so the teacher can't
+quietly invent things the document never said.
+
+Run from the repo root:
+    OPENAI_API_KEY=sk-... python examples/synthesize_from_doc.py
+"""
+from pathlib import Path
+
+import shadowlm as slm
+
+DOC = Path(__file__).resolve().parent / "data" / "handbook.md"
+PARAPHRASES_PER_FACT = 4
+
+
+def main():
+    run = slm.synthesize(
+        document=DOC,
+        teacher=slm.synth.frontier("gpt-4o"),
+        n=80,
+        method="more_plus",              # → paraphrase units, one per fact
+        per_scenario=PARAPHRASES_PER_FACT,
+    )
+    print(run.report.summary())          # the note tells you the group size
+
+    # The rows come out grouped: PARAPHRASES_PER_FACT consecutive rows per fact,
+    # which is exactly what more_plus_group_size expects.
+    for row in run.dataset.rows[:PARAPHRASES_PER_FACT]:
+        print("  q:", row["messages"][0]["content"])
+
+    model = slm.load("Qwen/Qwen2.5-1.5B-Instruct")
+    result = model.finetune(run.dataset, method="more_plus",
+                            more_plus_group_size=PARAPHRASES_PER_FACT)
+    print("final loss:", result.loss)
+    model.save("out/handbook_experts", fmt="adapter")
+
+    # ask it something phrased nothing like the document
+    print(model.generate("remind me how the expense thing works?"))
+
+
+if __name__ == "__main__":
+    main()

examples/synthesize_from_task.py

@@ -0,0 +1,52 @@
+"""shadow a task you have no data for yet
+
+The cold start. You know what the model should do, but nobody has run the agent
+in production, so there is nothing to capture and no traces to read. Describe
+the task in plain English and a teacher model writes the training set — then
+train a small open model on it and own the task.
+
+Run from the repo root:
+    OPENAI_API_KEY=sk-... python examples/synthesize_from_task.py
+
+The teacher here is a frontier model over an OpenAI-compatible endpoint. Swap it
+for `slm.synth.as_teacher(slm.load("Qwen/Qwen2.5-7B-Instruct"))` to keep the
+whole loop on your own hardware — nothing else changes.
+"""
+import shadowlm as slm
+
+TASK = (
+    "Triage inbound customer emails for a SaaS billing product. Classify the "
+    "urgency (low / normal / urgent), draft a short reply in a calm support "
+    "voice, and escalate to a human whenever a refund over $200 is requested."
+)
+
+
+def main():
+    # 1. a teacher writes the data. It expands the task into distinct scenarios
+    #    first, then fills each one — variety comes from the structure, not from
+    #    asking nicely for it.
+    run = slm.synthesize(
+        task=TASK,
+        teacher=slm.synth.frontier("gpt-4o"),
+        n=200,
+        method="lora",     # the method picks the output shape: chat rows here
+        min_score=0.7,     # the teacher also judges; weak rows are dropped
+    )
+    print(run.report.summary())
+
+    # 2. every rejection is counted, so you can see what you actually got
+    for traj in run.rejected[:3]:
+        print(f"  rejected ({traj.metadata['reject_reason']}): "
+              f"{traj.first_user_content()[:60]}")
+
+    # 3. train on it. run.dataset is a normal Dataset — nothing about it is
+    #    special because it was synthesized.
+    run.save("out/synth_task.jsonl")
+    model = slm.load("mlx-community/Qwen2.5-0.5B-Instruct-bf16", backend="mlx")
+    result = model.finetune(run.dataset, method="lora", max_steps=60)
+    print("final loss:", result.loss, result.sparkline())
+    model.save("out/shadow_from_task", fmt="adapter")
+
+
+if __name__ == "__main__":
+    main()

frontend/src/api.ts

@@ -163,6 +163,39 @@ export const addHFDataset = (
                            eval_split: evalSplit || null }) });
 export const deleteDataset = (id: string) =>
   api<{ ok: boolean }>(`/v1/datasets/${id}`, { method: "DELETE" });
+
+// ---- synthesis: make a dataset instead of bringing one ----------------------
+export interface SynthStatus {
+  synth_id: string;
+  name: string;
+  status: "running" | "succeeded" | "failed" | "stopped";
+  kept: number;
+  requested: number;
+  tokens?: number;   // what the provider billed — zero for a local teacher
+  // live phase counters — a round's generating/judging batches tick as each
+  // job lands, so the bar moves instead of waiting for the whole round
+  phase?: "starting" | "planning" | "generating" | "judging" | "kept";
+  done?: number;
+  total?: number;
+  dataset_id?: string;
+  error?: string;
+  logs?: string[];
+}
+export interface SynthRequest {
+  name: string;
+  n: number;
+  method?: string;
+  min_score?: number;
+  task?: string;
+  document?: string;
+  dataset_id?: string;
+  teacher: { kind: "openai" | "local"; model: string; base_url?: string; api_key?: string };
+}
+export const startSynth = (body: SynthRequest) =>
+  api<{ synth_id: string }>("/v1/synth", { method: "POST", body: JSON.stringify(body) });
+export const getSynthRun = (id: string) => api<SynthStatus>(`/v1/synth/${id}`);
+export const cancelSynth = (id: string) =>
+  api<{ ok: boolean }>(`/v1/synth/${id}/cancel`, { method: "POST" });
 export const getModels = () =>
   api<{ catalog: CatalogModel[]; recent: string[]; server_backend: string }>("/v1/models");
 export const getDownloads = () =>