diff --git a/CLAUDE.md b/CLAUDE.md index f908f0d..df24c3c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -87,14 +87,18 @@ touches one file and no others. ### The shadowing / agent-tuning loop +There are four ways data gets in, and they all end at a `Trajectory`: +`capture.py` (live), `traces.py` (already ran), `synth/` (doesn't exist yet), and +`Dataset.from_*` (you have a file). + - `capture.py` — `slm.capture(model)` is a drop-in OpenAI-compatible proxy that records an unmodified agent's traffic, reconstructing message-level trajectories (calls that extend a prior call's message prefix merge into one episode; use an `x-session-id` header to disambiguate interleaved conversations). - `traces.py` — the offline sibling of `capture.py`: ingests OpenTelemetry GenAI - spans (OTLP JSON, event/spec/OpenInference/raw-wire shapes), groups them into - conversations, and `to_dataset()`s them. For agents already instrumented — - no proxy in the path. + spans, groups them into conversations, and `to_dataset()`s them. Four dialects + are read (OTel spec `{role,parts}`, OpenInference, indexed OpenLLMetry, + OpenAI-wire blobs). For agents already instrumented — no proxy in the path. - `rl.py` — `Trajectory` / `TrajectoryGroup` / `judge_group` (LLM-judge scoring), fed into `method="grpo"`. - `apo.py` — `optimize_prompt()`: optimize the prompt instead of weights, same @@ -102,6 +106,38 @@ touches one file and no others. - `eval.py` — `slm.evaluate()` / `shadowlm eval`: score a model on a held-out set (exact / contains / numeric / JSON / LLM-judge scorers). +### The synthesizer (`synth/`) + +The fourth inlet — for traffic that doesn't exist yet (cold start, amplifying a +handful of episodes, covering cases production never hit). Two orthogonal axes +again, mirroring backends × methods: + +- **seeds** (`seeds.py`) — where scenarios come from: a plain-English `task`, a + `document` (chunked, facts extracted, answers judged against the passage), or + real `episodes` to vary. Any seed composes with any mode. +- **modes** (`generate.py`) — what gets written per scenario: a `conversation`, + a `preference` pair, or `paraphrases`. Generation is always **taxonomy first, + instances second** — that structure, not prompt wording, is what stops mode + collapse. + +Everything converges on `Trajectory` (the same type capture and traces produce), +then `emit.py` renders it into the shape the consumer takes. **Output shape is +chosen from the method's spec, never its name** (`resolve_output`). `to_otlp` is +the exact inverse of `traces._spec_message` — change one and you must change the +other; `tests/test_synth_otlp_roundtrip.py` is what holds them together. + +`quality.py` validates (the "must end on an assistant turn" rule is load-bearing +— see `torch.py:_train_dataset`), deduplicates, and gates on a judge score. +Nothing is dropped silently: `SynthReport` reconciles exactly, and +`report.balanced` asserts it. + +A run costs money, so it is meterable and stoppable. Tokens come from the +provider's own `usage` block — never estimated, and there is deliberately no +price table to go stale. `token_budget=` and `should_stop=` end a run early +while **keeping** what it produced; both gate *generation* only, because +leaving already-generated rows unscored fails them at the gate and wastes the +whole spend. Studio runs persist under `work_root/synth/` and are cancellable. + ### Signature methods (MoRE) `more.py` / `more_plus.py` implement "mixture of retrieval experts" — facts fused diff --git a/Makefile b/Makefile index 60b0a01..a32762f 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,18 @@ help: ## list the available targets $(PY): python3 -m venv $(VENV) +# Targets that run the CLI need the package *installed*, not just a venv. Guard +# them so a fresh clone gets told what to do instead of the bare +# "make: .venv/bin/shadowlm: No such file or directory". +$(SHADOWLM): + @echo "shadowlm isn't installed in $(VENV) yet. Install it:" + @echo " make install # Apple Silicon (adds the mlx backend)" + @echo " make install-torch # CUDA or CPU" + @echo "" + @echo "Or run it from an environment you already have:" + @echo " python3 -m shadowlm.serve --port $(PORT)" + @exit 1 + .PHONY: install install: $(PY) ## editable install with the CLI + a training backend (mlx) $(PIP) install -q -e '.[mlx,cli]' @@ -44,21 +56,21 @@ frontend: ## install + build the React studio into shadowlm/_static # ---- run -------------------------------------------------------------------- .PHONY: serve -serve: ## run the studio + API on one port (make serve PORT=8329) +serve: | $(SHADOWLM) ## run the studio + API on one port (make serve PORT=8329) $(SHADOWLM) serve --port $(PORT) .PHONY: dev -dev: ## serve with Vite hot-reload UI alongside the backend +dev: | $(SHADOWLM) ## serve with Vite hot-reload UI alongside the backend $(SHADOWLM) serve --port $(PORT) --dev .PHONY: demo -demo: ## end-to-end smoke: a tiny finetune through the CLI +demo: | $(SHADOWLM) ## end-to-end smoke: a tiny finetune through the CLI $(SHADOWLM) finetune examples/sample_dataset.jsonl \ --model mlx-community/Qwen2.5-0.5B-Instruct-4bit --method lora --max-steps 8 # ---- checks ----------------------------------------------------------------- .PHONY: check -check: ## compile the package + typecheck the frontend +check: | $(PY) ## compile the package + typecheck the frontend $(PY) -m compileall -q shadowlm cd frontend && npx tsc -b @@ -68,7 +80,7 @@ test: $(PY) ## the CPU test suite (what CI runs; tests/gpu needs a GPU box) $(PY) -m pytest tests/ --ignore=tests/gpu -q .PHONY: gpu-test -gpu-test: ## the CUDA verification suite (run on a GPU box) +gpu-test: | $(PY) ## the CUDA verification suite (run on a GPU box) $(PY) tests/gpu/test_cuda.py # ---- gpu (cloud demo box) --------------------------------------------------- diff --git a/README.md b/README.md index 3d558ee..8f8b53b 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,36 @@ run = model.finetune([group], method="grpo") # 3. train the shadowLM on them No reward math, no rewriting the agent into an RL framework — the model API is the one boundary every agent already has, so ShadowLM trains from it. +## No data yet? Synthesize it + +Capture needs a running agent and traces need one that already ran. When you +have neither, describe the task — a teacher model writes the training set: + +```python +run = slm.synthesize( # or document="handbook.md", + task="Triage billing emails: classify urgency, draft a reply, " # or episodes=[...] + "escalate refunds over $200.", + teacher=slm.synth.frontier("gpt-4o"), # or any slm.load(...) model + n=200, method="lora") # the method picks the shape +print(run.report.summary()) # kept 200/243 · 18 invalid · 19 dup · 6 low-scoring +model.finetune(run.dataset, method="lora") +``` + +The teacher expands your task into distinct scenarios before writing anything, +so you get coverage instead of one example rewritten 200 times. Every row is +validated, deduplicated and judged, and **every rejection is counted** — the +report reconciles exactly. + +`method=` picks the output shape from that method's spec: `dpo` gives preference +pairs, `grpo` gives scored trajectory groups, `more_plus` gives query-diverse +paraphrase units. `format="otlp"` emits OpenTelemetry GenAI spans that +round-trip through `traces.from_otlp` — so the output injects into any stack +that speaks OTel, not just this one. + +A run calls a paid API in a loop, so it reports the tokens the provider actually +billed (never an estimate), and takes a `token_budget=` throttle and a +`should_stop=` cancel. Both keep whatever the run has already produced. + ## What you get today The whole **capture → judge → train → own a shadowLM** loop runs on these: @@ -73,6 +103,7 @@ The whole **capture → judge → train → own a shadowLM** loop runs on these: |-------|--------------|-----| | **Capture proxy** | drop-in OpenAI endpoint that records your agent's traffic into trajectories — agent unchanged | `slm.capture()` | | **Trace ingestion** | already have OpenTelemetry GenAI spans? turn an OTLP dump into a training set, no proxy | `slm.traces.to_dataset()` | +| **Data synthesizer** | no traffic yet? describe the task, point at a document, or amplify a few real episodes — emitted in the shape your method takes, or as OTel spans | `slm.synthesize()` | | **13 methods** | LoRA · QLoRA · DoRA · full · CPT · DPO · GRPO · MoRE · MoRE+ · BitFit · prompt · p-tuning · adapter | `method=` | | **Judge → train** | score episodes with an LLM judge, train with trajectory-GRPO or DPO | `judge_group` | | **APO** | optimize the *prompt* instead of weights — same capture/judge front end, no GPU | `slm.optimize_prompt()` | @@ -178,6 +209,7 @@ Run output (mlx, a 0.5B model, ~3.5s): ## CLI & studio ```bash +shadowlm synth --task "triage billing email" --teacher gpt-4o -n 200 -o data.jsonl shadowlm finetune data.jsonl --model Qwen/Qwen2.5-0.5B-Instruct --method lora shadowlm finetune --config run.yaml --dry-run # reproducible runs, preview first shadowlm chat out/adapter/ # talk to what you trained @@ -187,8 +219,8 @@ shadowlm serve # studio UI + API on one port Headline hyperparameters are typed flags; every other `TrainConfig` field is reachable via `--set field=value` or a `--config` file (flags override config override defaults). `shadowlm serve` opens the **studio** at `http://127.0.0.1:8329` -— Datasets (upload + HuggingFace) → Models → guided Train → live Runs (loss -charts + training console) → Playground (compare base ↔ finetuned). It's the +— Datasets (upload + HuggingFace + synthesize) → Models → guided Train → live +Runs (loss charts + training console) → Playground (compare base ↔ finetuned). It's the built React app, shipped in the wheel; the same JSON protocol powers `backend="remote"`. @@ -215,7 +247,7 @@ API — nothing reimplemented — to turn the blocks into a one-click migration: ``` [x] SDK — datasets → finetune → inference on mlx / torch / remote [x] 13 methods incl. MoRE, MoRE+ (decoupled MoE), trajectory GRPO, judge rewards -[x] Capture proxy · OTLP trace ingestion · shadow accelerator · any-hardware +[x] Capture proxy · OTLP trace ingestion · data synthesizer · shadow accelerator [x] Remote backend + reference server + the studio dashboard + CLI [x] Eval scorers (`slm.evaluate`, `shadowlm eval`) · worker fleet (`shadowlm worker`) [ ] Studio orchestration — decision inbox · cost gates · shadow router · switch diff --git a/examples/README.md b/examples/README.md index a665ff9..62d9185 100644 --- a/examples/README.md +++ b/examples/README.md @@ -46,6 +46,22 @@ Notes: bitfit has nothing to train there — the examples note this and point you to a base that has biases (e.g. `Qwen/Qwen2.5-7B-Instruct`). +## Getting the data in + +The method examples above start from a file. These four start from wherever your +data actually is — or from nothing at all: + +| script | starts from | ends at | +|--------|-------------|---------| +| `synthesize_from_task.py` | a plain-English task description | chat rows → `lora` | +| `synthesize_from_doc.py` | a reference document | grounded paraphrase units → `more_plus` | +| `shadow_from_traces.py` | an OTLP export of production spans | chat rows → `lora` | +| `evaluate.py` | a trained model | a task-quality score | + +The two synthesis scripts call a frontier teacher, so they need +`OPENAI_API_KEY` — or swap in `slm.synth.as_teacher(slm.load(...))` to keep the +whole loop local. + ## Shared data The `data/` folder holds tiny sample datasets so the examples are self-contained: @@ -56,6 +72,8 @@ The `data/` folder holds tiny sample datasets so the examples are self-contained | `data/preference.jsonl` | preference (`prompt/chosen/rejected`) | dpo | | `data/domain.jsonl` | raw text (`text`) | cpt | | `data/facts.jsonl` | instruction (`instruction/output`) | more, more_plus | +| `data/handbook.md` | prose | synthesize_from_doc | +| `data/agent_traces.otlp.json` | OTel GenAI spans | shadow_from_traces | `grpo` defines its prompts and reward function inline in each script. diff --git a/examples/data/handbook.md b/examples/data/handbook.md new file mode 100644 index 0000000..1a9f4be --- /dev/null +++ b/examples/data/handbook.md @@ -0,0 +1,38 @@ +# Northwind Labs — Employee Handbook (excerpt) + +## Expenses + +Expenses are submitted through the Expensify workspace within 30 days of the +purchase. Anything at or under $75 is auto-approved. Above $75 it routes to your +manager, and above $2,000 it also needs a director sign-off. Receipts are +required for every line item over $25. Reimbursements land in the payroll run +following approval, which is the 15th and the last day of each month. + +## Travel + +Book flights through the Navan portal. Economy is the default; premium economy +is allowed on flights over six hours, and business class needs VP approval +regardless of duration. The nightly hotel cap is $280 in New York, London and +San Francisco, and $190 everywhere else. Rental cars are reimbursed only when +they are cheaper than the equivalent rideshare trips. + +## Time off + +Full-time staff accrue 1.75 vacation days per month, capped at 30 accrued days. +Unused days above the cap stop accruing rather than being forfeited. Sick leave +is separate and untracked. Requests of five consecutive days or more should be +filed at least three weeks ahead so the team can plan around them. + +## Equipment + +Every engineer gets a laptop refresh every three years, or sooner if the machine +fails a diagnostic check from IT. Monitors, keyboards and chairs come out of a +$1,200 home-office budget that resets every two years. Personal phone plans are +not reimbursed; a company line is available on request for on-call staff. + +## Security + +Production access requires hardware two-factor authentication — TOTP apps are +not accepted for production. Access reviews run quarterly and anything unused +for 90 days is revoked automatically. Customer data may never be copied to a +personal device, including for debugging. diff --git a/examples/synthesize_from_doc.py b/examples/synthesize_from_doc.py new file mode 100644 index 0000000..70564bb --- /dev/null +++ b/examples/synthesize_from_doc.py @@ -0,0 +1,47 @@ +"""turn a document into a model that knows it — grounded, with MoRE+ routing + +Point the synthesizer at reference material and it pulls out the facts, then +writes several differently-worded questions for each one. That phrasing variety +is the point: MoRE+ trains one expert per fact and routes to it with BM25 over +the *question* side, so a fact asked about only one way is an expert nobody can +reach. Answers are judged against the source passage, so the teacher can't +quietly invent things the document never said. + +Run from the repo root: + OPENAI_API_KEY=sk-... python examples/synthesize_from_doc.py +""" +from pathlib import Path + +import shadowlm as slm + +DOC = Path(__file__).resolve().parent / "data" / "handbook.md" +PARAPHRASES_PER_FACT = 4 + + +def main(): + run = slm.synthesize( + document=DOC, + teacher=slm.synth.frontier("gpt-4o"), + n=80, + method="more_plus", # → paraphrase units, one per fact + per_scenario=PARAPHRASES_PER_FACT, + ) + print(run.report.summary()) # the note tells you the group size + + # The rows come out grouped: PARAPHRASES_PER_FACT consecutive rows per fact, + # which is exactly what more_plus_group_size expects. + for row in run.dataset.rows[:PARAPHRASES_PER_FACT]: + print(" q:", row["messages"][0]["content"]) + + model = slm.load("Qwen/Qwen2.5-1.5B-Instruct") + result = model.finetune(run.dataset, method="more_plus", + more_plus_group_size=PARAPHRASES_PER_FACT) + print("final loss:", result.loss) + model.save("out/handbook_experts", fmt="adapter") + + # ask it something phrased nothing like the document + print(model.generate("remind me how the expense thing works?")) + + +if __name__ == "__main__": + main() diff --git a/examples/synthesize_from_task.py b/examples/synthesize_from_task.py new file mode 100644 index 0000000..adc322e --- /dev/null +++ b/examples/synthesize_from_task.py @@ -0,0 +1,52 @@ +"""shadow a task you have no data for yet + +The cold start. You know what the model should do, but nobody has run the agent +in production, so there is nothing to capture and no traces to read. Describe +the task in plain English and a teacher model writes the training set — then +train a small open model on it and own the task. + +Run from the repo root: + OPENAI_API_KEY=sk-... python examples/synthesize_from_task.py + +The teacher here is a frontier model over an OpenAI-compatible endpoint. Swap it +for `slm.synth.as_teacher(slm.load("Qwen/Qwen2.5-7B-Instruct"))` to keep the +whole loop on your own hardware — nothing else changes. +""" +import shadowlm as slm + +TASK = ( + "Triage inbound customer emails for a SaaS billing product. Classify the " + "urgency (low / normal / urgent), draft a short reply in a calm support " + "voice, and escalate to a human whenever a refund over $200 is requested." +) + + +def main(): + # 1. a teacher writes the data. It expands the task into distinct scenarios + # first, then fills each one — variety comes from the structure, not from + # asking nicely for it. + run = slm.synthesize( + task=TASK, + teacher=slm.synth.frontier("gpt-4o"), + n=200, + method="lora", # the method picks the output shape: chat rows here + min_score=0.7, # the teacher also judges; weak rows are dropped + ) + print(run.report.summary()) + + # 2. every rejection is counted, so you can see what you actually got + for traj in run.rejected[:3]: + print(f" rejected ({traj.metadata['reject_reason']}): " + f"{traj.first_user_content()[:60]}") + + # 3. train on it. run.dataset is a normal Dataset — nothing about it is + # special because it was synthesized. + run.save("out/synth_task.jsonl") + model = slm.load("mlx-community/Qwen2.5-0.5B-Instruct-bf16", backend="mlx") + result = model.finetune(run.dataset, method="lora", max_steps=60) + print("final loss:", result.loss, result.sparkline()) + model.save("out/shadow_from_task", fmt="adapter") + + +if __name__ == "__main__": + main() diff --git a/frontend/src/api.ts b/frontend/src/api.ts index ebd0659..6b94548 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -163,6 +163,39 @@ export const addHFDataset = ( eval_split: evalSplit || null }) }); export const deleteDataset = (id: string) => api<{ ok: boolean }>(`/v1/datasets/${id}`, { method: "DELETE" }); + +// ---- synthesis: make a dataset instead of bringing one ---------------------- +export interface SynthStatus { + synth_id: string; + name: string; + status: "running" | "succeeded" | "failed" | "stopped"; + kept: number; + requested: number; + tokens?: number; // what the provider billed — zero for a local teacher + // live phase counters — a round's generating/judging batches tick as each + // job lands, so the bar moves instead of waiting for the whole round + phase?: "starting" | "planning" | "generating" | "judging" | "kept"; + done?: number; + total?: number; + dataset_id?: string; + error?: string; + logs?: string[]; +} +export interface SynthRequest { + name: string; + n: number; + method?: string; + min_score?: number; + task?: string; + document?: string; + dataset_id?: string; + teacher: { kind: "openai" | "local"; model: string; base_url?: string; api_key?: string }; +} +export const startSynth = (body: SynthRequest) => + api<{ synth_id: string }>("/v1/synth", { method: "POST", body: JSON.stringify(body) }); +export const getSynthRun = (id: string) => api(`/v1/synth/${id}`); +export const cancelSynth = (id: string) => + api<{ ok: boolean }>(`/v1/synth/${id}/cancel`, { method: "POST" }); export const getModels = () => api<{ catalog: CatalogModel[]; recent: string[]; server_backend: string }>("/v1/models"); export const getDownloads = () => diff --git a/frontend/src/pages/Datasets.tsx b/frontend/src/pages/Datasets.tsx index ae78f35..773ab76 100644 --- a/frontend/src/pages/Datasets.tsx +++ b/frontend/src/pages/Datasets.tsx @@ -1,12 +1,14 @@ -// Dataset library — upload JSONL, or reference a HuggingFace dataset (with a -// streamed preview before you add it). Both become trainable by reference. +// Dataset library — upload JSONL, reference a HuggingFace dataset (with a +// streamed preview before you add it), or synthesize one from a task +// description. All three become trainable by reference. import { useEffect, useRef, useState } from "react"; -import { Database, Search, Upload } from "lucide-react"; +import { Database, Search, Sparkles, Upload } from "lucide-react"; import { - addHFDataset, createDataset, deleteDataset, getDataset, getDatasets, hfInfo, previewHF, + addHFDataset, createDataset, deleteDataset, getDataset, getDatasets, getMethods, + cancelSynth, getSynthRun, hfInfo, previewHF, startSynth, } from "../api"; -import type { DatasetMeta, HFPreview } from "../api"; -import { Modal, ModalHeader, PageHeader, btnGhost, btnPrimary } from "../ui"; +import type { DatasetMeta, HFPreview, MethodInfo, SynthStatus } from "../api"; +import { Field, Modal, ModalHeader, PageHeader, btnGhost, btnPrimary } from "../ui"; const FORMAT_COLORS: Record = { chat: "bg-primary/10 text-primary border-primary/30", @@ -33,7 +35,7 @@ export default function Datasets() { const [list, setList] = useState([]); const [search, setSearch] = useState(""); const [view, setView] = useState<"mine" | "explore">("mine"); - const [tab, setTab] = useState<"none" | "upload" | "hf">("none"); + const [tab, setTab] = useState<"none" | "upload" | "hf" | "synth">("none"); const [rowPreview, setRowPreview] = useState(null); const [previewing, setPreviewing] = useState(null); @@ -74,9 +76,13 @@ export default function Datasets() { + + ) : ( + + )} + + ); + } + + return ( +
+ setName(e.target.value)} /> + +