From 4a4e4037eba65ed60f69d51f75048a82768ecb72 Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Wed, 16 Sep 2026 17:26:47 +0100 Subject: [PATCH 001/122] Routing + in-engine agentic conversion (route -> alter -> fill) Group pipelines into connected components over control lineage and recommend a deterministic/agentic strategy per component (routing.py), recording the user's decision as an additive conversion plan (models/conversion_plan.py). Drive the in-engine agentic conversion flow (route_agentic.py) that routes, alters and fills the remaining gaps, with plan-bound gap combine/dedupe scoped to the routed pipeline's own tasks and mandatory bundle validation. Expose route and fill through the adapter CLI and MCP, and update AGENTS.md. Consolidates PRs #77 (per-connected-component routing agent), #92 and #93 (in-engine agentic conversion flow + CLI/MCP parity + cross-review fixes). Co-authored-by: Isaac --- AGENTS.md | 2 + src/flowx/adapter/__main__.py | 232 ++++++++++- src/flowx/mcp/server.py | 116 ++++++ src/flowx/models/conversion_plan.py | 135 +++++++ src/flowx/route_agentic.py | 435 +++++++++++++++++++++ src/flowx/routing.py | 539 ++++++++++++++++++++++++++ tests/unit/test_cli_route_agentic.py | 331 ++++++++++++++++ tests/unit/test_mcp_route.py | 157 ++++++++ tests/unit/test_route_agentic_flow.py | 524 +++++++++++++++++++++++++ tests/unit/test_routing_components.py | 152 ++++++++ tests/unit/test_routing_recommend.py | 206 ++++++++++ tests/unit/test_routing_record.py | 388 ++++++++++++++++++ 12 files changed, 3214 insertions(+), 3 deletions(-) create mode 100644 src/flowx/models/conversion_plan.py create mode 100644 src/flowx/route_agentic.py create mode 100644 src/flowx/routing.py create mode 100644 tests/unit/test_cli_route_agentic.py create mode 100644 tests/unit/test_mcp_route.py create mode 100644 tests/unit/test_route_agentic_flow.py create mode 100644 tests/unit/test_routing_components.py create mode 100644 tests/unit/test_routing_recommend.py create mode 100644 tests/unit/test_routing_record.py diff --git a/AGENTS.md b/AGENTS.md index e1ae8c40..7f2da251 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -86,6 +86,8 @@ intermediates under `.work/` (pruned by `package`). | `reporting/coverage.py` | Builds per-pipeline coverage rows from `metadata/` | | `reporting/results.py` | Writes per-run coverage to a UC table (run_id/run_date/run_by) via the SDK | | `reporting/dashboard.py` | Installs + publishes an AI/BI coverage dashboard over the results table | +| `routing.py` | Groups pipelines into connected components over control lineage; recommends deterministic/agentic per component (both options) and records the user's decision as `metadata/conversion_plan.json` (additive; convert untouched) | +| `models/conversion_plan.py` | Source-neutral conversion-plan artifact model (per-component decision + both options), the routing counterpart to `models/insights.py` | ## Activity Types diff --git a/src/flowx/adapter/__main__.py b/src/flowx/adapter/__main__.py index a2bb308e..64ca32b8 100644 --- a/src/flowx/adapter/__main__.py +++ b/src/flowx/adapter/__main__.py @@ -1,9 +1,9 @@ """Unified CLI entry point that the flowx skills and MCP tools drive via subprocesses. Exposes stateless subcommands -- the ``discover``/``convert``/``package`` phase runners plus -``inspect``, ``modify``, ``resolve-agentic``, ``enrich``, ``inputs``, ``materialize-lookup``, -``workspace-paths``, ``record-results``, and ``install-dashboard`` -- so each agent turn runs as an -independent process holding no session state across user prompts. +``inspect``, ``modify``, ``resolve-agentic``, ``enrich``, ``route``, ``fill-agentic``, ``inputs``, +``materialize-lookup``, ``workspace-paths``, ``record-results``, and ``install-dashboard`` -- so each +agent turn runs as an independent process holding no session state across user prompts. """ from __future__ import annotations @@ -87,6 +87,10 @@ def main(argv: list[str] | None = None) -> int: return _run_resolve_agentic(args) if args.command == "enrich": return _run_enrich(args) + if args.command == "route": + return _run_route(args) + if args.command == "fill-agentic": + return _run_fill_agentic(args) if args.command == "record-results": return _run_record_results(args) if args.command == "install-dashboard": @@ -163,6 +167,151 @@ def _run_enrich(args: argparse.Namespace) -> int: return 0 if result.get("ok") else 1 +def _run_route(args: argparse.Namespace) -> int: + """Implements the reshaped ``route``: one command that recommends, decides, records, and edits. + + Groups pipelines by connected component and computes the per-component recommendation (reusing + :mod:`flowx.routing`). Then: + + * with **no decision** on a non-TTY it emits the recommendation (components + both options + + findings + a ready-to-record default plan) and exits -- the dry run an agent reads first; + * with a **decision** -- ``--plan-path FILE`` (or ``--plan-path -`` to read the plan from stdin), + or an interactive TTY prompt -- it validates and records the fingerprint-bound + ``metadata/conversion_plan.json`` and then edits ``.work/translation_report.json`` + + ``gaps.json`` so every routed-agentic group's tasks become placeholder gaps (deterministic + groups untouched; nothing routed agentic leaves the report byte-identical). + + Triggers the convert phase in-process when the report is missing and ``--source`` / + ``--source-path`` are supplied. Returns 1 on a missing report it cannot produce, or on a plan + that fails validation (report + plan left untouched). + """ + from flowx import routing + from flowx.route_agentic import REPORT_FILENAME, WORK_DIRNAME, apply_plan_to_report + + inventory_path = args.output_dir / "metadata" / "inventory.json" + if not inventory_path.exists(): + print(f"No inventory.json under {inventory_path.parent}; run the discover phase first.", file=sys.stderr) + return 1 + try: + inventory = json.loads(inventory_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + print(f"Failed to read {inventory_path}: {error}", file=sys.stderr) + return 1 + recommendation = routing.build_recommendation(inventory) + + try: + plan = _route_decision(args, recommendation) + except (OSError, json.JSONDecodeError) as error: + print(f"Failed to read the conversion plan: {error}", file=sys.stderr) + return 1 + if plan is None: + # No decision on a non-TTY: emit the recommendation as a dry run and stop. + _emit_json(recommendation, args.out) + return 0 + + report_path = args.output_dir / WORK_DIRNAME / REPORT_FILENAME + if not report_path.exists(): + triggered = _trigger_convert(args) + if triggered != 0: + return triggered + if not report_path.exists(): + print( + f"No {REPORT_FILENAME} under {report_path.parent}; run the convert phase first " + "(or pass --source and --source-path so route can trigger it).", + file=sys.stderr, + ) + return 1 + + try: + result = routing.record_plan(args.output_dir, plan=plan) + except (FileNotFoundError, OSError, ValueError, json.JSONDecodeError) as error: + print(f"Failed to record conversion plan: {error}", file=sys.stderr) + return 1 + if not result.get("ok"): + _emit_json(result, args.out) + return 1 + try: + edit = apply_plan_to_report(args.output_dir, plan) + except (FileNotFoundError, OSError, ValueError, json.JSONDecodeError) as error: + print(f"Failed to edit translation report: {error}", file=sys.stderr) + return 1 + _emit_json({**result, "edit": edit}, args.out) + return 0 + + +def _route_decision(args: argparse.Namespace, recommendation: dict[str, Any]) -> dict[str, Any] | None: + """Resolve the routing decision as an authored plan, or ``None`` for a dry-run recommendation. + + Precedence: ``--plan-path FILE`` (a file), ``--plan-path -`` (stdin JSON), then -- when stdin is + a TTY -- an interactive per-component prompt. On a non-TTY with no ``--plan-path`` there is no + decision, so the caller emits the recommendation instead. + """ + from flowx.route_agentic import prompt_for_decisions + + if args.plan_path is not None: + if str(args.plan_path) == "-": + return json.loads(sys.stdin.read()) + return json.loads(Path(args.plan_path).read_text(encoding="utf-8")) + if sys.stdin.isatty(): + return prompt_for_decisions(recommendation) + return None + + +def _trigger_convert(args: argparse.Namespace) -> int: + """Run the convert phase in-process when ``--source`` / ``--source-path`` are supplied. + + Returns 0 when convert ran (or there was nothing to trigger because the inputs were absent), or + the convert phase's non-zero exit code on failure. + """ + source = getattr(args, "source", None) + source_path = getattr(args, "source_path", None) + if not source or not source_path: + return 0 + return _run_phase( + "convert", + ["--source", source, "--source-path", str(source_path), "--output-dir", str(args.output_dir)], + ) + + +def _run_fill_agentic(args: argparse.Namespace) -> int: + """Implements ``fill-agentic combine``: the cross-pipeline pipeline-grain fill. + + Replaces a routed-agentic group's pipelines (``--members`` as a comma-separated list) with the + agent-authored pipeline(s) read from ``--pipelines-path`` (a JSON list of pipeline IR dicts, each + typically carrying ``AgenticComponentActivity`` nodes). The membership must exactly match a + routed-agentic component in the recorded, fingerprint-bound ``metadata/conversion_plan.json``, and + the merged report is always validated structurally before it is written back -- there is no bypass. + Per-pipeline agentic fills reuse ``convert --merge-agentic`` instead and are not handled here. + """ + from flowx.route_agentic import apply_combine_fill + + if args.action != "combine": + print(f"Unknown fill-agentic action {args.action!r}; expected 'combine'.", file=sys.stderr) + return 2 + members = [member.strip() for member in args.members.split(",") if member.strip()] + if not members: + print("fill-agentic combine requires a non-empty --members list.", file=sys.stderr) + return 2 + try: + authored = json.loads(Path(args.pipelines_path).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + print(f"Failed to read {args.pipelines_path}: {error}", file=sys.stderr) + return 1 + if not isinstance(authored, list): + print("--pipelines-path must contain a JSON list of pipeline IR dicts.", file=sys.stderr) + return 2 + try: + result = apply_combine_fill(args.output_dir, members, authored) + except FileNotFoundError as error: + print(str(error), file=sys.stderr) + return 1 + except (OSError, ValueError, json.JSONDecodeError) as error: + print(f"Failed to apply combine fill: {error}", file=sys.stderr) + return 1 + _emit_json(result, args.out) + return 0 if result.get("ok") else 1 + + def _run_record_results(args: argparse.Namespace) -> int: """Implements ``record-results``: write per-pipeline coverage to a UC table. @@ -521,6 +670,83 @@ def _build_parser() -> argparse.ArgumentParser: help="Optional output file for the enrich result JSON; defaults to stdout.", ) + route = subparsers.add_parser( + "route", + help=( + "One command: recommend a per-connected-component route, take the decision (interactive on " + "a TTY, else --plan-path / stdin), record metadata/conversion_plan.json, and edit the " + "translation report so routed-agentic groups become placeholder gaps." + ), + ) + route.add_argument( + "--output-dir", + type=Path, + required=True, + help=( + "Migration output directory (reads metadata/inventory.json + .work/translation_report.json; " + "records metadata/conversion_plan.json and edits the report for routed-agentic groups)." + ), + ) + route.add_argument( + "--plan-path", + type=Path, + default=None, + help=( + "Path to the agent-authored conversion plan JSON to validate, record, and apply. Use '-' to " + "read the plan from stdin. Omit on a TTY for an interactive prompt, or on a non-TTY to emit " + "the recommendation as a dry run." + ), + ) + route.add_argument( + "--source", + default=None, + help="Migration source (adf | airflow); with --source-path, lets route trigger convert if needed.", + ) + route.add_argument( + "--source-path", + type=Path, + default=None, + help="Path to the source; with --source, lets route trigger the convert phase when the report is missing.", + ) + route.add_argument( + "--out", + type=Path, + default=None, + help="Optional output file for the recommendation / result JSON; defaults to stdout.", + ) + + fill_agentic = subparsers.add_parser( + "fill-agentic", + help=( + "Cross-pipeline COMBINE fill: replace a routed group's pipelines with agent-authored " + "pipeline(s), validated structurally before writing. Per-pipeline fills use convert --merge-agentic." + ), + ) + fill_agentic.add_argument("action", choices=("combine",), help="'combine' performs the pipeline-grain fill.") + fill_agentic.add_argument( + "--output-dir", + type=Path, + required=True, + help="Migration output directory (reads and rewrites .work/translation_report.json).", + ) + fill_agentic.add_argument( + "--members", + required=True, + help="Comma-separated pipeline names of the routed group to replace.", + ) + fill_agentic.add_argument( + "--pipelines-path", + type=Path, + required=True, + help="JSON file: a list of agent-authored pipeline IR dicts (typically AgenticComponentActivity nodes).", + ) + fill_agentic.add_argument( + "--out", + type=Path, + default=None, + help="Optional output file for the result JSON; defaults to stdout.", + ) + record = subparsers.add_parser( "record-results", help="Write per-pipeline migration coverage for this run to a Unity Catalog table.", diff --git a/src/flowx/mcp/server.py b/src/flowx/mcp/server.py index 06516401..c1bcbfab 100644 --- a/src/flowx/mcp/server.py +++ b/src/flowx/mcp/server.py @@ -53,6 +53,9 @@ def _transport_security() -> TransportSecuritySettings: flowx("discover", {"source": "adf", "adf_source_path": "...", "output_dir": "..."}) flowx("enrich", {"output_dir": "...", "insights": {...}}) # optional: merge agent-authored insights flowx("convert", {"source": "adf", "output_dir": "..."}) + flowx("route", {"output_dir": "..."}) # optional: recommend, then re-call with a plan + flowx("route", {"output_dir": "...", "plan": {...}}) # record the decision + edit the report + flowx("fill_agentic", {"output_dir": "...", "members": [...], "pipelines": [...]}) # combine fill (or merge_agentic) flowx("inspect", {"report_path": "/.work/translation_report.json"}) flowx("apply_answers", {"report_path": "...", "answers": ["id=value"], "output_dir": "..."}) flowx("package", {"output_dir": "...", "catalog": "main", "schema": "default"}) @@ -326,6 +329,100 @@ def _run(path: str) -> dict[str, Any]: return _run(str(inline_path)) +def _cmd_route(p: dict[str, Any]) -> dict[str, Any]: + """One command: recommend a per-component route, or record + apply the user's decision. + + With **no** ``plan`` / ``plan_path`` this emits the recommendation (components, both conversion + options per component, findings, and a ready-to-record default plan) -- the dry run an agent reads + first. With a decision (``plan`` inline or ``plan_path`` -- at most one) it validates and records + the fingerprint-bound ``metadata/conversion_plan.json`` AND edits ``.work/translation_report.json`` + + ``gaps.json`` so every routed-agentic group's tasks become placeholder gaps (deterministic groups + untouched). The returned ``ok`` reflects the CLI's success; ``result`` carries its JSON. + """ + output_dir = p.get("output_dir", "./flowx_output") + plan = p.get("plan") + plan_path = p.get("plan_path") + if plan is not None and plan_path is not None: + return {"ok": False, "error": "provide at most one of 'plan' (inline object) or 'plan_path'."} + + # Forward source + resolved source-path so the MCP path mirrors the CLI: route can then trigger + # convert when the report is missing. `source` is optional here (unlike discover/convert). + source_args, cleanup = _route_source_args(p) + try: + if plan is None and plan_path is None: + result = runner.run_adapter(["route", "--output-dir", output_dir, *source_args]) + return {"ok": result.ok, "result": runner.parse_stdout_json(result), "process": result.as_dict()} + + def _run(path: str) -> dict[str, Any]: + result = runner.run_adapter(["route", "--output-dir", output_dir, "--plan-path", path, *source_args]) + payload = runner.parse_stdout_json(result) + ok = bool(isinstance(payload, dict) and payload.get("ok")) + return {"ok": ok, "result": payload, "process": result.as_dict()} + + if plan_path is not None: + return _run(str(plan_path)) + with tempfile.TemporaryDirectory(prefix="flowx-plan-") as temporary: + inline_path = Path(temporary) / "conversion_plan.json" + inline_path.write_text(json.dumps(plan, indent=2), encoding="utf-8") + return _run(str(inline_path)) + finally: + cleanup() + + +def _route_source_args(p: dict[str, Any]) -> tuple[list[str], Callable[[], None]]: + """Resolve optional ``source`` / source-path into ``route`` CLI flags (empty when no source given). + + Mirrors :func:`_cmd_convert`'s source resolution so a materialized volume/workspace source is read + the same way, but ``source`` is optional for ``route`` -- absent it, no source flags are forwarded + and route simply skips the convert trigger. Returns ``(args, cleanup)``. + """ + if not p.get("source"): + return [], _noop + source_name = _source_name(p) + source, cleanup = _resolve_source(p) + args = ["--source", source_name] + if source: + args += ["--source-path", str(source)] + return args, cleanup + + +def _cmd_fill_agentic(p: dict[str, Any]) -> dict[str, Any]: + """Cross-pipeline COMBINE fill: replace a routed group's pipelines with agent-authored pipeline(s). + + ``members`` (a list of pipeline names, or a comma-separated string) names the routed group to + replace; the authored pipeline IR dicts come inline as ``pipelines`` (a JSON list) or via + ``pipelines_path`` (exactly one, staged to a temp file so the same CLI contract runs on both). The + merged report is validated structurally before it is written back; ``ok`` is False (nothing + written) when a structural invariant is violated. Per-pipeline agentic fills use ``merge_agentic``. + """ + output_dir = p.get("output_dir", "./flowx_output") + members = p.get("members") + if isinstance(members, list): + members = ",".join(str(member) for member in members) + if not members: + return {"ok": False, "error": "provide 'members' (a list of pipeline names or comma-separated string)."} + + pipelines = p.get("pipelines") + pipelines_path = p.get("pipelines_path") + if (pipelines is None) == (pipelines_path is None): + return {"ok": False, "error": "provide exactly one of 'pipelines' (inline list) or 'pipelines_path'."} + + def _run(path: str) -> dict[str, Any]: + args: list[Any] = ["fill-agentic", "combine", "--output-dir", output_dir, "--members", members] + args += ["--pipelines-path", path] + result = runner.run_adapter(args) + payload = runner.parse_stdout_json(result) + ok = bool(isinstance(payload, dict) and payload.get("ok")) + return {"ok": ok, "result": payload, "process": result.as_dict()} + + if pipelines_path is not None: + return _run(str(pipelines_path)) + with tempfile.TemporaryDirectory(prefix="flowx-fill-") as temporary: + inline_path = Path(temporary) / "authored_pipelines.json" + inline_path.write_text(json.dumps(pipelines, indent=2), encoding="utf-8") + return _run(str(inline_path)) + + def _cmd_inspect(p: dict[str, Any]) -> dict[str, Any]: args: list[Any] = ["inspect", p["report_path"]] for answer in p.get("answers") or []: @@ -529,6 +626,8 @@ def _cmd_install_dashboard(p: dict[str, Any]) -> dict[str, Any]: "merge_agentic": _cmd_merge_agentic, "resolve_agentic": _cmd_resolve_agentic, "enrich": _cmd_enrich, + "route": _cmd_route, + "fill_agentic": _cmd_fill_agentic, "inspect": _cmd_inspect, "apply_answers": _cmd_apply_answers, "materialize_lookup": _cmd_materialize_lookup, @@ -587,6 +686,23 @@ def flowx(command: str, parameters: dict[str, Any] | None = None) -> dict[str, A `insights` key (atomic, idempotent). `ok` reflects validation; `result.violations` lists any problems and the inventory is left untouched on failure. Author the insights by reading inventory.json + the source artifacts first (see the flowx-discover skill's insights guide). + - "route": output_dir(req), at most one of plan(inline object) | plan_path, plus optional + source + a source path (forwarded so route can trigger convert if the report is absent, like + the CLI) — one command that groups pipelines into connected components over control lineage + and routes each. With NO plan it emits the components with BOTH conversion options + (deterministic capability + motif/coverage evidence, and the agentic recommended patterns + with any simplification pattern surfaced), plus a ready-to-record default plan — the dry run + to read first. With a plan it validates + records metadata/conversion_plan.json AND edits the + translation report so every routed-agentic group's tasks become placeholder gaps + (deterministic groups untouched; convert's deterministic translation is never modified, and + with no agentic decision the report is byte-identical to today). + - "fill_agentic": output_dir(req), members(req: list of pipeline names or comma-separated + string), one of pipelines(inline list of pipeline IR dicts) | pipelines_path — cross-pipeline + COMBINE fill: replace a routed-agentic group's pipelines with the agent-authored pipeline(s) + (typically AgenticComponentActivity nodes). `members` must exactly match a routed-agentic + component in the recorded, fingerprint-bound conversion_plan.json, and the merged report is + always validated structurally before writing (no bypass). Per-pipeline agentic fills use + "merge_agentic" instead. - "inspect": report_path(req) — return the full translation-option schema (every option with a `show_when` condition) for the agent to walk locally. See "Collecting options" below. - "apply_answers": report_path(req), answers(req, list of "ID=VALUE"), output_dir, lookup_csv. diff --git a/src/flowx/models/conversion_plan.py b/src/flowx/models/conversion_plan.py new file mode 100644 index 00000000..bd0dc7ae --- /dev/null +++ b/src/flowx/models/conversion_plan.py @@ -0,0 +1,135 @@ +"""Conversion-plan artifact (routing, #77) -- the user-approved per-component conversion decision. + +The routing step (:mod:`flowx.routing`) groups pipelines into connected components over control +lineage, presents each component's two conversion options as first-class peers, and records the +user's per-component choice as ``metadata/conversion_plan.json``. These models are **source-neutral** +and document the shape of that artifact; the validate/record engine works on the raw dict form and +these dataclasses back the unit tests, mirroring the split in :mod:`flowx.models.insights`. + +The agent authors **only** :attr:`ComponentPlan.decision` (and an optional +:attr:`ComponentPlan.rationale`). Everything else -- ``component_id``, ``members``, ``recommended``, +and both :class:`ComponentOptions` -- is recomputed by the library on record so the recorded facts +can never drift from the inventory or be faked. The library also owns :attr:`ConversionPlan.schema_version` +and :attr:`ConversionPlan.inventory_sha256` (the fingerprint that binds the plan to the inventory). + +This is Phase-1, descriptive-only routing metadata: recording a plan does not alter ``convert``. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +# The plan schema version stamped onto the recorded artifact. Bump on any backwards-incompatible +# change to the recorded shape. +SCHEMA_VERSION = "1" + +# The two conversion routes a component can take. +DECISION_DETERMINISTIC = "deterministic" +DECISION_AGENTIC = "agentic" +DECISIONS: tuple[str, ...] = (DECISION_DETERMINISTIC, DECISION_AGENTIC) + + +@dataclass(slots=True, kw_only=True) +class DeterministicOption: + """The deterministic (1:1 engine) conversion option for a component. + + Attributes: + capable: ``True`` when every activity in the component is engine-capable -- each either has a + ``"deterministic"`` strategy or is claimed by a detected motif -- so the whole component + can convert deterministically with no gap. + activity_counts: Count of activities by strategy bucket (``deterministic`` / ``agentic`` / + ``unsupported``) across the component's pipelines. + motifs: The motif ids detected in the component (the #64 multi-activity capability signal), + sorted and de-duplicated. + uncovered: One entry per activity that keeps the component from being fully deterministic -- + a dict of ``pipeline`` / ``activity`` / ``type`` / ``strategy``. Empty when ``capable``. + """ + + capable: bool + activity_counts: dict[str, int] = field(default_factory=dict) + motifs: list[str] = field(default_factory=list) + uncovered: list[dict[str, Any]] = field(default_factory=list) + + +@dataclass(slots=True, kw_only=True) +class AgenticPattern: + """One recommended Databricks pattern for the agentic option, drawn from a pipeline's insights. + + Attributes: + pipeline: The member pipeline the pattern was recommended for. + pattern: The named, publicly-documented Databricks capability (verbatim from the insight). + fit: One line on why it fits / what it replaces. + simplification_pattern: ``True`` when the pattern uses a distinctive capability that collapses + a whole legacy pattern (e.g. a multi-pipeline -> Lakeflow Connect re-architecture). + """ + + pipeline: str + pattern: str + fit: str + simplification_pattern: bool + + +@dataclass(slots=True, kw_only=True) +class AgenticOption: + """The agentic conversion option for a component. + + Attributes: + recommended_patterns: The recommended patterns gathered from the member pipelines' insights, + each tagged with its pipeline. Empty when the inventory carries no insights. + has_simplification: ``True`` when any recommended pattern is a ``simplification_pattern`` -- + surfaced prominently so the user sees a re-architecture option, not a buried sub-key. + """ + + recommended_patterns: list[AgenticPattern] = field(default_factory=list) + has_simplification: bool = False + + +@dataclass(slots=True, kw_only=True) +class ComponentOptions: + """Both conversion options for a component, as first-class peers.""" + + deterministic: DeterministicOption + agentic: AgenticOption + + +@dataclass(slots=True, kw_only=True) +class ComponentPlan: + """One component's routing decision. + + Attributes: + component_id: Stable id assigned by the library (``"component-"``). + members: The component's pipeline names, sorted (library-computed). + recommended: The library's starting suggestion -- ``"deterministic"`` when the component is + engine-capable, else ``"agentic"``. + decision: The user's authored per-component choice (may override :attr:`recommended`). + options: Both conversion options with their evidence (library-computed). Optional here so the + authored input -- which carries only the decision -- can round-trip through this model. + rationale: Optional author note on why this decision was chosen. + """ + + component_id: str + members: list[str] = field(default_factory=list) + recommended: str | None = None + decision: str + options: ComponentOptions | None = None + rationale: str | None = None + + +@dataclass(slots=True, kw_only=True) +class ConversionPlan: + """The recorded conversion-plan artifact. + + Attributes: + components: One :class:`ComponentPlan` per connected component. + findings: Human-readable notes about unresolved/dangling control edges retained during + component computation (never silently severed). + schema_version: Library-owned plan schema version. + inventory_sha256: Library-owned fingerprint binding the plan to the deterministic inventory + base (see :func:`flowx.discovery_insights.inventory_fingerprint`). + """ + + components: list[ComponentPlan] = field(default_factory=list) + findings: list[str] = field(default_factory=list) + schema_version: str = SCHEMA_VERSION + inventory_sha256: str | None = None diff --git a/src/flowx/route_agentic.py b/src/flowx/route_agentic.py new file mode 100644 index 00000000..09cc7989 --- /dev/null +++ b/src/flowx/route_agentic.py @@ -0,0 +1,435 @@ +"""Phase-1 in-engine agentic conversion: apply a routing decision, then fill the gaps. + +``convert`` always produces a deterministic ``.work/translation_report.json`` -- that translation is +**unchanged** by this module. Routing then decides, per connected component, whether each part +converts deterministically or agentically (:mod:`flowx.routing` computes the recommendation and +records the fingerprint-bound ``metadata/conversion_plan.json``). This module carries out the +**post-convert alteration and fill** the decision implies, always keeping the work in-engine so the +package phase, structural validation, and provenance all still apply: + +* :func:`alter_report` rewrites the report for the routed-**agentic** groups only: every task in an + agentic-routed pipeline is removed and replaced by a :class:`~flowx.models.ir.PlaceholderActivity`, + and exactly one pipeline-tagged :class:`~flowx.models.ir.AgenticGap` is emitted per routed task, + replacing (never appending to) any prior gap for those pipelines' tasks, so the standard gap-fill + path handles them without duplicates and the edit is idempotent. Pipelines in deterministic groups + are left byte-identical, and when nothing is routed agentic the report and gaps are returned + unchanged -- the non-breaking guarantee. +* the agent authors the fill. **Per-pipeline** agentic reuses the existing name-matched + :func:`flowx.ir_serde.merge_agentic_results` (no new code here). **Cross-pipeline COMBINE** + (N pipelines -> M, e.g. one Lakeflow Connect pipeline) is the one net-new capability: + :func:`combine_group_fill` swaps the routed group's pipelines for the agent-authored pipeline(s), + which carry :class:`~flowx.models.ir.AgenticComponentActivity` nodes (the escape hatch). +* :func:`validate_report_structurally` packages the merged report through the same + ``prepare -> write_bundle`` path the package phase uses and runs the existing + :func:`flowx.validate.bundle_invariants.check_bundle_dir` over the output, so a fill can never + introduce duplicate keys, dangling dependencies, a cycle, or a dangling pipeline/run_job reference. + +There is no LLM here: the tool edits the report and validates the authored fill; the fill itself is +supplied by the agent/harness, following the same ask -> author -> continue pattern as the existing +agentic flow. +""" + +from __future__ import annotations + +import copy +import json +import os +import sys +import tempfile +from collections.abc import Callable, Iterable +from pathlib import Path +from typing import Any + +from flowx.models.conversion_plan import DECISION_AGENTIC + +# The report + gaps live under the shared output dir's transient .work/ folder, beside the pipeline IR. +WORK_DIRNAME = ".work" +REPORT_FILENAME = "translation_report.json" +GAPS_FILENAME = "gaps.json" +# The recorded plan + inventory the combine fill binds against live under metadata/. +METADATA_DIRNAME = "metadata" +INVENTORY_FILENAME = "inventory.json" + +# Guidance stamped onto every placeholder the alteration produces. +_PLACEHOLDER_COMMENT = ( + "Routed agentic by the conversion plan; author a replacement task (per-pipeline fill) or replace " + "the whole group with agent-authored pipeline(s) (cross-pipeline combine)." +) + + +# --------------------------------------------------------------------------- # +# Reading the recorded / authored plan. +# --------------------------------------------------------------------------- # + + +def agentic_pipeline_names(plan: dict[str, Any]) -> set[str]: + """Return the member pipelines of every component the plan decides ``"agentic"``. + + Reads either an authored plan (the agent-supplied ``{"components": [...]}``) or the recorded + ``conversion_plan.json`` -- both carry ``members`` and ``decision`` per component. + """ + names: set[str] = set() + for component in plan.get("components", []): + if isinstance(component, dict) and component.get("decision") == DECISION_AGENTIC: + names.update(str(member) for member in component.get("members", []) if isinstance(member, str)) + return names + + +# --------------------------------------------------------------------------- # +# Interactive decision prompt (one route from the user's seat). +# --------------------------------------------------------------------------- # + + +def _stderr(line: str) -> None: + """Default sink for the interactive prompt's narration -- stderr keeps stdout clean for JSON.""" + print(line, file=sys.stderr) + + +def prompt_for_decisions( + recommendation: dict[str, Any], + *, + input_fn: Callable[[str], str] | None = None, + output_fn: Callable[[str], None] | None = None, +) -> dict[str, Any]: + """Ask the user, per connected component, whether to convert deterministically or agentically. + + Shows each component's members, the library's recommendation, and -- when the agentic option + carries a re-architecture (e.g. a multi-pipeline -> Lakeflow Connect simplification) -- flags it + prominently. An empty answer accepts the recommendation; ``d``/``a`` (or the full words) choose + a route. Returns an authored plan dict (``{"components": [{component_id, members, decision}]}``) + ready to hand to :func:`flowx.routing.record_plan`. + + ``input_fn`` and ``output_fn`` are injectable so the prompt is exercised without a real TTY; both + are resolved at call time (defaulting to the builtin ``input`` and a stderr sink) so a test that + patches ``builtins.input`` is honoured. + """ + ask = input_fn if input_fn is not None else input + say = output_fn if output_fn is not None else _stderr + components_out: list[dict[str, Any]] = [] + for component in recommendation.get("components", []): + component_id = str(component.get("component_id")) + members = list(component.get("members", [])) + recommended = str(component.get("recommended", "deterministic")) + options = component.get("options") or {} + has_simplification = bool((options.get("agentic") or {}).get("has_simplification")) + + say(f"\nComponent {component_id}: {', '.join(members) or '(none)'}") + say(f" recommended: {recommended}") + if has_simplification: + say(" agentic option includes a simplification re-architecture (e.g. Lakeflow Connect)") + answer = ask(f" route [d]eterministic / [a]gentic (default={recommended}): ").strip().lower() + if answer in ("a", "agentic"): + decision = DECISION_AGENTIC + elif answer in ("d", "deterministic"): + decision = "deterministic" + else: + decision = recommended + components_out.append({"component_id": component_id, "members": members, "decision": decision}) + return {"components": components_out} + + +# --------------------------------------------------------------------------- # +# The edit step: rewrite the report for routed-agentic groups. +# --------------------------------------------------------------------------- # + + +def _report_pipelines(report: dict[str, Any]) -> list[dict[str, Any]]: + """The pipeline dicts in a report, whether it is single-pipeline or a ``{"pipelines": [...]}`` wrapper.""" + if isinstance(report, dict) and "pipelines" in report and isinstance(report["pipelines"], list): + return [pipeline for pipeline in report["pipelines"] if isinstance(pipeline, dict)] + return [report] + + +def _placeholder_and_gap(task: dict[str, Any], pipeline_name: str) -> tuple[dict[str, Any], dict[str, Any]]: + """Turn one task into a placeholder + gap, preserving its identity and edges. + + Idempotent: a task that is already a ``PlaceholderActivity`` (a re-run of the edit) is kept as-is + and its gap is rebuilt from the recorded ``original_type`` / ``raw_definition`` rather than + wrapping the placeholder in another placeholder. + """ + name = task.get("name") + if task.get("type") == "PlaceholderActivity": + original_type = str(task.get("original_type", "unknown")) + gap: dict[str, Any] = { + "activity_name": name, + "activity_type": original_type, + "raw_definition": task.get("raw_definition"), + "pipeline": pipeline_name, + } + return task, gap + + original_type = str(task.get("type", "unknown")) + placeholder: dict[str, Any] = { + "name": name, + "task_key": task.get("task_key"), + "type": "PlaceholderActivity", + "original_type": original_type, + "comment": _PLACEHOLDER_COMMENT, + # Keep the deterministic translation as context so the agent can author against it. + "raw_definition": task, + } + if task.get("depends_on"): + placeholder["depends_on"] = task["depends_on"] + gap = { + "activity_name": name, + "activity_type": original_type, + "raw_definition": task, + "pipeline": pipeline_name, + } + return placeholder, gap + + +def alter_report( + report: dict[str, Any], + gaps: list[dict[str, Any]], + agentic_pipelines: Iterable[str], +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + """Rewrite the report so every routed-agentic pipeline's tasks become placeholder gaps. + + Returns ``(new_report, new_gaps)``. Pipelines not in ``agentic_pipelines`` are copied through + untouched. When ``agentic_pipelines`` is empty the inputs are returned unchanged (identity), so a + no-decision / all-deterministic route leaves today's output byte-for-byte intact. + """ + agentic = {name for name in agentic_pipelines} + if not agentic: + return report, gaps + + report = copy.deepcopy(report) + + # Task names owned by pipelines that are NOT routed agentic. Convert gaps are untagged (they carry + # no pipeline), so a name that also belongs to a non-routed pipeline is ambiguous: dropping it + # could remove that pipeline's gap, so it is preserved. Computed before mutating routed pipelines. + nonrouted_task_names: set[str] = set() + for pipeline in _report_pipelines(report): + if pipeline.get("name") in agentic: + continue + for task in pipeline.get("tasks", []): + if isinstance(task, dict): + nonrouted_task_names.add(str(task.get("name"))) + + # Emit exactly one pipeline-tagged gap per routed task, replacing (never appending to) any prior + # gap for a routed pipeline's tasks. Without this, a task that convert already recorded as an + # agentic gap -- or a re-run of the edit -- would leave duplicate/again-appended gaps. + fresh_gaps: list[dict[str, Any]] = [] + routed_task_names: set[str] = set() + for pipeline in _report_pipelines(report): + if pipeline.get("name") not in agentic: + continue + placeholders: list[dict[str, Any]] = [] + for task in pipeline.get("tasks", []): + if not isinstance(task, dict): + continue + placeholder, gap = _placeholder_and_gap(task, str(pipeline.get("name"))) + placeholders.append(placeholder) + fresh_gaps.append(gap) + routed_task_names.add(str(task.get("name"))) + pipeline["tasks"] = placeholders + + kept_gaps: list[dict[str, Any]] = [] + for gap in gaps: + pipeline_name = gap.get("pipeline") if isinstance(gap, dict) else None + activity_name = gap.get("activity_name") if isinstance(gap, dict) else None + # Drop our own prior tagged gaps for now-routed pipelines (idempotent re-run). + if pipeline_name in agentic: + continue + # Drop an untagged convert gap only when the name belongs *exclusively* to a routed pipeline -- + # a fresh tagged gap now supersedes it. When the same name also names a task in a non-routed + # pipeline, the gap is ambiguous and kept, so a non-routed pipeline never loses its gap. + if pipeline_name is None and activity_name in routed_task_names and activity_name not in nonrouted_task_names: + continue + kept_gaps.append(gap) + return report, kept_gaps + fresh_gaps + + +def _write_json_atomic(path: Path, document: Any) -> None: + """Write JSON atomically (temp file + ``os.replace``), matching the report's 2-space indentation.""" + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.tmp") + temporary.write_text(json.dumps(document, indent=2, default=str), encoding="utf-8") + os.replace(temporary, path) + + +def apply_plan_to_report(output_dir: Path, plan: dict[str, Any]) -> dict[str, Any]: + """Apply a routing decision to the report on disk: placeholder the routed-agentic groups. + + Reads ``/.work/translation_report.json`` (and ``gaps.json`` when present), rewrites + them for the plan's agentic components, and writes them back atomically. When no component is + routed agentic the files are left untouched, so the non-breaking guarantee holds. + + Returns a summary dict with the altered pipeline names and the resulting gap count. + + Raises: + FileNotFoundError: when the translation report is missing (run convert first). + """ + work = Path(output_dir) / WORK_DIRNAME + report_path = work / REPORT_FILENAME + if not report_path.exists(): + raise FileNotFoundError(f"No {REPORT_FILENAME} under {work}; run the convert phase first.") + gaps_path = work / GAPS_FILENAME + + agentic = agentic_pipeline_names(plan) + if not agentic: + return {"agentic_pipelines": [], "gaps": 0, "altered": False} + + report = json.loads(report_path.read_text(encoding="utf-8")) + gaps = json.loads(gaps_path.read_text(encoding="utf-8")) if gaps_path.exists() else [] + if not isinstance(gaps, list): + gaps = [] + + new_report, new_gaps = alter_report(report, gaps, agentic) + _write_json_atomic(report_path, new_report) + _write_json_atomic(gaps_path, new_gaps) + return {"agentic_pipelines": sorted(agentic), "gaps": len(new_gaps), "altered": True} + + +# --------------------------------------------------------------------------- # +# Cross-pipeline COMBINE: the pipeline-grain fill. +# --------------------------------------------------------------------------- # + + +def combine_group_fill( + report: dict[str, Any], + group_members: Iterable[str], + authored_pipelines: list[dict[str, Any]], +) -> dict[str, Any]: + """Replace a routed group's pipelines with the agent-authored pipeline(s). + + Drops every pipeline whose name is in ``group_members`` and appends the ``authored_pipelines`` + (each a pipeline IR dict, typically carrying ``AgenticComponentActivity`` nodes). Pipelines + outside the group are preserved in order. Returns a ``{"pipelines": [...]}`` report; the input is + not mutated. + """ + members = {name for name in group_members} + kept = [copy.deepcopy(pipeline) for pipeline in _report_pipelines(report) if pipeline.get("name") not in members] + kept.extend(copy.deepcopy(pipeline) for pipeline in authored_pipelines) + return {"pipelines": kept} + + +def _resolve_agentic_component(output_dir: Path, members: set[str]) -> tuple[str | None, str | None]: + """Bind ``members`` to a routed-**agentic** component in the recorded, fingerprint-bound plan. + + Reads ``metadata/conversion_plan.json`` and ``metadata/inventory.json`` and returns + ``(component_id, error)``. The error is set (and ``component_id`` is ``None``) when: the plan or + inventory is missing; the plan's ``inventory_sha256`` no longer matches the current inventory (a + stale plan); ``members`` do not exactly equal one component's members (a partial, superset, or + mistyped group); or the exactly-matching component is routed deterministic rather than agentic. + Requiring an exact match to a routed-agentic component stops a caller from swapping deterministic + pipelines or a partial group. + """ + from flowx.discovery_insights import inventory_fingerprint + + metadata = Path(output_dir) / METADATA_DIRNAME + plan_path = metadata / "conversion_plan.json" + inventory_path = metadata / INVENTORY_FILENAME + if not plan_path.exists(): + return None, "No metadata/conversion_plan.json; record a routing decision with `route` first." + if not inventory_path.exists(): + return None, "No metadata/inventory.json; run the discover phase first." + + plan = json.loads(plan_path.read_text(encoding="utf-8")) + inventory = json.loads(inventory_path.read_text(encoding="utf-8")) + recorded_fingerprint = plan.get("inventory_sha256") + current_fingerprint = inventory_fingerprint(inventory) + if recorded_fingerprint != current_fingerprint: + return None, ( + "conversion_plan.json is stale: it was recorded against a different inventory " + f"({recorded_fingerprint!r} != {current_fingerprint!r}); re-run `route` before filling." + ) + + for component in plan.get("components", []): + if not isinstance(component, dict): + continue + component_members = {str(member) for member in component.get("members", [])} + if component_members != members: + continue + if component.get("decision") == DECISION_AGENTIC: + return str(component.get("component_id")), None + return None, ( + f"members {sorted(members)} match component {component.get('component_id')!r}, which is " + f"routed {component.get('decision')!r}, not agentic; only routed-agentic components can be combined." + ) + return None, ( + f"members {sorted(members)} do not exactly match any component in the recorded plan " + "(partial, superset, or mistyped); pass the exact member set of one routed-agentic component." + ) + + +def apply_combine_fill( + output_dir: Path, + group_members: Iterable[str], + authored_pipelines: list[dict[str, Any]], +) -> dict[str, Any]: + """Combine a routed-agentic group into agent-authored pipeline(s) on disk. + + The group's membership is **bound to the recorded plan**: ``group_members`` must exactly match a + routed-agentic component in ``metadata/conversion_plan.json`` (whose fingerprint must still match + the current inventory), so a caller cannot swap deterministic pipelines or a partial/typoed group. + The merged report is then **always** validated with the structural bundle invariants (a real + ``prepare -> write_bundle`` pass over :func:`validate_report_structurally`) -- there is no bypass -- + and written back only when it passes, so a dangling reference or duplicate key never lands on disk. + + Returns ``{"ok", "violations", "error", "component_id", "pipelines"}``. ``ok`` is ``False`` (and + nothing written) on a plan/membership error (``error`` set) or any structural violation + (``violations`` set). + + Raises: + FileNotFoundError: when the translation report is missing (run convert first). + """ + members = {str(member) for member in group_members} + component_id, error = _resolve_agentic_component(output_dir, members) + if error is not None: + return {"ok": False, "error": error, "violations": [], "pipelines": 0} + + work = Path(output_dir) / WORK_DIRNAME + report_path = work / REPORT_FILENAME + if not report_path.exists(): + raise FileNotFoundError(f"No {REPORT_FILENAME} under {work}; run the convert phase first.") + + report = json.loads(report_path.read_text(encoding="utf-8")) + merged = combine_group_fill(report, members, authored_pipelines) + + result = validate_report_structurally(merged) + if not result.ok: + violations = [f"[{finding.code}] {finding.location}: {finding.message}" for finding in result.violations] + return {"ok": False, "error": None, "violations": violations, "pipelines": 0} + + _write_json_atomic(report_path, merged) + return { + "ok": True, + "error": None, + "violations": [], + "component_id": component_id, + "pipelines": len(merged["pipelines"]), + } + + +# --------------------------------------------------------------------------- # +# Structural validation via the existing bundle invariants. +# --------------------------------------------------------------------------- # + + +def validate_report_structurally(report: dict[str, Any]) -> Any: + """Package the report to a throwaway bundle and run the existing structural invariants over it. + + Uses the same ``prepare_workflow -> write_bundle`` path as the package phase, then aggregates + :func:`flowx.validate.bundle_invariants.check_bundle_dir` findings across every emitted bundle so + duplicate keys, dangling dependencies, cycles, and dangling pipeline/run_job references are all + caught before the merged report is trusted. Returns a + :class:`~flowx.validate.bundle_invariants.BundleInvariantResult`. + """ + from flowx.bundler.dab_writer import pipeline_dict_to_ir, write_bundle + from flowx.preparer.workflow_preparer import prepare_workflow + from flowx.utils import normalize_task_key + from flowx.validate.bundle_invariants import BundleInvariantResult, check_bundle_dir + + pipelines = _report_pipelines(report) + findings: list[Any] = [] + with tempfile.TemporaryDirectory(prefix="flowx-fill-validate-") as temporary: + root = Path(temporary) + for pipeline_dict in pipelines: + pipeline, _skipped = pipeline_dict_to_ir(pipeline_dict) + workflow = prepare_workflow(pipeline) + bundle_dir = root / normalize_task_key(pipeline.name) if len(pipelines) > 1 else root + write_bundle(workflow, bundle_dir) + findings.extend(check_bundle_dir(bundle_dir).findings) + return BundleInvariantResult(findings=findings) diff --git a/src/flowx/routing.py b/src/flowx/routing.py new file mode 100644 index 00000000..a6f318cf --- /dev/null +++ b/src/flowx/routing.py @@ -0,0 +1,539 @@ +"""Compute a per-connected-component conversion route over the discover inventory (#77). + +The discover phase writes a deterministic ``metadata/inventory.json``; :mod:`flowx.discovery_insights` +optionally enriches it with an additive ``insights`` block. This module turns those signals into a +**routing recommendation** and records the **user's decision** as a fingerprint-bound +``metadata/conversion_plan.json`` artifact -- a Phase-1, descriptive-only step: + +* pipelines are grouped into weak/undirected **connected components** over the inventory's control + lineage (``lineage.control_edges``), so mutually-referencing pipelines are decided together and a + caller/callee reference is never split across incompatible routes; +* each component surfaces **both conversion options as first-class peers** -- a *deterministic* + option (the engine-capability assessment: every activity engine-capable via its ``strategy`` or + claimed by a detected motif, plus its motif/coverage evidence and any uncovered gaps) and an + *agentic* option (the recommended Databricks patterns from the ``insights`` block, with any + ``simplification_pattern`` such as a multi-pipeline -> Lakeflow Connect re-architecture surfaced + prominently) -- so the user can choose per group; +* ``recommended`` is a library-computed starting suggestion (deterministic when the whole component + is engine-capable), and ``decision`` is the user's authored per-component choice. + +Like :mod:`flowx.discovery_insights`, there is **no LLM here**: the tool computes the recommendation +deterministically and only *validates and records* the agent-authored decision. The agent authors +**only** the decision (and an optional rationale); the library recomputes ``members``, +``recommended``, and both options' evidence on record so they can never drift from the inventory or +be faked. + +The recorded plan is bound to the inventory via :func:`flowx.discovery_insights.inventory_fingerprint` +-- a SHA-256 over the deterministic inventory base (the ``insights`` block excluded). That base is +exactly the structural signal that decides component membership and engine capability (pipelines, +control lineage, per-activity strategy, motifs); insights are advisory evidence for the agentic +option, not part of the binding. The write is atomic (temp file + ``os.replace``) and idempotent, so +re-recording the same decision against the same inventory rewrites byte-identical bytes. + +This is additive, opt-in routing metadata only: with no recorded plan, ``convert`` and ``package`` +behave exactly as today. Component computation reads ``lineage.control_edges``, which both ADF +(``ExecutePipeline``) and Airflow (``RunJob``) emit, so the artifact is source-neutral. +""" + +from __future__ import annotations + +import json +import os +from collections import Counter +from collections.abc import Iterable +from pathlib import Path +from typing import Any + +from flowx.discovery_insights import inventory_fingerprint +from flowx.models.conversion_plan import DECISIONS, SCHEMA_VERSION + +# The strategy value that marks an activity as individually engine-capable (a 1:1 deterministic +# translation). Any other value (``"agentic"`` / ``"unsupported"`` / missing) is a gap unless the +# activity is claimed by a detected motif -- the multi-activity capability signal from #64. +_DETERMINISTIC_STRATEGY = "deterministic" + +# The recorded conversion-plan artifact lives beside inventory.json under metadata/. +PLAN_FILENAME = "conversion_plan.json" + +# Authored top-level keys (everything else the library owns and rejects on input). +_PLAN_TOP_KEYS = {"components"} +_LIBRARY_TOP_KEYS = {"schema_version", "inventory_sha256", "findings"} +# Authored per-component keys vs the fields the library recomputes and rejects on input. +_COMPONENT_AUTHORED_KEYS = {"component_id", "members", "decision", "rationale"} +_COMPONENT_LIBRARY_KEYS = {"recommended", "options"} + + +def _pipeline_names(inventory: dict[str, Any]) -> set[str]: + """The set of real pipeline names in the inventory (the foreign-key domain). + + A local copy of the same projection :mod:`flowx.discovery_insights` uses, kept here so routing + does not depend on that module's private helpers. + """ + return { + str(pipeline["name"]) + for pipeline in inventory.get("pipelines", []) + if isinstance(pipeline, dict) and pipeline.get("name") is not None + } + + +def _control_edges(inventory: dict[str, Any]) -> Iterable[tuple[str, str, str, bool]]: + """Yield ``(source_workflow, target_workflow, via_task_key, resolved)`` for every control edge. + + Lineage is placed per pipeline (one block beside each pipeline's ``activities``), so every + pipeline's ``lineage.control_edges`` are gathered into one stream, preserving inventory order. + """ + for pipeline in inventory.get("pipelines", []): + if not isinstance(pipeline, dict): + continue + lineage = pipeline.get("lineage") or {} + for edge in lineage.get("control_edges", []): + if not isinstance(edge, dict): + continue + yield ( + str(edge.get("source_workflow") or ""), + str(edge.get("target_workflow") or ""), + str(edge.get("via_task_key") or ""), + bool(edge.get("resolved", True)), + ) + + +def build_components(inventory: dict[str, Any]) -> tuple[list[list[str]], list[str]]: + """Group pipelines into weak/undirected connected components over control lineage. + + Two pipelines share a component when a **resolved** control edge joins them in either direction + and both endpoints are real inventory pipelines. Isolated pipelines each form their own + singleton component. An edge whose callee is unresolved (``resolved`` is False) or names a + pipeline absent from the inventory is **not** used to join anything and is **not** silently + severed -- it is recorded as a finding so a partial export never quietly collapses two + components into one or drops a coupling. + + Args: + inventory: The discover ``inventory.json`` document (deterministic or enriched). + + Returns: + ``(components, findings)`` where ``components`` is a list of member lists -- each sorted, + the whole list ordered by first member -- so the output is deterministic and idempotent for + a given inventory; and ``findings`` is a sorted list of human-readable notes about + unresolved/dangling edges. + """ + names = _pipeline_names(inventory) + parent: dict[str, str] = {name: name for name in names} + + def find(node: str) -> str: + root = node + while parent[root] != root: + root = parent[root] + while parent[node] != root: + parent[node], node = root, parent[node] + return root + + def union(left: str, right: str) -> None: + left_root, right_root = find(left), find(right) + if left_root != right_root: + # Attach the lexicographically larger root under the smaller for a stable shape. + low, high = sorted((left_root, right_root)) + parent[high] = low + + findings: set[str] = set() + for source, target, via, resolved in _control_edges(inventory): + if not resolved or not target or target not in names: + shown_target = target or "" + findings.add( + f"unresolved control edge {source!r} -> {shown_target!r} (via {via!r}) " + f"kept as a finding, not severed; {source!r} is routed within its own component" + ) + continue + if source in names: + union(source, target) + + groups: dict[str, list[str]] = {} + for name in names: + groups.setdefault(find(name), []).append(name) + components = sorted((sorted(members) for members in groups.values()), key=lambda members: members) + return components, sorted(findings) + + +def _activities_by_pipeline(inventory: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: + """Map each pipeline name to its inventory ``activities`` list (flattened, as emitted).""" + result: dict[str, list[dict[str, Any]]] = {} + for pipeline in inventory.get("pipelines", []): + if isinstance(pipeline, dict) and pipeline.get("name") is not None: + activities = pipeline.get("activities") or [] + result[str(pipeline["name"])] = [entry for entry in activities if isinstance(entry, dict)] + return result + + +def _motifs_by_pipeline(inventory: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: + """Map each pipeline name to its additive ``motifs`` list (the #64 capability signal).""" + result: dict[str, list[dict[str, Any]]] = {} + for pipeline in inventory.get("pipelines", []): + if isinstance(pipeline, dict) and pipeline.get("name") is not None: + motifs = pipeline.get("motifs") or [] + result[str(pipeline["name"])] = [entry for entry in motifs if isinstance(entry, dict)] + return result + + +def _pipeline_insights(inventory: dict[str, Any]) -> dict[str, dict[str, Any]]: + """Map each pipeline name to its ``insights.pipeline_insights`` entry, when insights are present. + + Returns an empty map when the inventory has not been enriched, so the agentic option degrades + to an empty pattern list rather than failing. + """ + insights = inventory.get("insights") + if not isinstance(insights, dict): + return {} + result: dict[str, dict[str, Any]] = {} + for entry in insights.get("pipeline_insights", []): + if isinstance(entry, dict) and entry.get("pipeline") is not None: + result[str(entry["pipeline"])] = entry + return result + + +def _deterministic_option(members: list[str], inventory: dict[str, Any]) -> dict[str, Any]: + """Assess the deterministic (1:1 engine) conversion of a component. + + An activity is engine-capable when its ``strategy`` is ``"deterministic"`` or it is claimed by a + detected motif (present in some motif's ``member_task_keys``). ``capable`` is True only when the + whole component leaves no gap; ``uncovered`` cites each activity that keeps it from being fully + deterministic. + """ + activities_by_pipeline = _activities_by_pipeline(inventory) + motifs_by_pipeline = _motifs_by_pipeline(inventory) + + motif_ids: list[str] = [] + # Motif coverage is keyed by (pipeline, activity name), never bare name: a motif claiming a task + # in one pipeline must not mark an unrelated same-named task in another pipeline of the component + # as covered, which would silently hide a real gap. + covered_activities: set[tuple[str, str]] = set() + for pipeline in members: + for motif in motifs_by_pipeline.get(pipeline, []): + if motif.get("motif_id") is not None: + motif_ids.append(str(motif["motif_id"])) + covered_activities.update((pipeline, str(key)) for key in motif.get("member_task_keys") or []) + + counts = {"deterministic": 0, "agentic": 0, "unsupported": 0} + uncovered: list[dict[str, Any]] = [] + for pipeline in members: + for activity in activities_by_pipeline.get(pipeline, []): + strategy = activity.get("strategy") + bucket = strategy if strategy in ("deterministic", "agentic") else "unsupported" + counts[bucket] += 1 + name = activity.get("name") + if strategy != _DETERMINISTIC_STRATEGY and (pipeline, name) not in covered_activities: + uncovered.append( + { + "pipeline": pipeline, + "activity": name, + "type": activity.get("type"), + "strategy": strategy, + } + ) + return { + "capable": not uncovered, + "activity_counts": counts, + "motifs": sorted(set(motif_ids)), + "uncovered": uncovered, + } + + +def _agentic_option(members: list[str], inventory: dict[str, Any]) -> dict[str, Any]: + """Surface the agent-authored recommended patterns for a component as a first-class option. + + Draws every ``recommended_patterns`` entry from the member pipelines' insights, tagging each with + its pipeline, and flags whether any is a ``simplification_pattern`` (a distinctive re-architecture + such as a multi-pipeline -> Lakeflow Connect collapse) so the user sees it prominently. + """ + insights_by_pipeline = _pipeline_insights(inventory) + recommended_patterns: list[dict[str, Any]] = [] + for pipeline in members: + insight = insights_by_pipeline.get(pipeline) + if not insight: + continue + for pattern in insight.get("recommended_patterns") or []: + if isinstance(pattern, dict): + recommended_patterns.append({"pipeline": pipeline, **pattern}) + has_simplification = any(pattern.get("simplification_pattern") for pattern in recommended_patterns) + return {"recommended_patterns": recommended_patterns, "has_simplification": has_simplification} + + +def recommend_component(members: list[str], inventory: dict[str, Any]) -> tuple[str, dict[str, Any]]: + """Compute the recommended route and both conversion options for one component. + + Args: + members: The component's pipeline names. + inventory: The discover ``inventory.json`` document (deterministic or enriched). + + Returns: + ``(recommended, options)`` where ``recommended`` is ``"deterministic"`` when the whole + component is engine-capable, else ``"agentic"``; and ``options`` carries the ``deterministic`` + and ``agentic`` peers as first-class entries. + """ + deterministic = _deterministic_option(members, inventory) + agentic = _agentic_option(members, inventory) + recommended = _DETERMINISTIC_STRATEGY if deterministic["capable"] else "agentic" + return recommended, {"deterministic": deterministic, "agentic": agentic} + + +def build_recommendation(inventory: dict[str, Any]) -> dict[str, Any]: + """Compute the full routing recommendation over every component in the inventory. + + Returns a dict with ``components`` (each carrying ``component_id``, sorted ``members``, + ``recommended`` and both ``options``), the ``findings`` from component computation, and a + ``default_plan`` that proposes ``decision == recommended`` for every component -- ready to hand + straight to :func:`record_plan` when the user accepts the recommendations wholesale, or to edit + per component for overrides. + """ + components, findings = build_components(inventory) + component_entries: list[dict[str, Any]] = [] + default_plan_components: list[dict[str, Any]] = [] + for index, members in enumerate(components, start=1): + component_id = f"component-{index}" + recommended, options = recommend_component(members, inventory) + component_entries.append( + {"component_id": component_id, "members": members, "recommended": recommended, "options": options} + ) + default_plan_components.append({"component_id": component_id, "members": members, "decision": recommended}) + return { + "components": component_entries, + "findings": findings, + "default_plan": {"components": default_plan_components}, + } + + +# --------------------------------------------------------------------------- # +# Validation. All violations are collected (never fail-fast) so the authoring +# agent can fix every problem in one pass. +# --------------------------------------------------------------------------- # + + +def _components_by_id(inventory: dict[str, Any]) -> dict[str, list[str]]: + """Computed components keyed by their library id (``component-``).""" + components, _ = build_components(inventory) + return {f"component-{index}": members for index, members in enumerate(components, start=1)} + + +def validate_plan(raw: Any, inventory: dict[str, Any]) -> list[str]: + """Validate an authored conversion plan against the inventory's connected components. + + Returns a list of human-readable violation strings; an empty list means the plan is valid. Never + raises on a malformed payload. The rules: + + * only the authored top-level key ``components`` is allowed (library-owned keys are rejected with + a hint), and each component entry may carry only the authored fields (``recommended`` / + ``options`` are library-computed and rejected on input); + * ``decision`` must be one of the known routes and ``rationale`` (when present) a non-empty string; + * every member must be a real inventory pipeline, and a component's ``members`` must exactly match + one computed connected component -- so a decision can never split a component or span two; + * the plan is a **bijection** over components: every component is decided exactly once (no + partial plan, no duplicate/conflicting decisions). + """ + if not isinstance(raw, dict): + return [f"conversion plan must be a JSON object, got {type(raw).__name__}"] + + violations: list[str] = [] + for key in sorted(set(raw) - _PLAN_TOP_KEYS): + hint = " (set by the library, not the author)" if key in _LIBRARY_TOP_KEYS else "" + violations.append(f"unknown top-level key: {key!r}{hint}") + + components = raw.get("components") + if not isinstance(components, list): + violations.append("'components' must be a list") + return violations + + names = _pipeline_names(inventory) + computed_by_id = _components_by_id(inventory) + computed_by_members = {frozenset(members): component_id for component_id, members in computed_by_id.items()} + + decided_ids: list[str] = [] + for index, component in enumerate(components): + loc = f"components[{index}]" + matched_id = _validate_component_entry(component, loc, names, computed_by_members, violations) + if matched_id is not None: + decided_ids.append(matched_id) + + for component_id, count in Counter(decided_ids).items(): + if count > 1: + violations.append( + f"component {component_id!r} is decided {count} times; each component needs exactly one decision" + ) + decided = set(decided_ids) + for component_id, members in computed_by_id.items(): + if component_id not in decided: + violations.append(f"component {component_id!r} ({members}) has no decision; every component must be routed") + return violations + + +def _validate_component_entry( + component: Any, + loc: str, + names: set[str], + computed_by_members: dict[frozenset[str], str], + violations: list[str], +) -> str | None: + """Validate one authored component entry, appending problems; return the matched component id. + + Returns the computed ``component-`` id this entry decides when its ``members`` exactly match a + connected component (so the caller can enforce the bijection), else ``None``. + """ + if not isinstance(component, dict): + violations.append(f"{loc} must be an object") + return None + + for key in sorted(set(component) - _COMPONENT_AUTHORED_KEYS): + hint = " (set by the library, not the author)" if key in _COMPONENT_LIBRARY_KEYS else "" + violations.append(f"{loc}: unknown field {key!r}{hint}") + + decision = component.get("decision") + if decision not in DECISIONS: + allowed = ", ".join(repr(value) for value in DECISIONS) + violations.append(f"{loc}: 'decision' must be one of {{{allowed}}}, got {decision!r}") + + rationale = component.get("rationale") + if rationale is not None and (not isinstance(rationale, str) or not rationale.strip()): + violations.append(f"{loc}: 'rationale' must be a non-empty string when present") + + component_id = component.get("component_id") + if not isinstance(component_id, str) or not component_id: + violations.append(f"{loc}: 'component_id' must be a non-empty string") + + members = component.get("members") + if not isinstance(members, list) or not all(isinstance(member, str) for member in members): + violations.append(f"{loc}: 'members' must be a list of pipeline names") + return None + + for member in members: + if member not in names: + violations.append(f"{loc}: pipeline {member!r} not in inventory") + + # Reject duplicate members explicitly: a frozenset match would collapse ["a", "a"] to {"a"} and + # wrongly accept it as the component {"a"}, breaking the members-match / bijection contract. + duplicates = sorted({member for member in members if members.count(member) > 1}) + if duplicates: + violations.append(f"{loc}: duplicate members {duplicates}; list each pipeline once") + return None + + matched_id = computed_by_members.get(frozenset(members)) + if matched_id is None: + violations.append( + f"{loc}: members {sorted(members)} do not form a connected component " + f"(they split or span computed components); route each component as a whole" + ) + return None + if isinstance(component_id, str) and component_id and component_id != matched_id: + violations.append( + f"{loc}: component_id {component_id!r} does not match the component for these members " + f"(expected {matched_id!r})" + ) + return matched_id + + +# --------------------------------------------------------------------------- # +# Loading, recording, and the atomic idempotent write. +# --------------------------------------------------------------------------- # + + +def load_plan(*, plan: dict[str, Any] | None = None, plan_path: Path | None = None) -> dict[str, Any]: + """Return the raw authored plan dict from exactly one source (inline or file). + + Raises: + ValueError: if neither or both sources are provided. + """ + if (plan is None) == (plan_path is None): + raise ValueError("provide exactly one of 'plan' (inline dict) or 'plan_path'") + if plan is not None: + return plan + assert plan_path is not None # guaranteed by the guard above + return json.loads(Path(plan_path).read_text(encoding="utf-8")) + + +def build_plan_document(inventory: dict[str, Any], raw: dict[str, Any]) -> dict[str, Any]: + """Build the recorded plan document from a validated authored plan. + + The library recomputes ``members`` / ``recommended`` / both ``options`` and overlays only the + authored ``decision`` (and optional ``rationale``) per component, so the recorded facts cannot + drift from the inventory. Stamps the library-owned ``schema_version`` and ``inventory_sha256``. + Does not mutate the inputs and performs no I/O. + """ + recommendation = build_recommendation(inventory) + computed_by_id = _components_by_id(inventory) + members_to_id = {frozenset(members): component_id for component_id, members in computed_by_id.items()} + + authored_by_id: dict[str, dict[str, Any]] = {} + for component in raw.get("components", []): + component_id = members_to_id[frozenset(component["members"])] + authored_by_id[component_id] = component + + components_out: list[dict[str, Any]] = [] + for entry in recommendation["components"]: + component_id = entry["component_id"] + authored = authored_by_id[component_id] + recorded: dict[str, Any] = { + "component_id": component_id, + "members": entry["members"], + "recommended": entry["recommended"], + "decision": authored["decision"], + "options": entry["options"], + } + rationale = authored.get("rationale") + if rationale is not None: + recorded["rationale"] = rationale + components_out.append(recorded) + + return { + "schema_version": SCHEMA_VERSION, + "inventory_sha256": inventory_fingerprint(inventory), + "components": components_out, + "findings": recommendation["findings"], + } + + +def _write_plan_atomic(path: Path, document: dict[str, Any]) -> None: + """Write the plan JSON atomically (temp file + ``os.replace``), matching the inventory formatting.""" + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.tmp") + temporary.write_text(json.dumps(document, indent=2), encoding="utf-8") + os.replace(temporary, path) + + +def record_plan( + output_dir: Path, + *, + plan: dict[str, Any] | None = None, + plan_path: Path | None = None, +) -> dict[str, Any]: + """Validate an authored plan against the inventory, then record it on success. + + Reads ``/metadata/inventory.json``, validates the authored plan, and -- only when + there are no violations -- writes ``/metadata/conversion_plan.json`` atomically. The + inventory file is never touched. Provide the authored plan via exactly one of ``plan`` (inline + dict) or ``plan_path`` (a JSON file). + + Returns ``{"ok", "violations", "inventory_sha256", "components", "findings"}``. ``ok`` is + ``False`` (and no plan written) when there are violations. + + Raises: + FileNotFoundError: when ``inventory.json`` does not exist (run discover first). + ValueError: when neither or both plan sources are provided, or the inventory is not a JSON + object. + """ + inventory_path = Path(output_dir) / "metadata" / "inventory.json" + if not inventory_path.exists(): + raise FileNotFoundError(f"No inventory.json under {inventory_path.parent}; run the discover phase first.") + inventory = json.loads(inventory_path.read_text(encoding="utf-8")) + if not isinstance(inventory, dict): + raise ValueError(f"inventory.json must contain a JSON object, got {type(inventory).__name__}") + + raw = load_plan(plan=plan, plan_path=plan_path) + violations = validate_plan(raw, inventory) + if violations: + return {"ok": False, "violations": violations, "components": 0} + + document = build_plan_document(inventory, raw) + _write_plan_atomic(inventory_path.with_name(PLAN_FILENAME), document) + return { + "ok": True, + "violations": [], + "inventory_sha256": document["inventory_sha256"], + "components": len(document["components"]), + "findings": len(document["findings"]), + } diff --git a/tests/unit/test_cli_route_agentic.py b/tests/unit/test_cli_route_agentic.py new file mode 100644 index 00000000..302d0bb3 --- /dev/null +++ b/tests/unit/test_cli_route_agentic.py @@ -0,0 +1,331 @@ +"""Tests for the reshaped ``route`` CLI and the ``fill-agentic`` combine surface. + +``route`` is now one command: with no decision on a non-TTY it emits the recommendation (a dry run +agents read first); with a decision (``--plan-path``, ``--plan-path -`` for stdin, or an interactive +TTY prompt) it records the fingerprint-bound plan AND edits the report for the routed-agentic groups. +``fill-agentic combine`` performs the cross-pipeline pipeline-grain fill, validating structurally +before writing. +""" + +from __future__ import annotations + +import io +import json +from pathlib import Path +from typing import Any + +import pytest + +from flowx.adapter.__main__ import main as adapter_cli_main +from flowx.discovery_inventory import STRATEGY_PROPERTY, build_source_inventory +from flowx.models.discovery import CONCEPT_NOTEBOOK, SourceGraph, SourceNode +from flowx.models.ir import ControlEdge, Lineage +from flowx.route_agentic import GAPS_FILENAME, REPORT_FILENAME, WORK_DIRNAME + + +def _node(task_key: str, native_type: str, *, strategy: str = "deterministic") -> SourceNode: + return SourceNode( + source_id=task_key, + task_key=task_key, + concept=CONCEPT_NOTEBOOK, + source="unit", + name=task_key, + native_type=native_type, + properties={STRATEGY_PROPERTY: strategy}, + raw={"name": task_key, "type": native_type}, + ) + + +def _inventory() -> dict[str, Any]: + """One component 'parent -> child' plus a standalone 'solo'.""" + parent = SourceGraph( + name="parent", + source="unit", + tasks=[_node("call_child", "ExecutePipeline")], + lineage=Lineage( + control_edges=[ControlEdge(source_workflow="parent", target_workflow="child", via_task_key="call_child")] + ), + ) + child = SourceGraph(name="child", source="unit", tasks=[_node("copy_orders", "Copy")]) + solo = SourceGraph(name="solo", source="unit", tasks=[_node("load", "Notebook")]) + return build_source_inventory([parent, child, solo], source="unit", source_dir="/tmp/src") + + +def _report() -> dict[str, Any]: + return { + "pipelines": [ + {"name": "parent", "tasks": [{"name": "call_child", "task_key": "call_child", "type": "CopyActivity"}]}, + {"name": "child", "tasks": [{"name": "copy_orders", "task_key": "copy_orders", "type": "CopyActivity"}]}, + { + "name": "solo", + "tasks": [ + {"name": "load", "task_key": "load", "type": "NotebookActivity", "notebook_path": "/x"}, + ], + }, + ] + } + + +def _setup(output_dir: Path) -> None: + metadata = output_dir / "metadata" + metadata.mkdir(parents=True, exist_ok=True) + (metadata / "inventory.json").write_text(json.dumps(_inventory(), indent=2), encoding="utf-8") + work = output_dir / WORK_DIRNAME + work.mkdir(parents=True, exist_ok=True) + (work / REPORT_FILENAME).write_text(json.dumps(_report(), indent=2), encoding="utf-8") + (work / GAPS_FILENAME).write_text("[]", encoding="utf-8") + + +def _agentic_plan() -> dict[str, Any]: + """Route the parent<->child component agentic; leave solo deterministic.""" + return { + "components": [ + {"component_id": "component-1", "members": ["child", "parent"], "decision": "agentic"}, + {"component_id": "component-2", "members": ["solo"], "decision": "deterministic"}, + ] + } + + +def test_route_without_a_decision_on_a_non_tty_emits_the_recommendation( + tmp_path: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch +) -> None: + _setup(tmp_path) + monkeypatch.setattr("sys.stdin", io.StringIO("")) # not a TTY + code = adapter_cli_main(["route", "--output-dir", str(tmp_path)]) + assert code == 0 + payload = json.loads(capsys.readouterr().out) + assert [c["component_id"] for c in payload["components"]] == ["component-1", "component-2"] + assert "default_plan" in payload + # No decision => report untouched. + report = json.loads((tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_text(encoding="utf-8")) + assert all(pipeline["tasks"][0]["type"] != "PlaceholderActivity" for pipeline in report["pipelines"]) + + +def test_route_with_plan_path_records_and_edits_the_report(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + _setup(tmp_path) + plan_path = tmp_path / "plan.json" + plan_path.write_text(json.dumps(_agentic_plan()), encoding="utf-8") + code = adapter_cli_main(["route", "--output-dir", str(tmp_path), "--plan-path", str(plan_path)]) + assert code == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["ok"] is True + assert payload["edit"]["agentic_pipelines"] == ["child", "parent"] + + # The recorded plan is written (fingerprint binding preserved). + assert (tmp_path / "metadata" / "conversion_plan.json").exists() + # The routed-agentic pipelines were placeholdered; the deterministic 'solo' is untouched. + report = json.loads((tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_text(encoding="utf-8")) + by_name = {pipeline["name"]: pipeline for pipeline in report["pipelines"]} + assert by_name["parent"]["tasks"][0]["type"] == "PlaceholderActivity" + assert by_name["child"]["tasks"][0]["type"] == "PlaceholderActivity" + assert by_name["solo"]["tasks"][0]["type"] == "NotebookActivity" + gaps = json.loads((tmp_path / WORK_DIRNAME / GAPS_FILENAME).read_text(encoding="utf-8")) + assert sorted(gap["pipeline"] for gap in gaps) == ["child", "parent"] + + +def test_route_reads_a_plan_from_stdin_when_plan_path_is_dash( + tmp_path: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch +) -> None: + _setup(tmp_path) + monkeypatch.setattr("sys.stdin", io.StringIO(json.dumps(_agentic_plan()))) + code = adapter_cli_main(["route", "--output-dir", str(tmp_path), "--plan-path", "-"]) + assert code == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["ok"] is True and payload["edit"]["agentic_pipelines"] == ["child", "parent"] + + +def test_route_interactive_prompt_records_the_users_decision( + tmp_path: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch +) -> None: + _setup(tmp_path) + + class _Tty(io.StringIO): + def isatty(self) -> bool: + return True + + monkeypatch.setattr("sys.stdin", _Tty("")) + answers = iter(["a", ""]) # component-1 -> agentic; component-2 -> accept recommendation (deterministic) + monkeypatch.setattr("builtins.input", lambda _prompt="": next(answers)) + + code = adapter_cli_main(["route", "--output-dir", str(tmp_path)]) + assert code == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["ok"] is True and payload["edit"]["agentic_pipelines"] == ["child", "parent"] + + +def test_route_validation_failure_returns_1_and_leaves_report_untouched( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + _setup(tmp_path) + report_before = (tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_bytes() + bad = _agentic_plan() + bad["components"].pop() # partial plan: component-2 undecided + plan_path = tmp_path / "plan.json" + plan_path.write_text(json.dumps(bad), encoding="utf-8") + code = adapter_cli_main(["route", "--output-dir", str(tmp_path), "--plan-path", str(plan_path)]) + assert code == 1 + payload = json.loads(capsys.readouterr().out) + assert payload["ok"] is False and payload["violations"] + assert (tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_bytes() == report_before + + +def test_route_with_a_missing_plan_file_fails_cleanly(tmp_path: Path) -> None: + _setup(tmp_path) + code = adapter_cli_main(["route", "--output-dir", str(tmp_path), "--plan-path", str(tmp_path / "nope.json")]) + assert code == 1 + + +def test_route_errors_when_report_missing_and_no_source_to_trigger_convert( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + metadata = tmp_path / "metadata" + metadata.mkdir(parents=True) + (metadata / "inventory.json").write_text(json.dumps(_inventory(), indent=2), encoding="utf-8") + plan_path = tmp_path / "plan.json" + plan_path.write_text(json.dumps(_agentic_plan()), encoding="utf-8") + code = adapter_cli_main(["route", "--output-dir", str(tmp_path), "--plan-path", str(plan_path)]) + assert code == 1 + + +# --------------------------------------------------------------------------- # +# fill-agentic combine. +# --------------------------------------------------------------------------- # + + +def _lfc_pipeline() -> dict[str, Any]: + definition = {"name": "orders_ingestion", "catalog": "${var.catalog}", "target": "${var.schema}"} + return { + "name": "orders_lfc", + "tasks": [ + { + "name": "Ingest orders", + "task_key": "ingest_orders", + "type": "AgenticComponentActivity", + "files": [], + "resources": [{"resource_key": "orders_ingestion", "definition": definition}], + "task": {"pipeline_task": {"pipeline_id": "${resources.pipelines.orders_ingestion.id}"}}, + } + ], + } + + +def _record_agentic_plan(tmp_path: Path) -> None: + """Route the parent<->child component agentic and record the plan so combine can bind to it.""" + plan_path = tmp_path / "route_plan.json" + plan_path.write_text(json.dumps(_agentic_plan()), encoding="utf-8") + assert adapter_cli_main(["route", "--output-dir", str(tmp_path), "--plan-path", str(plan_path)]) == 0 + + +def _run_combine(tmp_path: Path, members: str, authored: list[dict[str, Any]]) -> int: + pipelines_path = tmp_path / "authored.json" + pipelines_path.write_text(json.dumps(authored), encoding="utf-8") + return adapter_cli_main( + [ + "fill-agentic", + "combine", + "--output-dir", + str(tmp_path), + "--members", + members, + "--pipelines-path", + str(pipelines_path), + ] + ) + + +def test_fill_agentic_combine_writes_merged_report(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + _setup(tmp_path) + _record_agentic_plan(tmp_path) + capsys.readouterr() # drop the route-record output so only the combine JSON remains + code = _run_combine(tmp_path, "child,parent", [_lfc_pipeline()]) + assert code == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["ok"] is True + report = json.loads((tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_text(encoding="utf-8")) + names = sorted(pipeline["name"] for pipeline in report["pipelines"]) + assert names == ["orders_lfc", "solo"] + + +def test_fill_agentic_combine_rejects_dangling_reference(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + _setup(tmp_path) + _record_agentic_plan(tmp_path) + capsys.readouterr() # drop the route-record output so only the combine JSON remains + report_before = (tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_bytes() + dangling = _lfc_pipeline() + dangling["tasks"][0]["task"] = {"pipeline_task": {"pipeline_id": "${resources.pipelines.ghost.id}"}} + code = _run_combine(tmp_path, "child,parent", [dangling]) + assert code == 1 + payload = json.loads(capsys.readouterr().out) + assert payload["ok"] is False and payload["violations"] + assert (tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_bytes() == report_before + + +def test_fill_agentic_combine_rejects_a_deterministic_member_set( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + _setup(tmp_path) + _record_agentic_plan(tmp_path) # component-1 (child, parent) agentic; solo is deterministic + capsys.readouterr() # drop the route-record output so only the combine JSON remains + code = _run_combine(tmp_path, "solo", [_lfc_pipeline()]) + assert code == 1 + payload = json.loads(capsys.readouterr().out) + assert payload["ok"] is False and payload["error"] + + +def test_fill_agentic_has_no_no_validate_flag(tmp_path: Path) -> None: + # The validation bypass must not exist as a CLI surface; argparse rejects the unknown flag. + with pytest.raises(SystemExit) as excinfo: + adapter_cli_main( + [ + "fill-agentic", + "combine", + "--output-dir", + str(tmp_path), + "--members", + "child,parent", + "--pipelines-path", + str(tmp_path / "x.json"), + "--no-validate", + ] + ) + assert excinfo.value.code == 2 + + +def test_route_triggers_convert_when_report_absent( + tmp_path: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch +) -> None: + # Inventory present, report absent: with --source/--source-path, route triggers convert. Stub the + # phase runner to write the report the trigger would have produced, then assert route records+edits. + metadata = tmp_path / "metadata" + metadata.mkdir(parents=True) + (metadata / "inventory.json").write_text(json.dumps(_inventory(), indent=2), encoding="utf-8") + + triggered: list[list[str]] = [] + + def fake_run_phase(phase: str, forward: list[str]) -> int: + triggered.append([phase, *forward]) + work = tmp_path / WORK_DIRNAME + work.mkdir(parents=True, exist_ok=True) + (work / REPORT_FILENAME).write_text(json.dumps(_report(), indent=2), encoding="utf-8") + return 0 + + monkeypatch.setattr("flowx.adapter.__main__._run_phase", fake_run_phase) + plan_path = tmp_path / "plan.json" + plan_path.write_text(json.dumps(_agentic_plan()), encoding="utf-8") + code = adapter_cli_main( + [ + "route", + "--output-dir", + str(tmp_path), + "--plan-path", + str(plan_path), + "--source", + "adf", + "--source-path", + str(tmp_path / "adf_src"), + ] + ) + assert code == 0 + assert triggered and triggered[0][0] == "convert" + payload = json.loads(capsys.readouterr().out) + assert payload["ok"] is True and payload["edit"]["agentic_pipelines"] == ["child", "parent"] diff --git a/tests/unit/test_mcp_route.py b/tests/unit/test_mcp_route.py new file mode 100644 index 00000000..1dfd663a --- /dev/null +++ b/tests/unit/test_mcp_route.py @@ -0,0 +1,157 @@ +"""Tests that the reshaped MCP ``route`` / ``fill_agentic`` commands forward to the adapter CLI. + +``route`` is one command: no plan => the adapter emits the recommendation; a plan (inline or path) +=> the adapter records + edits via ``--plan-path``. ``fill_agentic`` performs the cross-pipeline +combine fill. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +pytest.importorskip("mcp") + +from flowx.mcp import runner, server # noqa: E402 + + +class _Result: + ok = True + returncode = 0 + + def __init__(self, stdout: str) -> None: + self.stdout = stdout + self.stderr = "" + + def as_dict(self) -> dict[str, Any]: + return {"returncode": 0, "stdout": self.stdout, "stderr": ""} + + +@pytest.fixture +def captured(monkeypatch): + calls: list[list[str]] = [] + + def fake_run_adapter(args, **_kwargs): + calls.append([str(a) for a in args]) + if args and args[0] == "route" and "--plan-path" not in [str(a) for a in args]: + return _Result(json.dumps({"components": [{"component_id": "component-1"}], "default_plan": {}})) + return _Result(json.dumps({"ok": True, "violations": [], "components": 1, "edit": {"agentic_pipelines": []}})) + + monkeypatch.setattr(runner, "run_adapter", fake_run_adapter) + return calls + + +def test_route_without_a_plan_emits_the_recommendation(captured, tmp_path: Path) -> None: + out = server._cmd_route({"output_dir": str(tmp_path)}) + assert out["ok"] is True + assert out["result"]["components"][0]["component_id"] == "component-1" + argv = captured[0] + assert argv[0] == "route" and "--plan-path" not in argv + assert "--output-dir" in argv + + +def test_route_inline_plan_is_staged_and_forwarded_as_plan_path(captured, tmp_path: Path) -> None: + plan = {"components": [{"component_id": "component-1", "members": ["a"], "decision": "agentic"}]} + out = server._cmd_route({"output_dir": str(tmp_path), "plan": plan}) + assert out["ok"] is True + argv = captured[0] + assert argv[0] == "route" + assert "--plan-path" in argv # inline plan staged to a real temp file + + +def test_route_forwards_an_explicit_plan_path(captured, tmp_path: Path) -> None: + plan_path = tmp_path / "plan.json" + plan_path.write_text("{}", encoding="utf-8") + server._cmd_route({"output_dir": str(tmp_path), "plan_path": str(plan_path)}) + argv = captured[0] + assert argv[argv.index("--plan-path") + 1] == str(plan_path) + + +def test_route_rejects_both_plan_and_plan_path(tmp_path: Path) -> None: + out = server._cmd_route({"output_dir": str(tmp_path), "plan": {}, "plan_path": "x.json"}) + assert out["ok"] is False and "at most one" in out["error"] + + +def test_route_forwards_source_so_convert_can_be_triggered(captured, tmp_path: Path) -> None: + # Parity with the CLI: MCP route forwards source + source-path so the CLI can trigger convert when + # the report is absent. Recommend path (no plan). + server._cmd_route({"output_dir": str(tmp_path), "source": "adf", "adf_source_path": "/tmp/adf"}) + argv = captured[0] + assert argv[argv.index("--source") + 1] == "adf" + assert argv[argv.index("--source-path") + 1] == "/tmp/adf" + + +def test_route_forwards_source_on_the_record_path(captured, tmp_path: Path) -> None: + plan = {"components": [{"component_id": "component-1", "members": ["a"], "decision": "agentic"}]} + server._cmd_route({"output_dir": str(tmp_path), "plan": plan, "source": "adf", "adf_source_path": "/tmp/adf"}) + argv = captured[0] + assert "--plan-path" in argv + assert argv[argv.index("--source") + 1] == "adf" + assert argv[argv.index("--source-path") + 1] == "/tmp/adf" + + +def test_route_without_source_forwards_no_source_flags(captured, tmp_path: Path) -> None: + server._cmd_route({"output_dir": str(tmp_path)}) + assert "--source" not in captured[0] + + +def test_route_registered_in_command_map() -> None: + assert "route" in server._COMMANDS + + +# --------------------------------------------------------------------------- # +# fill_agentic (combine). +# --------------------------------------------------------------------------- # + + +@pytest.fixture +def captured_fill(monkeypatch): + calls: list[list[str]] = [] + + def fake_run_adapter(args, **_kwargs): + calls.append([str(a) for a in args]) + return _Result(json.dumps({"ok": True, "violations": [], "pipelines": 1})) + + monkeypatch.setattr(runner, "run_adapter", fake_run_adapter) + return calls + + +def test_fill_agentic_inline_pipelines_are_staged(captured_fill, tmp_path: Path) -> None: + out = server._cmd_fill_agentic( + {"output_dir": str(tmp_path), "members": ["parent", "child"], "pipelines": [{"name": "lfc", "tasks": []}]} + ) + assert out["ok"] is True + argv = captured_fill[0] + assert argv[0] == "fill-agentic" and argv[1] == "combine" + assert argv[argv.index("--members") + 1] == "parent,child" + assert "--pipelines-path" in argv + + +def test_fill_agentic_forwards_members_string_and_path(captured_fill, tmp_path: Path) -> None: + pipelines_path = tmp_path / "authored.json" + pipelines_path.write_text("[]", encoding="utf-8") + server._cmd_fill_agentic({"output_dir": str(tmp_path), "members": "a,b", "pipelines_path": str(pipelines_path)}) + argv = captured_fill[0] + assert argv[argv.index("--members") + 1] == "a,b" + assert argv[argv.index("--pipelines-path") + 1] == str(pipelines_path) + + +def test_fill_agentic_requires_members(tmp_path: Path) -> None: + out = server._cmd_fill_agentic({"output_dir": str(tmp_path), "pipelines": []}) + assert out["ok"] is False and "members" in out["error"] + + +def test_fill_agentic_requires_exactly_one_pipelines_source(tmp_path: Path) -> None: + both = server._cmd_fill_agentic( + {"output_dir": str(tmp_path), "members": ["a"], "pipelines": [], "pipelines_path": "x.json"} + ) + neither = server._cmd_fill_agentic({"output_dir": str(tmp_path), "members": ["a"]}) + assert both["ok"] is False and "exactly one" in both["error"] + assert neither["ok"] is False and "exactly one" in neither["error"] + + +def test_fill_agentic_registered_in_command_map() -> None: + assert "fill_agentic" in server._COMMANDS diff --git a/tests/unit/test_route_agentic_flow.py b/tests/unit/test_route_agentic_flow.py new file mode 100644 index 00000000..29f9ea09 --- /dev/null +++ b/tests/unit/test_route_agentic_flow.py @@ -0,0 +1,524 @@ +"""Tests for the Phase-1 in-engine agentic conversion flow (route -> alter -> fill). + +The confirmed flow, built here: + +* ``route`` groups pipelines by connected component (reusing :mod:`flowx.routing`), takes a + per-component deterministic/agentic decision, records the fingerprint-bound conversion plan, and + then **edits** the deterministic ``translation_report.json``: for every pipeline in an + agentic-routed component it removes the deterministic tasks, replaces them with + ``PlaceholderActivity`` entries, and appends one ``AgenticGap`` per task to ``gaps.json``. Pipelines + in deterministic components are left byte-identical, and with no agentic decision the report and + gaps are untouched (the non-breaking guarantee). +* the agent then authors the fill. **Per-pipeline** agentic reuses the existing name-matched + :func:`flowx.ir_serde.merge_agentic_results`. **Cross-pipeline COMBINE** (N pipelines -> M, e.g. one + Lakeflow Connect pipeline) uses a new pipeline-grain fill that swaps the routed group's pipelines + for the agent-authored pipeline(s), which carry ``AgenticComponentActivity`` nodes. +* the merged report is validated structurally via the existing + :func:`flowx.validate.bundle_invariants.check_bundle_dir` (unique keys, no dangling deps, acyclic, + no dangling pipeline/run_job references). +""" + +from __future__ import annotations + +import copy +import inspect +import json +from pathlib import Path +from typing import Any + +from flowx.discovery_inventory import STRATEGY_PROPERTY, build_source_inventory +from flowx.ir_serde import merge_agentic_results +from flowx.models.discovery import CONCEPT_NOTEBOOK, SourceGraph, SourceNode +from flowx.models.ir import ControlEdge, Lineage +from flowx.route_agentic import ( + GAPS_FILENAME, + REPORT_FILENAME, + WORK_DIRNAME, + agentic_pipeline_names, + alter_report, + apply_combine_fill, + apply_plan_to_report, + combine_group_fill, + prompt_for_decisions, + validate_report_structurally, +) +from flowx.routing import record_plan + +# --------------------------------------------------------------------------- # +# Fixtures. +# --------------------------------------------------------------------------- # + + +def _notebook_task(name: str, task_key: str, path: str = "/Workspace/Shared/x") -> dict[str, Any]: + return {"name": name, "task_key": task_key, "type": "NotebookActivity", "notebook_path": path} + + +def _copy_task(name: str, task_key: str) -> dict[str, Any]: + return {"name": name, "task_key": task_key, "type": "CopyActivity"} + + +def _report_two_pipelines() -> dict[str, Any]: + """A 'parent' pipeline (one Copy) and a 'child' pipeline (one Notebook).""" + return { + "pipelines": [ + {"name": "parent", "tasks": [_copy_task("Extract", "extract")]}, + {"name": "child", "tasks": [_notebook_task("Load", "load")]}, + ] + } + + +def _plan(*, parent: str = "agentic", child: str = "deterministic") -> dict[str, Any]: + """An authored/recorded plan: 'parent' and 'child' each their own component.""" + return { + "components": [ + {"component_id": "component-1", "members": ["parent"], "decision": parent}, + {"component_id": "component-2", "members": ["child"], "decision": child}, + ] + } + + +def _recommendation() -> dict[str, Any]: + return { + "components": [ + { + "component_id": "component-1", + "members": ["parent"], + "recommended": "deterministic", + "options": {"deterministic": {"capable": True}, "agentic": {"has_simplification": True}}, + }, + { + "component_id": "component-2", + "members": ["child"], + "recommended": "agentic", + "options": {"deterministic": {"capable": False}, "agentic": {"has_simplification": False}}, + }, + ], + "findings": [], + "default_plan": {"components": []}, + } + + +def _lfc_pipeline() -> dict[str, Any]: + """One agent-authored Lakeflow Connect pipeline via the AgenticComponentActivity escape hatch.""" + pipeline_definition = { + "name": "orders_ingestion", + "catalog": "${var.catalog}", + "target": "${var.schema}", + "ingestion_definition": {"connection_name": "c", "objects": []}, + } + return { + "name": "orders_lfc", + "tasks": [ + { + "name": "Ingest orders", + "task_key": "ingest_orders", + "type": "AgenticComponentActivity", + "files": [], + "resources": [{"resource_key": "orders_ingestion", "definition": pipeline_definition}], + "task": {"pipeline_task": {"pipeline_id": "${resources.pipelines.orders_ingestion.id}"}}, + } + ], + } + + +def _write_work(output_dir: Path, report: dict[str, Any], gaps: list[dict[str, Any]] | None = None) -> None: + work = output_dir / WORK_DIRNAME + work.mkdir(parents=True, exist_ok=True) + (work / REPORT_FILENAME).write_text(json.dumps(report, indent=2), encoding="utf-8") + if gaps is not None: + (work / GAPS_FILENAME).write_text(json.dumps(gaps, indent=2), encoding="utf-8") + + +def _node(task_key: str, native_type: str) -> SourceNode: + return SourceNode( + source_id=task_key, + task_key=task_key, + concept=CONCEPT_NOTEBOOK, + source="unit", + name=task_key, + native_type=native_type, + properties={STRATEGY_PROPERTY: "deterministic"}, + raw={"name": task_key, "type": native_type}, + ) + + +def _routed_inventory() -> dict[str, Any]: + """Inventory whose single 'parent -> child' control edge forms one component {child, parent}.""" + parent = SourceGraph( + name="parent", + source="unit", + tasks=[_node("call_child", "ExecutePipeline")], + lineage=Lineage( + control_edges=[ControlEdge(source_workflow="parent", target_workflow="child", via_task_key="call_child")] + ), + ) + child = SourceGraph(name="child", source="unit", tasks=[_node("copy_orders", "Copy")]) + return build_source_inventory([parent, child], source="unit", source_dir="/tmp/src") + + +def _setup_routed_agentic(output_dir: Path, *, decision: str = "agentic") -> None: + """Write inventory + report + a recorded conversion_plan.json routing {child, parent} per ``decision``.""" + _write_work(output_dir, _report_two_pipelines()) + metadata = output_dir / "metadata" + metadata.mkdir(parents=True, exist_ok=True) + (metadata / "inventory.json").write_text(json.dumps(_routed_inventory(), indent=2), encoding="utf-8") + plan = {"components": [{"component_id": "component-1", "members": ["child", "parent"], "decision": decision}]} + result = record_plan(output_dir, plan=plan) + assert result["ok"], result + + +# --------------------------------------------------------------------------- # +# Decision selection. +# --------------------------------------------------------------------------- # + + +def test_agentic_pipeline_names_selects_only_agentic_component_members() -> None: + plan = { + "components": [ + {"component_id": "component-1", "members": ["parent", "shared"], "decision": "agentic"}, + {"component_id": "component-2", "members": ["child"], "decision": "deterministic"}, + ] + } + assert agentic_pipeline_names(plan) == {"parent", "shared"} + + +def test_prompt_for_decisions_defaults_to_recommendation_and_honours_overrides() -> None: + answers = iter(["", "d"]) # component-1: accept (deterministic); component-2: override to deterministic + lines: list[str] = [] + plan = prompt_for_decisions(_recommendation(), input_fn=lambda _prompt: next(answers), output_fn=lines.append) + decisions = {c["component_id"]: c["decision"] for c in plan["components"]} + assert decisions == {"component-1": "deterministic", "component-2": "deterministic"} + # Each component carries its members so the plan validates against the inventory on record. + assert plan["components"][0]["members"] == ["parent"] + # The user saw the members and the prominent simplification option before answering. + assert any("component-1" in line for line in lines) + assert any("simplification" in line.lower() for line in lines) + + +def test_prompt_for_decisions_selects_agentic_on_a() -> None: + plan = prompt_for_decisions(_recommendation(), input_fn=lambda _prompt: "a", output_fn=lambda _line: None) + assert [c["decision"] for c in plan["components"]] == ["agentic", "agentic"] + + +# --------------------------------------------------------------------------- # +# Report alteration (the edit step). +# --------------------------------------------------------------------------- # + + +def test_alter_report_with_no_agentic_pipelines_is_identity() -> None: + report = _report_two_pipelines() + gaps: list[dict[str, Any]] = [] + original = copy.deepcopy(report) + new_report, new_gaps = alter_report(report, gaps, set()) + assert new_report == original + assert new_gaps == [] + + +def test_alter_report_placeholders_agentic_pipeline_and_leaves_deterministic_untouched() -> None: + report = _report_two_pipelines() + child_before = copy.deepcopy(report["pipelines"][1]) + new_report, new_gaps = alter_report(report, [], {"parent"}) + + parent = next(p for p in new_report["pipelines"] if p["name"] == "parent") + child = next(p for p in new_report["pipelines"] if p["name"] == "child") + # The agentic pipeline's deterministic Copy task became a placeholder marking the gap. + assert [task["type"] for task in parent["tasks"]] == ["PlaceholderActivity"] + placeholder = parent["tasks"][0] + assert placeholder["name"] == "Extract" and placeholder["task_key"] == "extract" + assert placeholder["original_type"] == "CopyActivity" + # The deterministic pipeline is byte-identical. + assert child == child_before + + +def test_alter_report_emits_one_gap_per_removed_task_tagged_with_pipeline() -> None: + _report, gaps = alter_report(_report_two_pipelines(), [], {"parent"}) + assert len(gaps) == 1 + gap = gaps[0] + assert gap["activity_name"] == "Extract" + assert gap["activity_type"] == "CopyActivity" + assert gap["pipeline"] == "parent" + + +def test_alter_report_emits_exactly_one_gap_per_task_replacing_prior_gaps() -> None: + report = { + "pipelines": [ + { + "name": "parent", + "tasks": [_copy_task("Extract", "extract"), _copy_task("Flow", "flow")], + } + ] + } + # A pre-existing (untagged) convert gap for a task that will be routed must not be duplicated. + existing_gaps = [{"activity_name": "Flow", "activity_type": "ExecuteDataFlow", "raw_definition": None}] + _report, gaps = alter_report(report, existing_gaps, {"parent"}) + # One gap per routed task, no duplicates, all tagged with the pipeline. + assert len(gaps) == 2 + identities = sorted((gap["pipeline"], gap["activity_name"]) for gap in gaps) + assert identities == [("parent", "Extract"), ("parent", "Flow")] + assert len(identities) == len(set(identities)) + + +def test_alter_report_is_idempotent_on_gaps() -> None: + report = {"pipelines": [{"name": "parent", "tasks": [_copy_task("Extract", "extract")]}]} + once_report, once_gaps = alter_report(report, [], {"parent"}) + twice_report, twice_gaps = alter_report(once_report, once_gaps, {"parent"}) + # Running the edit again does not append a second gap for the same routed task. + assert twice_gaps == once_gaps + assert len(twice_gaps) == 1 + + +def test_alter_report_keeps_a_non_routed_gap_that_shares_a_task_name_with_a_routed_task() -> None: + # Both pipelines have a task named "Sync"; only 'parent' is routed agentic. The untagged convert + # gap belongs to the non-routed 'other' and must survive -- the drop must not match by global name. + report = { + "pipelines": [ + {"name": "parent", "tasks": [_copy_task("Sync", "sync_parent")]}, + {"name": "other", "tasks": [_copy_task("Sync", "sync_other")]}, + ] + } + existing = [{"activity_name": "Sync", "activity_type": "ExecuteDataFlow", "raw_definition": None}] + _report, gaps = alter_report(report, existing, {"parent"}) + + # The non-routed pipeline's untagged "Sync" gap survives. + assert existing[0] in gaps + # The routed pipeline contributes exactly one tagged gap for its own "Sync" task. + tagged = [gap for gap in gaps if gap.get("pipeline") == "parent"] + assert tagged == [ + { + "activity_name": "Sync", + "activity_type": "CopyActivity", + "raw_definition": tagged[0]["raw_definition"], + "pipeline": "parent", + } + ] + # No duplicate tagged gap for the routed task. + assert len(tagged) == 1 + + +def test_alter_report_keeps_gaps_for_non_routed_pipelines() -> None: + report = { + "pipelines": [ + {"name": "parent", "tasks": [_copy_task("Extract", "extract")]}, + {"name": "other", "tasks": [_copy_task("Keep", "keep")]}, + ] + } + existing = [{"activity_name": "OtherGap", "activity_type": "Custom", "raw_definition": None, "pipeline": "other"}] + _report, gaps = alter_report(report, existing, {"parent"}) + # The non-routed pipeline's gap survives; the routed pipeline contributes exactly one. + assert {gap["activity_name"] for gap in gaps} == {"OtherGap", "Extract"} + + +def test_alter_report_preserves_depends_on_edges_on_placeholders() -> None: + report = { + "pipelines": [ + { + "name": "parent", + "tasks": [ + _copy_task("A", "a"), + {**_copy_task("B", "b"), "depends_on": [{"task_key": "a", "outcome": "Succeeded"}]}, + ], + } + ] + } + new_report, _gaps = alter_report(report, [], {"parent"}) + task_b = new_report["pipelines"][0]["tasks"][1] + assert task_b["type"] == "PlaceholderActivity" + assert task_b["depends_on"] == [{"task_key": "a", "outcome": "Succeeded"}] + + +def test_alter_report_handles_the_single_pipeline_report_shape() -> None: + report = {"name": "parent", "tasks": [_copy_task("Extract", "extract")]} + new_report, gaps = alter_report(report, [], {"parent"}) + assert new_report["tasks"][0]["type"] == "PlaceholderActivity" + assert len(gaps) == 1 + + +# --------------------------------------------------------------------------- # +# apply_plan_to_report: I/O + non-breaking golden. +# --------------------------------------------------------------------------- # + + +def test_apply_plan_to_report_edits_report_and_gaps_files(tmp_path: Path) -> None: + _write_work(tmp_path, _report_two_pipelines(), gaps=[]) + summary = apply_plan_to_report(tmp_path, _plan(parent="agentic", child="deterministic")) + assert summary["agentic_pipelines"] == ["parent"] + + report = json.loads((tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_text(encoding="utf-8")) + parent = next(p for p in report["pipelines"] if p["name"] == "parent") + assert parent["tasks"][0]["type"] == "PlaceholderActivity" + gaps = json.loads((tmp_path / WORK_DIRNAME / GAPS_FILENAME).read_text(encoding="utf-8")) + assert [gap["pipeline"] for gap in gaps] == ["parent"] + + +def test_apply_plan_to_report_all_deterministic_leaves_files_byte_identical(tmp_path: Path) -> None: + _write_work(tmp_path, _report_two_pipelines(), gaps=[]) + report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME + gaps_path = tmp_path / WORK_DIRNAME / GAPS_FILENAME + report_before = report_path.read_bytes() + gaps_before = gaps_path.read_bytes() + + summary = apply_plan_to_report(tmp_path, _plan(parent="deterministic", child="deterministic")) + assert summary["agentic_pipelines"] == [] + assert report_path.read_bytes() == report_before + assert gaps_path.read_bytes() == gaps_before + + +# --------------------------------------------------------------------------- # +# Per-pipeline agentic fill: reuse the existing name-matched merge. +# --------------------------------------------------------------------------- # + + +def test_per_pipeline_fill_reuses_merge_agentic_results(tmp_path: Path) -> None: + _write_work(tmp_path, _report_two_pipelines(), gaps=[]) + apply_plan_to_report(tmp_path, _plan(parent="agentic", child="deterministic")) + report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME + + results_dir = tmp_path / "agentic_results" + results_dir.mkdir() + (results_dir / "extract.json").write_text( + json.dumps( + { + "pipeline": "parent", + "activity_name": "Extract", + "task": { + "type": "NotebookActivity", + "name": "Extract", + "task_key": "extract", + "notebook_path": "/Workspace/Shared/agentic_extract", + }, + } + ), + encoding="utf-8", + ) + merged, unmatched = merge_agentic_results(report_path, results_dir) + assert (merged, unmatched) == (1, 0) + + report = json.loads(report_path.read_text(encoding="utf-8")) + parent = next(p for p in report["pipelines"] if p["name"] == "parent") + assert parent["tasks"][0]["type"] == "NotebookActivity" + assert validate_report_structurally(report).ok + + +# --------------------------------------------------------------------------- # +# Cross-pipeline COMBINE: the new pipeline-grain fill (N pipelines -> one LFC). +# --------------------------------------------------------------------------- # + + +def test_combine_group_fill_replaces_group_pipelines_with_authored_ones() -> None: + report = _report_two_pipelines() + merged = combine_group_fill(report, {"parent", "child"}, [_lfc_pipeline()]) + names = [pipeline["name"] for pipeline in merged["pipelines"]] + assert names == ["orders_lfc"] + + +def test_combine_group_fill_keeps_pipelines_outside_the_group() -> None: + report = { + "pipelines": [ + {"name": "parent", "tasks": [_copy_task("Extract", "extract")]}, + {"name": "child", "tasks": [_notebook_task("Load", "load")]}, + {"name": "unrelated", "tasks": [_notebook_task("Keep", "keep")]}, + ] + } + merged = combine_group_fill(report, {"parent", "child"}, [_lfc_pipeline()]) + names = sorted(pipeline["name"] for pipeline in merged["pipelines"]) + assert names == ["orders_lfc", "unrelated"] + + +def test_combine_fill_of_two_pipelines_into_one_lfc_passes_structural_validation() -> None: + report = _report_two_pipelines() + merged = combine_group_fill(report, {"parent", "child"}, [_lfc_pipeline()]) + result = validate_report_structurally(merged) + assert result.ok, [f"{finding.code}: {finding.message}" for finding in result.findings] + + +def test_combine_fill_with_a_dangling_pipeline_reference_is_caught() -> None: + dangling = _lfc_pipeline() + # Point the pipeline_task at a resource that is never declared in this activity's resources list. + dangling["tasks"][0]["task"] = {"pipeline_task": {"pipeline_id": "${resources.pipelines.ghost.id}"}} + merged = combine_group_fill(_report_two_pipelines(), {"parent", "child"}, [dangling]) + result = validate_report_structurally(merged) + assert not result.ok + assert any(finding.code == "dangling_pipeline_reference" for finding in result.violations) + + +def test_apply_combine_fill_writes_merged_report_when_valid(tmp_path: Path) -> None: + _setup_routed_agentic(tmp_path, decision="agentic") + result = apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()]) + assert result["ok"] is True + assert result["component_id"] == "component-1" + report = json.loads((tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_text(encoding="utf-8")) + assert [pipeline["name"] for pipeline in report["pipelines"]] == ["orders_lfc"] + + +def test_apply_combine_fill_rejects_and_does_not_write_on_dangling_reference(tmp_path: Path) -> None: + _setup_routed_agentic(tmp_path, decision="agentic") + report_before = (tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_bytes() + dangling = _lfc_pipeline() + dangling["tasks"][0]["task"] = {"pipeline_task": {"pipeline_id": "${resources.pipelines.ghost.id}"}} + + result = apply_combine_fill(tmp_path, ["parent", "child"], [dangling]) + assert result["ok"] is False + assert any("dangling_pipeline_reference" in violation for violation in result["violations"]) + # The report on disk is untouched when validation fails. + assert (tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_bytes() == report_before + + +# --------------------------------------------------------------------------- # +# Combine membership is bound to the recorded, fingerprint-bound plan. +# --------------------------------------------------------------------------- # + + +def test_combine_rejects_a_deterministic_component(tmp_path: Path) -> None: + _setup_routed_agentic(tmp_path, decision="deterministic") + result = apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()]) + assert result["ok"] is False + assert "not agentic" in result["error"] + # A deterministic component is never swapped out. + report = json.loads((tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_text(encoding="utf-8")) + assert sorted(pipeline["name"] for pipeline in report["pipelines"]) == ["child", "parent"] + + +def test_combine_rejects_a_partial_member_set(tmp_path: Path) -> None: + _setup_routed_agentic(tmp_path, decision="agentic") + result = apply_combine_fill(tmp_path, ["parent"], [_lfc_pipeline()]) + assert result["ok"] is False + assert "do not exactly match" in result["error"] + + +def test_combine_rejects_a_superset_member_set(tmp_path: Path) -> None: + _setup_routed_agentic(tmp_path, decision="agentic") + result = apply_combine_fill(tmp_path, ["parent", "child", "extra"], [_lfc_pipeline()]) + assert result["ok"] is False + assert "do not exactly match" in result["error"] + + +def test_combine_rejects_a_typoed_member(tmp_path: Path) -> None: + _setup_routed_agentic(tmp_path, decision="agentic") + result = apply_combine_fill(tmp_path, ["parent", "chld"], [_lfc_pipeline()]) + assert result["ok"] is False + assert "do not exactly match" in result["error"] + + +def test_combine_rejects_a_stale_fingerprint_plan(tmp_path: Path) -> None: + _setup_routed_agentic(tmp_path, decision="agentic") + # Mutate the inventory after recording so the recorded fingerprint no longer matches. + inventory_path = tmp_path / "metadata" / "inventory.json" + inventory = json.loads(inventory_path.read_text(encoding="utf-8")) + inventory["pipelines"].append({"name": "late_addition", "activities": [], "motifs": []}) + inventory_path.write_text(json.dumps(inventory, indent=2), encoding="utf-8") + + result = apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()]) + assert result["ok"] is False + assert "stale" in result["error"] + + +def test_combine_requires_a_recorded_plan(tmp_path: Path) -> None: + _write_work(tmp_path, _report_two_pipelines()) # report only; no plan recorded + result = apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()]) + assert result["ok"] is False + assert "conversion_plan.json" in result["error"] + + +def test_apply_combine_fill_has_no_validation_bypass() -> None: + # There must be no surface that writes the combined report without structural validation. + assert "validate" not in inspect.signature(apply_combine_fill).parameters diff --git a/tests/unit/test_routing_components.py b/tests/unit/test_routing_components.py new file mode 100644 index 00000000..d5b458e2 --- /dev/null +++ b/tests/unit/test_routing_components.py @@ -0,0 +1,152 @@ +"""Tests for connected-component computation over control lineage (routing, #77). + +The inventory fixtures are built by hand through the source-agnostic emitter +(:func:`flowx.discovery_inventory.build_source_inventory`) -- no ADF, no Airflow -- so the +routing engine is proven against the standardised inventory shape and its per-pipeline +control-edge lineage, exactly as it will see it in production. Components are weak/undirected: +two pipelines land in the same component when a resolved control edge joins them in either +direction. +""" + +from __future__ import annotations + +from typing import Any + +from flowx.discovery_inventory import STRATEGY_PROPERTY, build_source_inventory +from flowx.models.discovery import CONCEPT_NOTEBOOK, SourceGraph, SourceNode +from flowx.models.ir import ControlEdge, Lineage +from flowx.routing import build_components + + +def _node(task_key: str, native_type: str = "Notebook", *, strategy: str = "deterministic") -> SourceNode: + return SourceNode( + source_id=task_key, + task_key=task_key, + concept=CONCEPT_NOTEBOOK, + source="unit", + name=task_key, + native_type=native_type, + properties={STRATEGY_PROPERTY: strategy}, + raw={"name": task_key, "type": native_type}, + ) + + +def _graph(name: str, *, edges: list[ControlEdge] | None = None, nodes: list[SourceNode] | None = None) -> SourceGraph: + """A one-node graph named *name*, optionally carrying control edges to other graphs.""" + return SourceGraph( + name=name, + source="unit", + tasks=nodes if nodes is not None else [_node(f"{name}_task")], + lineage=Lineage(control_edges=edges or []), + ) + + +def _edge(source: str, target: str, via: str, *, resolved: bool = True) -> ControlEdge: + return ControlEdge(source_workflow=source, target_workflow=target, via_task_key=via, resolved=resolved) + + +def _inventory(graphs: list[SourceGraph]) -> dict[str, Any]: + return build_source_inventory(graphs, source="unit", source_dir="/tmp/src") + + +def test_isolated_pipelines_each_form_their_own_component() -> None: + inventory = _inventory([_graph("a"), _graph("b"), _graph("c")]) + components, findings = build_components(inventory) + assert components == [["a"], ["b"], ["c"]] + assert findings == [] + + +def test_a_chain_of_calls_forms_one_component() -> None: + graphs = [ + _graph("a", edges=[_edge("a", "b", "call_b")], nodes=[_node("call_b", "ExecutePipeline")]), + _graph("b", edges=[_edge("b", "c", "call_c")], nodes=[_node("call_c", "ExecutePipeline")]), + _graph("c"), + ] + components, findings = build_components(_inventory(graphs)) + assert components == [["a", "b", "c"]] + assert findings == [] + + +def test_a_branch_fans_into_one_component() -> None: + graphs = [ + _graph( + "a", + edges=[_edge("a", "b", "call_b"), _edge("a", "c", "call_c")], + nodes=[_node("call_b", "ExecutePipeline"), _node("call_c", "ExecutePipeline")], + ), + _graph("b"), + _graph("c"), + ] + components, _ = build_components(_inventory(graphs)) + assert components == [["a", "b", "c"]] + + +def test_a_cycle_forms_one_component() -> None: + graphs = [ + _graph("a", edges=[_edge("a", "b", "call_b")], nodes=[_node("call_b", "ExecutePipeline")]), + _graph("b", edges=[_edge("b", "a", "call_a")], nodes=[_node("call_a", "ExecutePipeline")]), + ] + components, findings = build_components(_inventory(graphs)) + assert components == [["a", "b"]] + assert findings == [] + + +def test_multiple_edges_between_the_same_pair_still_one_component() -> None: + graphs = [ + _graph( + "a", + edges=[_edge("a", "b", "call_b1"), _edge("a", "b", "call_b2")], + nodes=[_node("call_b1", "ExecutePipeline"), _node("call_b2", "ExecutePipeline")], + ), + _graph("b"), + ] + components, _ = build_components(_inventory(graphs)) + assert components == [["a", "b"]] + + +def test_two_separate_clusters_are_distinct_components() -> None: + graphs = [ + _graph("a", edges=[_edge("a", "b", "call_b")], nodes=[_node("call_b", "ExecutePipeline")]), + _graph("b"), + _graph("x", edges=[_edge("x", "y", "call_y")], nodes=[_node("call_y", "ExecutePipeline")]), + _graph("y"), + ] + components, _ = build_components(_inventory(graphs)) + assert components == [["a", "b"], ["x", "y"]] + + +def test_unresolved_callee_is_a_finding_not_a_severed_edge() -> None: + graphs = [ + _graph( + "a", edges=[_edge("a", "", "call_ghost", resolved=False)], nodes=[_node("call_ghost", "ExecutePipeline")] + ) + ] + components, findings = build_components(_inventory(graphs)) + # 'a' still forms its own component; the unresolved edge is recorded, not dropped. + assert components == [["a"]] + assert len(findings) == 1 + assert "call_ghost" in findings[0] and "unresolved" in findings[0].lower() + + +def test_edge_to_pipeline_absent_from_inventory_is_a_finding() -> None: + graphs = [ + _graph("a", edges=[_edge("a", "ghost", "call_ghost")], nodes=[_node("call_ghost", "ExecutePipeline")]), + _graph("b"), + ] + components, findings = build_components(_inventory(graphs)) + assert components == [["a"], ["b"]] + assert len(findings) == 1 + assert "ghost" in findings[0] + + +def test_component_membership_and_ordering_are_deterministic() -> None: + # Declared out of alphabetical order; members and components must still sort deterministically. + graphs = [ + _graph("z"), + _graph("m", edges=[_edge("m", "a", "call_a")], nodes=[_node("call_a", "ExecutePipeline")]), + _graph("a"), + ] + first, _ = build_components(_inventory(graphs)) + second, _ = build_components(_inventory(graphs)) + assert first == second + assert first == [["a", "m"], ["z"]] diff --git a/tests/unit/test_routing_recommend.py b/tests/unit/test_routing_recommend.py new file mode 100644 index 00000000..43369b7d --- /dev/null +++ b/tests/unit/test_routing_recommend.py @@ -0,0 +1,206 @@ +"""Tests for the per-component conversion recommendation (routing, #77). + +Each component surfaces BOTH options as first-class peers: a deterministic option (engine-capability +assessment + motif/coverage evidence + uncovered gaps) and an agentic option (recommended Databricks +patterns from the insights block, with any simplification pattern surfaced prominently). The +``recommended`` field is a library-computed starting suggestion -- deterministic only when the whole +component is engine-capable. +""" + +from __future__ import annotations + +from typing import Any + +from flowx.discovery_inventory import STRATEGY_PROPERTY, build_source_inventory +from flowx.models.discovery import CONCEPT_NOTEBOOK, SourceGraph, SourceNode +from flowx.models.ir import ControlEdge, Lineage +from flowx.models.motifs import MOTIF_METADATA_DRIVEN_BULK_COPY, DetectedMotif +from flowx.routing import build_recommendation, recommend_component + + +def _node(task_key: str, native_type: str, *, strategy: str = "deterministic") -> SourceNode: + return SourceNode( + source_id=task_key, + task_key=task_key, + concept=CONCEPT_NOTEBOOK, + source="unit", + name=task_key, + native_type=native_type, + properties={STRATEGY_PROPERTY: strategy}, + raw={"name": task_key, "type": native_type}, + ) + + +def _inventory( + graphs: list[SourceGraph], + *, + motifs: dict[str, list[DetectedMotif]] | None = None, + insights: dict[str, Any] | None = None, +) -> dict[str, Any]: + inventory = build_source_inventory(graphs, source="unit", source_dir="/tmp/src", motifs_by_pipeline=motifs) + if insights is not None: + inventory["insights"] = insights + return inventory + + +def test_all_deterministic_component_recommends_deterministic() -> None: + graphs = [SourceGraph(name="a", source="unit", tasks=[_node("load", "Notebook"), _node("copy", "Copy")])] + recommended, options = recommend_component(["a"], _inventory(graphs)) + assert recommended == "deterministic" + assert options["deterministic"]["capable"] is True + assert options["deterministic"]["uncovered"] == [] + assert options["deterministic"]["activity_counts"] == {"deterministic": 2, "agentic": 0, "unsupported": 0} + + +def test_uncovered_agentic_activity_recommends_agentic_and_cites_the_gap() -> None: + graphs = [ + SourceGraph( + name="a", + source="unit", + tasks=[_node("load", "Notebook"), _node("flow", "ExecuteDataFlow", strategy="agentic")], + ) + ] + recommended, options = recommend_component(["a"], _inventory(graphs)) + assert recommended == "agentic" + assert options["deterministic"]["capable"] is False + assert options["deterministic"]["uncovered"] == [ + {"pipeline": "a", "activity": "flow", "type": "ExecuteDataFlow", "strategy": "agentic"} + ] + assert options["deterministic"]["activity_counts"] == {"deterministic": 1, "agentic": 1, "unsupported": 0} + + +def test_motif_covered_agentic_activity_is_engine_capable() -> None: + # An agentic-strategy activity that a detected motif claims is covered -> the component stays + # engine-capable and the recommendation is deterministic. + graphs = [ + SourceGraph( + name="a", + source="unit", + tasks=[_node("lookup", "Lookup"), _node("each", "ForEach", strategy="agentic")], + ) + ] + motif = DetectedMotif( + definition=MOTIF_METADATA_DRIVEN_BULK_COPY, + matched_activities=["lookup", "each"], + source_type_hint="database", + ) + recommended, options = recommend_component(["a"], _inventory(graphs, motifs={"a": [motif]})) + assert recommended == "deterministic" + assert options["deterministic"]["capable"] is True + assert options["deterministic"]["uncovered"] == [] + assert options["deterministic"]["motifs"] == ["metadata_driven_bulk_copy"] + + +def test_motif_coverage_does_not_leak_across_pipelines_in_a_component() -> None: + # A component spans A -> B. Pipeline A has a motif claiming an activity named 'shared'; pipeline B + # has an UNRELATED activity also named 'shared' with no motif. Motif coverage must be keyed by + # (pipeline, activity), so B's 'shared' stays an uncovered gap rather than being masked by A's. + graphs = [ + SourceGraph( + name="A", + source="unit", + tasks=[_node("call_b", "ExecutePipeline"), _node("shared", "Copy", strategy="agentic")], + lineage=Lineage( + control_edges=[ControlEdge(source_workflow="A", target_workflow="B", via_task_key="call_b")] + ), + ), + SourceGraph(name="B", source="unit", tasks=[_node("shared", "Copy", strategy="agentic")]), + ] + motif = DetectedMotif( + definition=MOTIF_METADATA_DRIVEN_BULK_COPY, matched_activities=["shared"], source_type_hint="database" + ) + recommended, options = recommend_component(["A", "B"], _inventory(graphs, motifs={"A": [motif]})) + assert recommended == "agentic" + assert options["deterministic"]["capable"] is False + # A's 'shared' is motif-covered; only B's unrelated 'shared' is the uncovered gap. + assert options["deterministic"]["uncovered"] == [ + {"pipeline": "B", "activity": "shared", "type": "Copy", "strategy": "agentic"} + ] + + +def test_missing_strategy_counts_as_unsupported_gap() -> None: + node = _node("mystery", "Custom") + node.properties.pop(STRATEGY_PROPERTY) # no strategy recorded at all + graphs = [SourceGraph(name="a", source="unit", tasks=[node])] + recommended, options = recommend_component(["a"], _inventory(graphs)) + assert recommended == "agentic" + assert options["deterministic"]["activity_counts"] == {"deterministic": 0, "agentic": 0, "unsupported": 1} + assert options["deterministic"]["uncovered"][0]["activity"] == "mystery" + + +def test_agentic_option_surfaces_insight_patterns_with_simplification_prominent() -> None: + graphs = [SourceGraph(name="a", source="unit", tasks=[_node("load", "Notebook")])] + insights = { + "pipeline_insights": [ + { + "pipeline": "a", + "recommended_patterns": [ + { + "pattern": "Lakeflow Connect SQL Server connector", + "fit": "Replaces the bespoke extractor", + "simplification_pattern": True, + }, + {"pattern": "Parameterised Lakeflow Job", "fit": "Like-for-like", "simplification_pattern": False}, + ], + } + ] + } + _, options = recommend_component(["a"], _inventory(graphs, insights=insights)) + agentic = options["agentic"] + assert agentic["has_simplification"] is True + assert agentic["recommended_patterns"] == [ + { + "pipeline": "a", + "pattern": "Lakeflow Connect SQL Server connector", + "fit": "Replaces the bespoke extractor", + "simplification_pattern": True, + }, + { + "pipeline": "a", + "pattern": "Parameterised Lakeflow Job", + "fit": "Like-for-like", + "simplification_pattern": False, + }, + ] + + +def test_agentic_option_empty_when_no_insights() -> None: + graphs = [SourceGraph(name="a", source="unit", tasks=[_node("load", "Notebook")])] + _, options = recommend_component(["a"], _inventory(graphs)) + assert options["agentic"] == {"recommended_patterns": [], "has_simplification": False} + + +def test_build_recommendation_emits_both_options_and_a_default_plan() -> None: + graphs = [ + SourceGraph( + name="parent", + source="unit", + tasks=[_node("call_child", "ExecutePipeline")], + lineage=Lineage( + control_edges=[ + ControlEdge(source_workflow="parent", target_workflow="child", via_task_key="call_child") + ] + ), + ), + SourceGraph(name="child", source="unit", tasks=[_node("flow", "ExecuteDataFlow", strategy="agentic")]), + SourceGraph(name="solo", source="unit", tasks=[_node("load", "Notebook")]), + ] + recommendation = build_recommendation(_inventory(graphs)) + ids = [component["component_id"] for component in recommendation["components"]] + assert ids == ["component-1", "component-2"] + first = recommendation["components"][0] + assert first["members"] == ["child", "parent"] + assert set(first["options"]) == {"deterministic", "agentic"} + assert first["recommended"] == "agentic" # child's ExecuteDataFlow is an uncovered gap + assert recommendation["components"][1]["recommended"] == "deterministic" # solo notebook + # The default plan proposes decision == recommended for every component, ready to record as-is. + assert recommendation["default_plan"]["components"] == [ + {"component_id": "component-1", "members": ["child", "parent"], "decision": "agentic"}, + {"component_id": "component-2", "members": ["solo"], "decision": "deterministic"}, + ] + + +def test_recommendation_is_pure_and_idempotent() -> None: + graphs = [SourceGraph(name="a", source="unit", tasks=[_node("load", "Notebook")])] + inventory = _inventory(graphs) + assert build_recommendation(inventory) == build_recommendation(inventory) diff --git a/tests/unit/test_routing_record.py b/tests/unit/test_routing_record.py new file mode 100644 index 00000000..c8ea44fc --- /dev/null +++ b/tests/unit/test_routing_record.py @@ -0,0 +1,388 @@ +"""Tests for validating an authored conversion plan and recording it (routing, #77). + +The agent authors ONLY the per-component decision (and an optional rationale); the library recomputes +members, recommended, and both options' evidence on record, and binds the plan to the inventory with +a SHA-256 fingerprint. The recorded ``conversion_plan.json`` is a separate additive artifact -- it +never mutates ``inventory.json``. Fixtures are built through the source-agnostic emitter so the engine +is proven against the production inventory shape. +""" + +from __future__ import annotations + +import copy +import json +from pathlib import Path +from typing import Any + +import pytest + +from flowx.adapter.__main__ import main as adapter_cli_main +from flowx.discovery_insights import inventory_fingerprint +from flowx.discovery_inventory import STRATEGY_PROPERTY, build_source_inventory +from flowx.models.conversion_plan import ( + SCHEMA_VERSION, + ComponentPlan, + ConversionPlan, +) +from flowx.models.discovery import CONCEPT_NOTEBOOK, CONCEPT_RUN_WORKFLOW, SourceGraph, SourceNode +from flowx.models.ir import ControlEdge, Lineage +from flowx.routing import ( + PLAN_FILENAME, + build_recommendation, + load_plan, + record_plan, + validate_plan, +) + + +def _node(task_key: str, native_type: str, *, strategy: str = "deterministic") -> SourceNode: + return SourceNode( + source_id=task_key, + task_key=task_key, + concept=CONCEPT_NOTEBOOK, + source="unit", + name=task_key, + native_type=native_type, + properties={STRATEGY_PROPERTY: strategy}, + raw={"name": task_key, "type": native_type}, + ) + + +def _inventory() -> dict[str, Any]: + """A two-pipeline component (parent -> child, both engine-capable) plus a standalone 'solo'. + + 'child' carries a Lakeflow Connect simplification insight, so its component is deterministic-capable + AND has a prominent agentic re-architecture option -- the deterministic-vs-LFC choice #77 surfaces. + """ + parent = SourceGraph( + name="parent", + source="unit", + tasks=[_node("call_child", "ExecutePipeline")], + lineage=Lineage( + control_edges=[ControlEdge(source_workflow="parent", target_workflow="child", via_task_key="call_child")] + ), + ) + child = SourceGraph(name="child", source="unit", tasks=[_node("copy_orders", "Copy")]) + solo = SourceGraph(name="solo", source="unit", tasks=[_node("load", "Notebook")]) + inventory = build_source_inventory([parent, child, solo], source="unit", source_dir="/tmp/src") + inventory["insights"] = { + "pipeline_insights": [ + { + "pipeline": "child", + "recommended_patterns": [ + { + "pattern": "Lakeflow Connect SQL Server connector", + "fit": "Replaces the bespoke Copy extractor with a managed pipeline", + "simplification_pattern": True, + } + ], + } + ] + } + return inventory + + +def _authored_plan() -> dict[str, Any]: + """Accept-the-recommendation plan: decision == recommended for both components.""" + return { + "components": [ + {"component_id": "component-1", "members": ["child", "parent"], "decision": "deterministic"}, + {"component_id": "component-2", "members": ["solo"], "decision": "deterministic"}, + ] + } + + +def _write_inventory(output_dir: Path, inventory: dict[str, Any]) -> Path: + metadata = output_dir / "metadata" + metadata.mkdir(parents=True, exist_ok=True) + path = metadata / "inventory.json" + path.write_text(json.dumps(inventory, indent=2), encoding="utf-8") + return path + + +# --------------------------------------------------------------------------- # +# Models. +# --------------------------------------------------------------------------- # + + +def test_models_construct_and_document_the_contract() -> None: + plan = ConversionPlan( + inventory_sha256="abc", + components=[ + ComponentPlan( + component_id="component-1", + members=["child", "parent"], + recommended="deterministic", + decision="agentic", + ) + ], + ) + assert plan.schema_version == SCHEMA_VERSION + assert plan.components[0].decision == "agentic" + + +# --------------------------------------------------------------------------- # +# Validation: success (including the default plan round-trip). +# --------------------------------------------------------------------------- # + + +def test_valid_plan_passes_validation() -> None: + assert validate_plan(_authored_plan(), _inventory()) == [] + + +def test_default_plan_from_recommendation_validates() -> None: + inventory = _inventory() + default_plan = build_recommendation(inventory)["default_plan"] + assert validate_plan(default_plan, inventory) == [] + + +# --------------------------------------------------------------------------- # +# Validation: failure modes. +# --------------------------------------------------------------------------- # + + +def test_non_dict_payload_is_a_violation() -> None: + assert validate_plan([1, 2], _inventory()) == ["conversion plan must be a JSON object, got list"] + + +def test_unknown_top_level_key_including_library_owned_fields() -> None: + raw = _authored_plan() + raw["schema_version"] = "1" + raw["bogus"] = True + violations = validate_plan(raw, _inventory()) + assert any("unknown top-level key: 'schema_version' (set by the library, not the author)" in v for v in violations) + assert any("unknown top-level key: 'bogus'" in v for v in violations) + + +def test_library_owned_component_fields_are_rejected() -> None: + raw = _authored_plan() + raw["components"][0]["recommended"] = "deterministic" + raw["components"][0]["options"] = {} + violations = validate_plan(raw, _inventory()) + assert any("unknown field 'recommended' (set by the library, not the author)" in v for v in violations) + assert any("unknown field 'options' (set by the library, not the author)" in v for v in violations) + + +def test_invalid_decision_value_is_rejected() -> None: + raw = _authored_plan() + raw["components"][0]["decision"] = "maybe" + violations = validate_plan(raw, _inventory()) + assert any("'decision' must be one of" in v for v in violations) + + +def test_unknown_pipeline_in_members_is_rejected() -> None: + raw = _authored_plan() + raw["components"][0]["members"] = ["child", "ghost"] + violations = validate_plan(raw, _inventory()) + assert any("pipeline 'ghost' not in inventory" in v for v in violations) + + +def test_members_that_do_not_form_a_component_are_rejected() -> None: + # 'parent' and 'solo' are in different components; pairing them is not a coherent route. + raw = { + "components": [ + {"component_id": "component-1", "members": ["parent", "solo"], "decision": "deterministic"}, + ] + } + violations = validate_plan(raw, _inventory()) + assert any("do not form a connected component" in v for v in violations) + + +def test_partial_plan_missing_a_component_is_rejected() -> None: + raw = { + "components": [ + {"component_id": "component-1", "members": ["child", "parent"], "decision": "deterministic"}, + ] + } + violations = validate_plan(raw, _inventory()) + assert any("every component must be routed" in v for v in violations) + + +def test_duplicate_decision_for_one_component_is_rejected() -> None: + raw = { + "components": [ + {"component_id": "component-1", "members": ["child", "parent"], "decision": "deterministic"}, + {"component_id": "component-1", "members": ["child", "parent"], "decision": "agentic"}, + {"component_id": "component-2", "members": ["solo"], "decision": "deterministic"}, + ] + } + violations = validate_plan(raw, _inventory()) + assert any("decided" in v and "times" in v for v in violations) + + +def test_component_id_mismatch_is_rejected() -> None: + raw = _authored_plan() + raw["components"][0]["component_id"] = "component-9" + violations = validate_plan(raw, _inventory()) + assert any("does not match" in v for v in violations) + + +def test_duplicate_members_are_rejected() -> None: + # A frozenset would collapse ["solo", "solo"] to {"solo"} and wrongly match component-2; the + # members-match/bijection contract must reject the duplicate explicitly. + raw = _authored_plan() + raw["components"][1]["members"] = ["solo", "solo"] + violations = validate_plan(raw, _inventory()) + assert any("duplicate" in v.lower() and "solo" in v for v in violations) + + +def test_rationale_must_be_non_empty_when_present() -> None: + raw = _authored_plan() + raw["components"][0]["rationale"] = " " + violations = validate_plan(raw, _inventory()) + assert any("'rationale' must be a non-empty string" in v for v in violations) + + +# --------------------------------------------------------------------------- # +# Record: fingerprint binding, both-options shape, atomicity, idempotency. +# --------------------------------------------------------------------------- # + + +def test_record_writes_plan_with_both_options_recommended_and_fingerprint(tmp_path: Path) -> None: + inventory = _inventory() + _write_inventory(tmp_path, inventory) + result = record_plan(tmp_path, plan=_authored_plan()) + assert result["ok"] is True + assert result["components"] == 2 + + plan = json.loads((tmp_path / "metadata" / PLAN_FILENAME).read_text(encoding="utf-8")) + assert plan["schema_version"] == SCHEMA_VERSION + assert plan["inventory_sha256"] == inventory_fingerprint(inventory) + component = plan["components"][0] + assert component["members"] == ["child", "parent"] + assert component["recommended"] == "deterministic" + assert component["decision"] == "deterministic" + assert set(component["options"]) == {"deterministic", "agentic"} + # The agentic option's simplification pattern is surfaced as a first-class peer, not buried. + assert component["options"]["agentic"]["has_simplification"] is True + assert component["options"]["deterministic"]["capable"] is True + + +def test_record_preserves_a_user_override_with_both_recommended_and_decision(tmp_path: Path) -> None: + inventory = _inventory() + _write_inventory(tmp_path, inventory) + raw = _authored_plan() + # User overrides the deterministic recommendation to take the Lakeflow Connect re-architecture. + raw["components"][0]["decision"] = "agentic" + raw["components"][0]["rationale"] = "Adopt the managed connector to retire the extractor" + record_plan(tmp_path, plan=raw) + plan = json.loads((tmp_path / "metadata" / PLAN_FILENAME).read_text(encoding="utf-8")) + component = plan["components"][0] + assert component["recommended"] == "deterministic" + assert component["decision"] == "agentic" + assert component["rationale"] == "Adopt the managed connector to retire the extractor" + + +def test_record_leaves_inventory_byte_identical(tmp_path: Path) -> None: + inventory = _inventory() + inventory_path = _write_inventory(tmp_path, inventory) + original_bytes = inventory_path.read_bytes() + record_plan(tmp_path, plan=_authored_plan()) + assert inventory_path.read_bytes() == original_bytes + + +def test_record_is_idempotent(tmp_path: Path) -> None: + inventory = _inventory() + _write_inventory(tmp_path, inventory) + plan_path = tmp_path / "metadata" / PLAN_FILENAME + record_plan(tmp_path, plan=_authored_plan()) + first = plan_path.read_bytes() + record_plan(tmp_path, plan=_authored_plan()) + assert plan_path.read_bytes() == first + + +def test_record_leaves_plan_untouched_on_validation_failure(tmp_path: Path) -> None: + inventory = _inventory() + _write_inventory(tmp_path, inventory) + record_plan(tmp_path, plan=_authored_plan()) # write a good plan first + good_bytes = (tmp_path / "metadata" / PLAN_FILENAME).read_bytes() + + bad = _authored_plan() + bad["components"][0]["decision"] = "nonsense" + result = record_plan(tmp_path, plan=bad) + assert result["ok"] is False and result["violations"] + assert (tmp_path / "metadata" / PLAN_FILENAME).read_bytes() == good_bytes + + +def test_record_raises_when_inventory_missing(tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + record_plan(tmp_path, plan=_authored_plan()) + + +def test_load_plan_requires_exactly_one_source(tmp_path: Path) -> None: + with pytest.raises(ValueError): + load_plan() + with pytest.raises(ValueError): + load_plan(plan={}, plan_path=tmp_path / "x.json") + + +# --------------------------------------------------------------------------- # +# CLI wiring. The reshaped `route` is one command: no plan on a non-TTY emits the recommendation; a +# `--plan-path` records the plan and edits the report. (Report editing is covered end-to-end in +# test_cli_route_agentic.py; here we exercise the record/recommend wiring against a bare inventory.) +# --------------------------------------------------------------------------- # + + +def _write_empty_report(output_dir: Path) -> None: + """A minimal report matching the inventory's pipelines so the edit step has something to rewrite.""" + work = output_dir / ".work" + work.mkdir(parents=True, exist_ok=True) + report = { + "pipelines": [ + {"name": "parent", "tasks": [{"name": "call_child", "task_key": "call_child", "type": "CopyActivity"}]}, + {"name": "child", "tasks": [{"name": "copy_orders", "task_key": "copy_orders", "type": "CopyActivity"}]}, + {"name": "solo", "tasks": [{"name": "load", "task_key": "load", "type": "NotebookActivity"}]}, + ] + } + (work / "translation_report.json").write_text(json.dumps(report, indent=2), encoding="utf-8") + + +def test_cli_route_without_plan_emits_components( + tmp_path: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch +) -> None: + import io + + _write_inventory(tmp_path, _inventory()) + monkeypatch.setattr("sys.stdin", io.StringIO("")) # not a TTY -> dry-run recommendation + code = adapter_cli_main(["route", "--output-dir", str(tmp_path)]) + assert code == 0 + payload = json.loads(capsys.readouterr().out) + assert [c["component_id"] for c in payload["components"]] == ["component-1", "component-2"] + assert "default_plan" in payload + + +def test_cli_route_with_plan_records_and_reports_the_edit(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + _write_inventory(tmp_path, _inventory()) + _write_empty_report(tmp_path) + plan_path = tmp_path / "plan.json" + plan_path.write_text(json.dumps(_authored_plan()), encoding="utf-8") + code = adapter_cli_main(["route", "--output-dir", str(tmp_path), "--plan-path", str(plan_path)]) + assert code == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["ok"] is True and payload["components"] == 2 + assert "edit" in payload + assert (tmp_path / "metadata" / PLAN_FILENAME).exists() + + +def test_cli_route_validation_failure_returns_1(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + _write_inventory(tmp_path, _inventory()) + _write_empty_report(tmp_path) + raw = _authored_plan() + raw["components"].pop() # partial plan: component-2 undecided + plan_path = tmp_path / "plan.json" + plan_path.write_text(json.dumps(raw), encoding="utf-8") + code = adapter_cli_main(["route", "--output-dir", str(tmp_path), "--plan-path", str(plan_path)]) + assert code == 1 + payload = json.loads(capsys.readouterr().out) + assert payload["ok"] is False and payload["violations"] + + +def test_fixture_isolation() -> None: + a = _authored_plan() + b = _authored_plan() + a["components"][0]["decision"] = "agentic" + assert b["components"][0]["decision"] == "deterministic" + assert copy.deepcopy(a) == a + # CONCEPT_RUN_WORKFLOW is imported for symmetry with the discovery fixtures; touch it so linters + # see the dependency used. + assert isinstance(CONCEPT_RUN_WORKFLOW, str) From cd843d393da6d1bd8839ed38482434277176e9f7 Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Thu, 17 Sep 2026 15:21:20 +0100 Subject: [PATCH 002/122] Add flowx-route skill and five-step migrate flow; register routing skills MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the first-class flowx-route skill (interactive route → decide → record plan → fill routed-agentic groups) and updates flowx-migrate to the discover → enrich → route → convert/fill → package flow. Registers flowx-route and the already-present flowx-resolve-airflow-gaps in the manifest, so it now lists both flowx-enrich (from the agentic-insights PR) and flowx-route. Convert runs as the deterministic baseline before routing; no plain convert re-run after routing (only the additive merge/combine fills), so routed-agentic placeholders survive to package. Customer decides the route per component; the agent presents options and authors the approved fill. Skill/manifest docs only; no src/ changes. Co-authored-by: Isaac --- .claude-plugin/plugin.json | 2 + skills/flowx-migrate/SKILL.md | 103 +++++++- skills/flowx-migrate/references/workflow.md | 22 ++ skills/flowx-route/SKILL.md | 262 ++++++++++++++++++++ 4 files changed, 376 insertions(+), 13 deletions(-) create mode 100644 skills/flowx-route/SKILL.md diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 48b905b4..fa3e9047 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -12,7 +12,9 @@ "./skills/flowx-setup", "./skills/flowx-discover", "./skills/flowx-enrich", + "./skills/flowx-route", "./skills/flowx-convert", + "./skills/flowx-resolve-airflow-gaps", "./skills/flowx-package", "./skills/flowx-migrate" ] diff --git a/skills/flowx-migrate/SKILL.md b/skills/flowx-migrate/SKILL.md index e12b05cd..e90467e6 100644 --- a/skills/flowx-migrate/SKILL.md +++ b/skills/flowx-migrate/SKILL.md @@ -2,7 +2,8 @@ name: flowx-migrate description: > End-to-end migration of a source orchestrator's pipelines (Azure Data Factory, Apache Airflow) - to Databricks Lakeflow Jobs. Orchestrates discover, convert, and package phases in sequence. + to Databricks Lakeflow Jobs. Orchestrates discover → enrich → route → convert/fill → package in + sequence. triggers: - "migrate pipelines" - "migrate ADF" @@ -16,16 +17,25 @@ triggers: # End-to-End Source to Databricks Migration Orchestrate the complete migration of a source orchestrator's pipelines to Databricks Lakeflow Jobs -via Declarative Automation Bundles. This skill runs all three phases in sequence: discover, convert, -package. +via Declarative Automation Bundles. This skill runs the full flow in sequence: discover → enrich +(default) → route → convert/fill → package. ## Context This is the top-level orchestration skill. It runs the full migration pipeline: 1. **Discover** — Parse the source's definitions into a typed inventory -2. **Convert** — Convert the source's tasks to Databricks IR (deterministic + agentic) -3. **Package** — Generate Databricks Declarative Automation Bundles for deployment +2. **Enrich** *(default)* — Author the agentic `insights` layer over the inventory and merge it + (`flowx-enrich`); skippable for a deterministic-only pass +3. **Convert (deterministic baseline)** — Convert the source's tasks to Databricks IR, producing + `.work/translation_report.json`. This runs **once, before routing** (routing reads and edits it); + route can also trigger it in-process (`flowx-convert`) +4. **Route + fill** — Decide, per connected component, deterministic vs. agentic conversion; record + the fingerprint-bound `metadata/conversion_plan.json`; edit the baseline report so routed-agentic + groups become placeholder gaps; then **fill** those gaps additively — per-pipeline + `convert --merge-agentic` or cross-pipeline `fill-agentic combine`. **Never re-run a plain + `convert` after routing** — it would rewrite the report and erase the placeholders (`flowx-route`) +5. **Package** — Generate Databricks Declarative Automation Bundles for deployment Each phase builds on the output of the previous phase. The user is shown a summary and asked to confirm before proceeding to the next phase. @@ -40,7 +50,7 @@ source path (`--adf-source-path` / `--airflow-source-path`, both aliases of `--s ## How to run this skill — MCP tools or venv CLI -This skill orchestrates all three phases. Run the **`setup`** skill first if you haven't. There are +This skill orchestrates the full flow. Run the **`setup`** skill first if you haven't. There are two execution paths: ### MCP tools (Databricks Genie Code, or a local stdio registration) @@ -101,8 +111,11 @@ discover/convert and for `inputs discover`/`inputs convert`; for Airflow, swap ` ``` flowx(command="inputs", parameters={"phase": "discover", "source": "adf"}) # source req for discover/convert flowx(command="discover", parameters={"source": "adf", "adf_definitions": {...}, "output_dir": ..., "pipeline": ...}) +flowx(command="enrich", parameters={"output_dir": ..., "insights": {...}}) # default: author + merge the insights layer (see flowx-enrich) +flowx(command="route", parameters={"output_dir": ...}) # recommend; re-call with "plan": {...} to record + edit the report (see flowx-route) flowx(command="convert", parameters={"source": "adf", "output_dir": ..., "pipeline": ...}) -flowx(command="merge_agentic", parameters={"source": "adf", "report_path": ..., "agentic_results_dir": ..., "output_path": ...}) # ADF only, if agentic results +flowx(command="merge_agentic", parameters={"source": "adf", "report_path": ..., "agentic_results_dir": ..., "output_path": ...}) # ADF only, per-pipeline agentic fill +flowx(command="fill_agentic", parameters={"output_dir": ..., "members": [...], "pipelines": [...]}) # cross-pipeline combine of a routed-agentic group flowx(command="inspect", parameters={"report_path": ...}) flowx(command="apply_answers", parameters={"report_path": ..., "answers": [...], "output_dir": ...}) flowx(command="package", parameters={"output_dir": ..., "catalog": ..., "schema": ...}) @@ -209,15 +222,60 @@ If the user says no, explain the options: - Review `/metadata/inventory.json` (and `profile_report.csv`) to understand unsupported activities and pipeline complexity - Manually classify activities before proceeding -If the user says yes, proceed to step 4. - -### Step 4 — Phase 2: Convert - -Invoke the `flowx:flowx-convert` skill with: +If the user says yes, proceed to step 3.5. + +### Step 3.5 — Enrich the inventory (default) + +By default, chain into enrichment: invoke the **`flowx:flowx-enrich`** skill to author the agentic +`insights` layer (factory-wide recommendation, per-pipeline intent + recommended Databricks patterns, +cross-pipeline relationships) and merge it into `/metadata/inventory.json`. flowx has no +LLM — you author the insights JSON and `enrich` validates + merges it additively, leaving every +deterministic inventory key byte-identical. The routing step consumes this block to present the +agentic conversion option per group. + +**Skip only for a deterministic-only pass.** When the user explicitly asked for a headless, +deterministic-only migration (no agent/LLM authoring), skip enrich — the deterministic +`inventory.json` is complete and `flowx-route` still works from the structure alone. Otherwise enrich +by default. + +### Step 3.6 — Route the conversion (deterministic vs. agentic) + +Routing reads and edits the **deterministic baseline** report `.work/translation_report.json`, so +convert (Step 4) must have produced it first. The clean path is to let `route` **trigger convert once +in-process** — pass `--source` / `--source-path` and route builds the baseline and then edits it in a +single step (this *is* the Phase-2 convert; a separate Step 4 run is then unnecessary). Otherwise run +Step 4 before this step. + +Invoke the **`flowx:flowx-route`** skill to present the per-connected-component recommendation, take +the **customer's** decision (interactive, or an authored plan — the customer decides +deterministic-vs-agentic per component; the agent presents the options and serializes only the +approved decision), and record the fingerprint-bound `/metadata/conversion_plan.json`. +Recording a plan edits `.work/translation_report.json` so every routed-**agentic** group's tasks +become placeholder gaps; deterministic groups (and a no-agentic-route plan) leave the report +untouched — non-breaking. + +**Once routing has recorded agentic placeholders, never run a plain `convert` again** — a second +`convert --source-dir` overwrites `.work/translation_report.json` and erases the placeholders. +Routed-agentic groups are **filled** additively in Step 5.05 (per-pipeline `convert --merge-agentic` +or cross-pipeline `fill-agentic combine`), which is the only convert after routing. If the user wants +a straight deterministic migration, they accept the all-deterministic recommendation here and the +rest of the flow is unchanged. + +### Step 4 — Phase 2: Convert (deterministic baseline, runs before routing) + +Convert produces the deterministic baseline report `/.work/translation_report.json` that +routing (Step 3.6) reads and edits — so it runs **before** the plan is recorded. If you let `route` +trigger convert in-process (Step 3.6), this step is already done; otherwise invoke the +`flowx:flowx-convert` skill directly, before routing, with: - `--source `: the same source discover used - Source path: the original source path (same one discover used) - Output dir: the same shared `` (convert writes its report to `/.work/`) +> **Run this convert exactly once, before routing.** After Step 3.6 records agentic placeholders, do +> **not** re-run a plain `convert` — it rewrites `.work/translation_report.json` and erases the +> routed-agentic placeholders. The only convert after routing is the additive `convert --merge-agentic` +> fill in Step 5.05. + Wait for the translation to complete and present the summary: ``` @@ -241,6 +299,25 @@ For failures, suggest: - Retry with additional context - Skip and add placeholder +### Step 5.05 — Fill routed-agentic groups (before just-in-time config) + +If Step 3.6 routed any component **agentic**, its pipelines' tasks are now `PlaceholderActivity` +nodes with one tagged gap each. Fill them via the **`flowx:flowx-route`** skill (Step 3 there) before +the just-in-time config below, so the filled tasks get configuration-stamped and packaged: + +- **Per-pipeline** (the pipeline stays 1:1): author one result JSON per gap and merge with + `convert --source adf --merge-agentic --report .work/translation_report.json --agentic-results ` + (ADF only; `--source` is mandatory for the convert phase — it exits 2 without it). This merge is + additive: it replaces only the placeholder tasks and leaves every other pipeline byte-identical. + Airflow per-gap fills use the `flowx-resolve-airflow-gaps` skill instead. +- **Cross-pipeline COMBINE** (N pipelines → M, e.g. one Lakeflow Connect pipeline): author the + replacement pipeline IR (typically with `AgenticComponentActivity` nodes) and run + `fill-agentic combine --output-dir --members "" --pipelines-path `; + the members must exactly match the routed-agentic component and the merged report is validated + structurally before it is written. + +Skip this step entirely when no component was routed agentic. + ### Step 5.1 — Gather just-in-time translation configuration Run `inspect` **once** to get the full option schema (every option carries a `show_when` condition), @@ -432,7 +509,7 @@ See `references/workflow.md` for a detailed description of the three-phase archi ## Output Artifacts -All three phases write into a single shared `` (default `./flowx_output`): +All phases write into a single shared `` (default `./flowx_output`): | Path | Phase | Contents | |---|---|---| diff --git a/skills/flowx-migrate/references/workflow.md b/skills/flowx-migrate/references/workflow.md index faf083b9..cb345ef5 100644 --- a/skills/flowx-migrate/references/workflow.md +++ b/skills/flowx-migrate/references/workflow.md @@ -60,6 +60,28 @@ ADF JSON Exports - Datasets and linked services are parsed for context but not independently translated — they inform the activity translators. - Triggers are included in the inventory and translated in phase 2. +## Between Discover and Convert: Enrich (default) + Route + +**Skills:** `flowx:flowx-enrich`, `flowx:flowx-route` + +After discover writes the deterministic inventory, the standard flow enriches and routes before +convert. Both are additive and contain **no LLM** — the agent authors, the library validates and +merges: + +- **Enrich (default):** the agent authors an `insights` layer (factory-wide recommendation, + per-pipeline intent + recommended Databricks patterns, cross-pipeline relationships) and `enrich` + merges it into `metadata/inventory.json` under a single additive `insights` key, leaving every + deterministic key byte-identical. Skippable for a deterministic-only, headless pass. +- **Route:** groups pipelines into connected components over control lineage and, per component, + records a `deterministic` or `agentic` decision as the fingerprint-bound + `metadata/conversion_plan.json`. Recording an agentic decision edits `.work/translation_report.json` + so those groups' tasks become placeholder gaps; a fully-deterministic plan (or no plan) leaves + convert/package behaving exactly as before — the non-breaking guarantee. + +Routed-agentic groups are then filled during convert: per-pipeline via `convert --merge-agentic`, or +cross-pipeline (N→M, e.g. a Lakeflow Connect collapse) via `fill-agentic combine` using authored +`AgenticComponentActivity` nodes. + ## Phase 2: Convert **Skill:** `flowx:flowx-convert` diff --git a/skills/flowx-route/SKILL.md b/skills/flowx-route/SKILL.md new file mode 100644 index 00000000..f1e2ad35 --- /dev/null +++ b/skills/flowx-route/SKILL.md @@ -0,0 +1,262 @@ +--- +name: flowx-route +description: > + Route each connected component of the discovered inventory to a deterministic (1:1 engine) or + agentic (LLM-assisted re-architecture) conversion, record the fingerprint-bound conversion plan, + and fill the routed-agentic groups — per-pipeline via convert --merge-agentic, or cross-pipeline + via fill-agentic combine. Runs after enrich, before/with convert. +triggers: + - "route pipelines" + - "route conversion" + - "conversion plan" + - "deterministic or agentic" + - "fill agentic" + - "combine pipelines" + - "agentic conversion" + - "recommend conversion route" +--- + +# Route the Conversion (deterministic vs. agentic) and Fill the Gaps + +After discover (and, by default, `flowx-enrich`), routing decides **per connected component** whether +each part of the factory converts **deterministically** (the typed engine's 1:1 translation) or +**agentically** (an LLM-authored re-architecture, e.g. collapsing five extractors onto one Lakeflow +Connect pipeline). It records the decision as a fingerprint-bound `metadata/conversion_plan.json`, +edits the translation report so routed-agentic groups become placeholder gaps, and then you author +the fill. + +**There is no LLM inside flowx.** The library computes the recommendation deterministically and only +*validates and records* the decision and the authored fill — the same author → validate → merge +contract `enrich` uses. This is additive and non-breaking: with **no recorded plan**, `convert` and +`package` behave exactly as before. + +**Who decides what.** The library **recommends** a route per component; the **customer decides** +deterministic-vs-agentic for each component; the **agent** presents the options and the +recommendation, serializes only the customer's *approved* decision into the plan, and authors the +agentic fill. The agent never picks the route on the customer's behalf — it records the customer's +choice and does the mechanical work of the approved fill. + +## Where routing sits + +``` +discover → enrich (default) → convert (deterministic baseline) → route (decide + edit) → fill-agentic → package +``` + +Convert builds the deterministic baseline report **before** routing (route can trigger it in-process +via `--source` / `--source-path`); routing then edits that report. After routing has recorded agentic +placeholders, the only convert is the additive `convert --merge-agentic` fill — never a second plain +`convert`, which would overwrite the report and erase the placeholders. + +Pipelines are grouped into weak/undirected **connected components** over the inventory's control +lineage (`lineage.control_edges`), so mutually-referencing pipelines are decided together and a +caller/callee reference is never split across incompatible routes. Both ADF (`ExecutePipeline`) and +Airflow (`RunJob`) emit control edges, so routing is source-neutral. + +Run the **`setup`** skill first if you haven't. Everything below has an MCP-tool path (Genie Code, or +a local stdio registration — call the single **`flowx`** tool, run no `python3`/`$PY`) and a venv-CLI +path (local; `PY="$(cat /.migration-venv)"` and `export PYTHONPATH="/src"`). + +## Step 1 — Recommend (the dry run you read first) + +Call `route` with **no decision** to get the recommendation: every component with its `members`, both +first-class conversion `options` (a *deterministic* option carrying the engine-capability assessment ++ any uncovered gaps, and an *agentic* option carrying the `recommended_patterns` from `enrich`, with +any `simplification_pattern` flagged), the `findings` (unresolved/dangling control edges kept, never +severed), and a ready-to-record `default_plan` proposing `decision == recommended` for every +component. + +- **MCP tool:** `flowx(command="route", parameters={"output_dir": ""})` +- **venv CLI:** + + ```bash + export PYTHONPATH="/src" + PY="$(cat /.migration-venv)" + "$PY" -m flowx.adapter route --output-dir + ``` + + On a non-TTY with no `--plan-path`, this emits the recommendation and exits 0 (it edits nothing). + `route` reads `metadata/inventory.json`; run discover first. `--out ` writes the JSON to a + file instead of stdout. + +`recommended` is the library's starting suggestion — `deterministic` when the whole component is +engine-capable, else `agentic`. Present each component's members, its recommendation, and the agentic +option's patterns (flag any `has_simplification` re-architecture prominently). + +## Step 2 — Decide and record the plan + +Take the per-component decision and record it. `route` validates the plan, writes the fingerprint-bound +`metadata/conversion_plan.json`, and then edits `.work/translation_report.json` + `gaps.json` so every +routed-**agentic** group's tasks become placeholder gaps (deterministic groups are left byte-identical; +when nothing is routed agentic the report is untouched — the non-breaking guarantee). + +There are three ways to supply the decision: + +- **Interactive prompt (TTY).** Run `route` with no `--plan-path` on a real terminal and it asks, per + component, `route [d]eterministic / [a]gentic (default=)`. An empty answer accepts the + recommendation. This is the from-the-seat path. +- **Authored plan file** — `--plan-path ` (or `--plan-path -` to read the plan JSON from stdin). +- **MCP tool** — pass the plan inline as `plan` (or `plan_path`): + + ``` + flowx(command="route", parameters={"output_dir": "", "plan": { ...authored plan... }}) + ``` + +### The plan shape + +The plan carries **only** the decision (and an optional rationale) per component — and the decision is +the **customer's**, not yours. Your job is to present each component's members, its `recommended` +route, and both `options`, then serialize the customer's pick; you do not choose the route yourself. +The library recomputes `members`, `recommended`, and both `options` on record, so recorded facts +cannot drift from the inventory or be faked. Start from the recommendation's `default_plan` and flip +the components the customer chose to override: + +```json +{ + "components": [ + {"component_id": "component-1", "members": ["IngestSalesforce", "IngestWorkday"], "decision": "agentic", + "rationale": "Collapse both extractors onto one Lakeflow Connect pipeline"}, + {"component_id": "component-2", "members": ["BuildMart"], "decision": "deterministic"} + ] +} +``` + +Rules the validator enforces (all violations returned at once; nothing written on failure): + +- `decision` must be `"deterministic"` or `"agentic"`; `rationale` (optional) must be a non-empty + string when present. +- Every `member` must be a real inventory pipeline, and a component's `members` must **exactly match + one computed connected component** — a decision can never split a component or span two. +- The plan is a **bijection**: every component is decided exactly once (no partial plan, no + duplicate/conflicting decisions). +- `component_id` is **required** on every component — a non-empty string that must match the computed + component for those `members`. Start from the recommendation's `default_plan`, which already carries + the correct `component_id` for each component. + +### Triggering convert if the report is missing + +`route` edits `.work/translation_report.json`, which the convert phase produces. If it is missing, +pass `--source ` **and** `--source-path ` (MCP: `source` plus the source-specific +path param — `adf_source_path` or `airflow_source_path`; a generic `source_path` is **ignored**, and +ADF also accepts `adf_definitions` / `adf_volume_path` / `adf_workspace_path`) and `route` triggers +the convert phase in-process first. Otherwise run `flowx-convert` before routing. + +### venv CLI + +```bash +"$PY" -m flowx.adapter route --output-dir --plan-path plan.json \ + [--source adf --source-path ] # only needed to trigger convert when the report is missing +``` + +Exit 1 (nothing written) on a missing report it cannot produce, or a plan that fails validation. + +## Step 3 — Fill the routed-agentic groups + +Every routed-agentic pipeline's tasks are now `PlaceholderActivity` nodes with one pipeline-tagged +`AgenticGap` each. Two fills exist depending on the grain; you author the replacement (no LLM in the +library — it validates and merges). + +### 3a — Per-pipeline fill (keep the pipeline 1:1) + +When a routed-agentic pipeline stays one pipeline and you just author a Databricks task per gap, reuse +the **existing** name-matched merge — the same path the agentic gap flow uses. Write one JSON file per +resolved gap into an agentic-results directory: + +```json +{ + "activity_name": "", + "pipeline": "", + "task": {"type": "NotebookActivity", "name": "", "task_key": "", + "notebook_path": "/Workspace/.../your_translated_notebook"} +} +``` + +Then merge (`task_key`/`depends_on` are inherited from the placeholder when omitted, preserving edges): + +```bash +"$PY" -m flowx.adapter convert --source adf --merge-agentic \ + --report /.work/translation_report.json \ + --agentic-results \ + [--output ] # default: overwrite --report +``` + +`--source` is **mandatory** for the convert phase — the phase runner exits 2 without it, even for +`--merge-agentic`. Use `--source adf` here (`merge_agentic` is **ADF-only**; an Airflow per-gap fill +would use `--source airflow`, but Airflow gaps normally go through the `flowx-resolve-airflow-gaps` +skill). This merge is **additive** — it replaces only the placeholder tasks in the existing report and +leaves every other pipeline byte-identical, so it never erases routing's edits. + +MCP: `flowx(command="merge_agentic", parameters={"source": "adf", "report_path": ..., "agentic_results_dir": ..., "output_path": ...})`. +Placeholders are replaced in place, status → `translated`; exits non-zero if any result can't be +matched. + +### 3b — Cross-pipeline COMBINE (N pipelines → M) + +When the decision is a re-architecture that changes pipeline count — e.g. five extractor pipelines +collapse onto **one** Lakeflow Connect pipeline — use `fill-agentic combine`. You author the +replacement pipeline(s) as IR dicts and the whole routed group is swapped for them. + +```bash +"$PY" -m flowx.adapter fill-agentic combine \ + --output-dir \ + --members "IngestSalesforce,IngestWorkday" \ + --pipelines-path authored_pipelines.json \ + [--out ] +``` + +MCP: `flowx(command="fill_agentic", parameters={"output_dir": ..., "members": [...], "pipelines": [...]})` +(pass `pipelines` inline as a list, or `pipelines_path`). + +`combine` is the only action. Its guarantees: + +- `--members` (comma-separated; MCP accepts a list) must **exactly match** a routed-**agentic** + component in the recorded `metadata/conversion_plan.json`, whose `inventory_sha256` must still match + the current inventory. A partial group, a superset, a typo, or a deterministic component is refused + — you can't swap pipelines the plan didn't route agentic. +- `--pipelines-path` is a JSON **list** of pipeline IR dicts (the authored replacements), typically + carrying `AgenticComponentActivity` nodes (see below). +- The merged report is **always** validated with the structural bundle invariants (a real + `prepare → write_bundle` pass) before it is written — no bypass — so a duplicate key, dangling + dependency, cycle, or dangling pipeline/run_job reference can never land on disk. On any violation, + `ok` is `false`, `violations` lists them, and nothing is written. + +### Authoring an `AgenticComponentActivity` (the escape hatch) + +When the target can't be expressed by the typed engine (e.g. a managed Lakeflow Connect ingestion +pipeline), emit an `AgenticComponentActivity` task inside the authored pipeline. It carries the raw +bundle components the package phase writes verbatim: + +```json +{ + "name": "IngestAll", + "task_key": "ingest_all", + "type": "AgenticComponentActivity", + "files": [ + {"path": "src/ingest/lakeflow_connect.py", "content": "# authored pipeline source ..."} + ], + "resources": [ + {"resource_key": "ingest_all_pipeline", "definition": { ...raw pipeline resource... }} + ], + "task": {"pipeline_task": {"pipeline_id": "${resources.pipelines.ingest_all_pipeline.id}"}}, + "raw_definition": { ...original source definitions, retained for auditing... } +} +``` + +- `files` — files written below the bundle `src/`; each is `path` + either UTF-8 `content` or + base64 `binary_content`. +- `resources` — bundle resources in the `resource_key` + raw `definition` shape the bundle writer + expects. +- `task` — the raw Databricks task fragment (`pipeline_task` or `notebook_task`) wiring to an + authored resource or file. +- `raw_definition` — the original source definition, retained for provenance. + +## Step 4 — Continue to package + +Once the routed-agentic groups are filled and the report validates, continue with `flowx-convert`'s +just-in-time configuration (`inspect`/`modify`) as usual, then `flowx-package`. The recorded +`metadata/conversion_plan.json` is kept alongside `inventory.json` as the routing record. + +## Reference + +- `flowx-enrich` skill — the `insights` layer routing's agentic option consumes. +- `flowx-convert` — the deterministic engine, per-pipeline `merge_agentic`, and just-in-time config. +- `flowx-package` — turns the (filled) report into the deployable DAB bundle. From a8d9921a254600850f4e94f00546942e0d18a52a Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Thu, 17 Sep 2026 15:35:36 +0100 Subject: [PATCH 003/122] Fix re-review findings: Airflow merge path, merge behavior, migrate order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Airflow per-gap fill: convert --merge-agentic is ADF-only. Airflow's --merge-agentic is disabled and exits 2, so direct Airflow gaps to the flowx-resolve-airflow-gaps skill; removed the false --source airflow suggestion (route + migrate). 2. Merge behavior: describe it accurately — merge_agentic_results replaces the FIRST task whose name matches activity_name (matched by name, not verified to be a PlaceholderActivity). Removed the unfounded "replaces only placeholder tasks" guarantee (route + migrate). 3. Migrate sequencing: make the whole skill consistently linear — discover -> enrich -> convert (deterministic baseline) -> route (decide + edit) -> fill agentic gaps -> package. Fixed frontmatter, intro, and phase list; reordered and renumbered the workflow steps (convert now Step 5 before route Step 7, fill Step 8) so it reads top-to-bottom in execution order with no forward/back jumps; reordered the MCP command sequence to put convert before route. Grounded against adapter/__main__.py, sources/airflow/convert.py, and ir_serde.py. Skill-doc edits only; no src/ changes. make fmt + make test green. Co-authored-by: Isaac --- skills/flowx-migrate/SKILL.md | 126 ++++++++++++++++------------------ skills/flowx-route/SKILL.md | 18 +++-- 2 files changed, 72 insertions(+), 72 deletions(-) diff --git a/skills/flowx-migrate/SKILL.md b/skills/flowx-migrate/SKILL.md index e90467e6..b09d2b50 100644 --- a/skills/flowx-migrate/SKILL.md +++ b/skills/flowx-migrate/SKILL.md @@ -2,8 +2,8 @@ name: flowx-migrate description: > End-to-end migration of a source orchestrator's pipelines (Azure Data Factory, Apache Airflow) - to Databricks Lakeflow Jobs. Orchestrates discover → enrich → route → convert/fill → package in - sequence. + to Databricks Lakeflow Jobs. Orchestrates discover → enrich → convert (deterministic baseline) → + route → fill → package in sequence. triggers: - "migrate pipelines" - "migrate ADF" @@ -18,7 +18,7 @@ triggers: Orchestrate the complete migration of a source orchestrator's pipelines to Databricks Lakeflow Jobs via Declarative Automation Bundles. This skill runs the full flow in sequence: discover → enrich -(default) → route → convert/fill → package. +(default) → convert (deterministic baseline) → route (decide + edit) → fill agentic gaps → package. ## Context @@ -28,14 +28,15 @@ This is the top-level orchestration skill. It runs the full migration pipeline: 2. **Enrich** *(default)* — Author the agentic `insights` layer over the inventory and merge it (`flowx-enrich`); skippable for a deterministic-only pass 3. **Convert (deterministic baseline)** — Convert the source's tasks to Databricks IR, producing - `.work/translation_report.json`. This runs **once, before routing** (routing reads and edits it); - route can also trigger it in-process (`flowx-convert`) -4. **Route + fill** — Decide, per connected component, deterministic vs. agentic conversion; record - the fingerprint-bound `metadata/conversion_plan.json`; edit the baseline report so routed-agentic - groups become placeholder gaps; then **fill** those gaps additively — per-pipeline - `convert --merge-agentic` or cross-pipeline `fill-agentic combine`. **Never re-run a plain - `convert` after routing** — it would rewrite the report and erase the placeholders (`flowx-route`) -5. **Package** — Generate Databricks Declarative Automation Bundles for deployment + `.work/translation_report.json` (`flowx-convert`). Runs **before** routing +4. **Route** — Decide, per connected component, deterministic vs. agentic conversion; record the + fingerprint-bound `metadata/conversion_plan.json`; edit the baseline report so routed-agentic + groups become placeholder gaps (`flowx-route`) +5. **Fill agentic gaps** — Replace the routed-agentic placeholders: per-pipeline + `convert --merge-agentic` (ADF only) or cross-pipeline `fill-agentic combine`; Airflow gaps go + through `flowx-resolve-airflow-gaps`. **Never re-run a plain `convert` after routing** — it + rewrites the report and erases the placeholders +6. **Package** — Generate Databricks Declarative Automation Bundles for deployment Each phase builds on the output of the previous phase. The user is shown a summary and asked to confirm before proceeding to the next phase. @@ -112,8 +113,8 @@ discover/convert and for `inputs discover`/`inputs convert`; for Airflow, swap ` flowx(command="inputs", parameters={"phase": "discover", "source": "adf"}) # source req for discover/convert flowx(command="discover", parameters={"source": "adf", "adf_definitions": {...}, "output_dir": ..., "pipeline": ...}) flowx(command="enrich", parameters={"output_dir": ..., "insights": {...}}) # default: author + merge the insights layer (see flowx-enrich) +flowx(command="convert", parameters={"source": "adf", "output_dir": ..., "pipeline": ...}) # deterministic baseline, BEFORE route flowx(command="route", parameters={"output_dir": ...}) # recommend; re-call with "plan": {...} to record + edit the report (see flowx-route) -flowx(command="convert", parameters={"source": "adf", "output_dir": ..., "pipeline": ...}) flowx(command="merge_agentic", parameters={"source": "adf", "report_path": ..., "agentic_results_dir": ..., "output_path": ...}) # ADF only, per-pipeline agentic fill flowx(command="fill_agentic", parameters={"output_dir": ..., "members": [...], "pipelines": [...]}) # cross-pipeline combine of a routed-agentic group flowx(command="inspect", parameters={"report_path": ...}) @@ -222,9 +223,9 @@ If the user says no, explain the options: - Review `/metadata/inventory.json` (and `profile_report.csv`) to understand unsupported activities and pipeline complexity - Manually classify activities before proceeding -If the user says yes, proceed to step 3.5. +If the user says yes, proceed to Step 4. -### Step 3.5 — Enrich the inventory (default) +### Step 4 — Enrich the inventory (default) By default, chain into enrichment: invoke the **`flowx:flowx-enrich`** skill to author the agentic `insights` layer (factory-wide recommendation, per-pipeline intent + recommended Databricks patterns, @@ -238,43 +239,18 @@ deterministic-only migration (no agent/LLM authoring), skip enrich — the deter `inventory.json` is complete and `flowx-route` still works from the structure alone. Otherwise enrich by default. -### Step 3.6 — Route the conversion (deterministic vs. agentic) +### Step 5 — Phase 2: Convert (deterministic baseline) -Routing reads and edits the **deterministic baseline** report `.work/translation_report.json`, so -convert (Step 4) must have produced it first. The clean path is to let `route` **trigger convert once -in-process** — pass `--source` / `--source-path` and route builds the baseline and then edits it in a -single step (this *is* the Phase-2 convert; a separate Step 4 run is then unnecessary). Otherwise run -Step 4 before this step. - -Invoke the **`flowx:flowx-route`** skill to present the per-connected-component recommendation, take -the **customer's** decision (interactive, or an authored plan — the customer decides -deterministic-vs-agentic per component; the agent presents the options and serializes only the -approved decision), and record the fingerprint-bound `/metadata/conversion_plan.json`. -Recording a plan edits `.work/translation_report.json` so every routed-**agentic** group's tasks -become placeholder gaps; deterministic groups (and a no-agentic-route plan) leave the report -untouched — non-breaking. - -**Once routing has recorded agentic placeholders, never run a plain `convert` again** — a second -`convert --source-dir` overwrites `.work/translation_report.json` and erases the placeholders. -Routed-agentic groups are **filled** additively in Step 5.05 (per-pipeline `convert --merge-agentic` -or cross-pipeline `fill-agentic combine`), which is the only convert after routing. If the user wants -a straight deterministic migration, they accept the all-deterministic recommendation here and the -rest of the flow is unchanged. - -### Step 4 — Phase 2: Convert (deterministic baseline, runs before routing) - -Convert produces the deterministic baseline report `/.work/translation_report.json` that -routing (Step 3.6) reads and edits — so it runs **before** the plan is recorded. If you let `route` -trigger convert in-process (Step 3.6), this step is already done; otherwise invoke the -`flowx:flowx-convert` skill directly, before routing, with: +Convert the source's tasks to Databricks IR, producing the **deterministic baseline** report +`/.work/translation_report.json`. This runs **before** routing, because routing (Step 7) +reads and edits this report. Invoke the `flowx:flowx-convert` skill with: - `--source `: the same source discover used - Source path: the original source path (same one discover used) - Output dir: the same shared `` (convert writes its report to `/.work/`) -> **Run this convert exactly once, before routing.** After Step 3.6 records agentic placeholders, do -> **not** re-run a plain `convert` — it rewrites `.work/translation_report.json` and erases the -> routed-agentic placeholders. The only convert after routing is the additive `convert --merge-agentic` -> fill in Step 5.05. +Run this deterministic convert **exactly once, here, before routing.** The only convert *after* +routing is the additive `convert --merge-agentic` fill in Step 8 — a second plain `convert` would +rewrite `.work/translation_report.json` and erase the routed-agentic placeholders. Wait for the translation to complete and present the summary: @@ -287,7 +263,7 @@ Failed: 4 ( 8.5%) Overall coverage: 91.5% ``` -### Step 5 — Present translation details +### Step 6 — Present translation details Show the user: 1. What was translated deterministically (bulk — just counts by type) @@ -299,26 +275,46 @@ For failures, suggest: - Retry with additional context - Skip and add placeholder -### Step 5.05 — Fill routed-agentic groups (before just-in-time config) +### Step 7 — Route the conversion (deterministic vs. agentic) -If Step 3.6 routed any component **agentic**, its pipelines' tasks are now `PlaceholderActivity` -nodes with one tagged gap each. Fill them via the **`flowx:flowx-route`** skill (Step 3 there) before -the just-in-time config below, so the filled tasks get configuration-stamped and packaged: +Routing reads the deterministic baseline report `.work/translation_report.json` (built in Step 5) and +edits it. Invoke the **`flowx:flowx-route`** skill to present the per-connected-component +recommendation, take the **customer's** decision (interactive, or an authored plan — the customer +decides deterministic-vs-agentic per component; the agent presents the options and serializes only the +approved decision), and record the fingerprint-bound `/metadata/conversion_plan.json`. +Recording a plan edits `.work/translation_report.json` so every routed-**agentic** group's tasks +become placeholder gaps; deterministic groups (and a no-agentic-route plan) leave the report +untouched — non-breaking. -- **Per-pipeline** (the pipeline stays 1:1): author one result JSON per gap and merge with - `convert --source adf --merge-agentic --report .work/translation_report.json --agentic-results ` - (ADF only; `--source` is mandatory for the convert phase — it exits 2 without it). This merge is - additive: it replaces only the placeholder tasks and leaves every other pipeline byte-identical. - Airflow per-gap fills use the `flowx-resolve-airflow-gaps` skill instead. +If Step 5 was skipped and the report is missing, `route` can trigger the convert phase in-process +once (pass `--source` / `--source-path`); it never runs a second plain convert once a report exists. +If the user wants a straight deterministic migration, they accept the all-deterministic recommendation +here and the rest of the flow is unchanged (no placeholders, so Step 8 is a no-op). + +### Step 8 — Fill routed-agentic gaps + +If Step 7 routed any component **agentic**, its pipelines' tasks are now `PlaceholderActivity` nodes +with one tagged gap each. Author the replacements and merge them (no LLM in the library — it validates +and merges) **before** the just-in-time config below, so the filled tasks get configuration-stamped +and packaged: + +- **Per-pipeline** (the pipeline stays 1:1, **ADF only**): author one result JSON per gap and merge + with `convert --source adf --merge-agentic --report .work/translation_report.json --agentic-results ` + (`--source adf` is mandatory; the convert phase exits 2 without it). The merge replaces, per result, + the **first task whose `name` matches `activity_name`** — it matches by name and does **not** check + the target is a placeholder, so make sure each name targets the intended agentic gap. Airflow's + `--merge-agentic` is disabled (exits 2) — for Airflow per-gap fills use the + **`flowx-resolve-airflow-gaps`** skill instead. - **Cross-pipeline COMBINE** (N pipelines → M, e.g. one Lakeflow Connect pipeline): author the replacement pipeline IR (typically with `AgenticComponentActivity` nodes) and run `fill-agentic combine --output-dir --members "" --pipelines-path `; the members must exactly match the routed-agentic component and the merged report is validated structurally before it is written. -Skip this step entirely when no component was routed agentic. +Skip this step entirely when no component was routed agentic. See the **`flowx-route`** skill for the +full fill details and the `AgenticComponentActivity` shape. -### Step 5.1 — Gather just-in-time translation configuration +### Step 9 — Gather just-in-time translation configuration Run `inspect` **once** to get the full option schema (every option carries a `show_when` condition), then drive the chain locally — ask an option only when its `show_when` clauses are all satisfied by @@ -373,7 +369,7 @@ incremental_load_watermark, metadata_driven_bulk_copy, ...) the adapter emits on `consolidate_motif:` option. The user must explicitly opt in to `consolidate` for each detected pattern. -### Step 6 — Checkpoint: confirm proceed to bundle generation +### Step 10 — Checkpoint: confirm proceed to bundle generation > Translation is 91.5% complete. 4 activities could not be translated automatically. > Options: @@ -383,7 +379,7 @@ for each detected pattern. > > What would you like to do? -### Step 6.5 — Detect workspace artifacts and authenticate +### Step 11 — Detect workspace artifacts and authenticate Before invoking the package phase, run the adapter's `workspace-paths` subcommand to detect any absolute workspace paths @@ -403,14 +399,14 @@ When the response carries `needs_auth: true`: services in the ADF export). 2. Run `databricks auth login --host ` interactively to set up a local profile. -3. Pass `--profile ` to the prepare invocation in Step 7 so +3. Pass `--profile ` to the prepare invocation in Step 12 so flowx downloads the referenced notebooks and downloads them under `bundle/src/notebooks/` with the task references rewritten to the relative `../src/notebooks/...` paths. Skip this step entirely when `needs_auth` is `false`. -### Step 7 — Phase 3: Package +### Step 12 — Phase 3: Package Invoke the `flowx:flowx-package` skill with: - Output dir: the same shared `` — package reads the stamped report from @@ -421,7 +417,7 @@ Invoke the `flowx:flowx-package` skill with: Package prunes the transient `/.work/` after a successful build, leaving the bundle (databricks.yml, resources/, src/, SETUP.md) plus the kept `metadata/` folder. -### Step 7.5 — (Optional) Persist coverage results and install a dashboard +### Step 13 — (Optional) Persist coverage results and install a dashboard When running with workspace auth (Genie Code or a configured profile), offer to record this run's migration coverage to a Unity Catalog table and optionally install a coverage dashboard. @@ -447,7 +443,7 @@ Creates and publishes an AI/BI coverage dashboard over the table and prints its auto-detect a SQL warehouse when `--warehouse-id` is omitted and degrade gracefully without workspace auth. See the `package` skill (Step 8) for details. -### Step 8 — Present final summary +### Step 14 — Present final summary Display the complete migration summary: @@ -486,7 +482,7 @@ Next Steps: 7. Verify job output and promote to staging/prod ``` -### Step 9 — Offer follow-up actions +### Step 15 — Offer follow-up actions Ask if the user wants to: 1. Validate the bundle now (`databricks bundle validate`) diff --git a/skills/flowx-route/SKILL.md b/skills/flowx-route/SKILL.md index f1e2ad35..73bb1f1f 100644 --- a/skills/flowx-route/SKILL.md +++ b/skills/flowx-route/SKILL.md @@ -170,7 +170,7 @@ resolved gap into an agentic-results directory: } ``` -Then merge (`task_key`/`depends_on` are inherited from the placeholder when omitted, preserving edges): +Then merge (`task_key`/`depends_on` are inherited from the matched task when omitted, preserving edges): ```bash "$PY" -m flowx.adapter convert --source adf --merge-agentic \ @@ -179,14 +179,18 @@ Then merge (`task_key`/`depends_on` are inherited from the placeholder when omit [--output ] # default: overwrite --report ``` -`--source` is **mandatory** for the convert phase — the phase runner exits 2 without it, even for -`--merge-agentic`. Use `--source adf` here (`merge_agentic` is **ADF-only**; an Airflow per-gap fill -would use `--source airflow`, but Airflow gaps normally go through the `flowx-resolve-airflow-gaps` -skill). This merge is **additive** — it replaces only the placeholder tasks in the existing report and -leaves every other pipeline byte-identical, so it never erases routing's edits. +This per-pipeline merge is **ADF-only**. `--source adf` is **mandatory** — the convert phase runner +exits 2 without it, and `--source airflow` is explicitly rejected here (Airflow's `--merge-agentic` +is disabled and exits 2). For **Airflow** per-gap fills, use the **`flowx-resolve-airflow-gaps`** +skill (the fingerprint-bound `resolve-agentic` workflow), not this command. + +The merge finds, per result, the **first task whose `name` matches `activity_name`** (recursing into +`IfCondition`/`ForEach`/`Switch` containers) and replaces it — it matches by name and does **not** +verify the target is a `PlaceholderActivity`, so make sure each `activity_name` targets the intended +agentic gap. Other pipelines and unmatched tasks are left untouched. MCP: `flowx(command="merge_agentic", parameters={"source": "adf", "report_path": ..., "agentic_results_dir": ..., "output_path": ...})`. -Placeholders are replaced in place, status → `translated`; exits non-zero if any result can't be +The matched task's status becomes `translated`; the command exits non-zero if any result can't be matched. ### 3b — Cross-pipeline COMBINE (N pipelines → M) From c37d830af5898e8f56c7f2f3d94f4f4d6d985ddc Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Thu, 17 Sep 2026 15:41:22 +0100 Subject: [PATCH 004/122] Fix merge status claim: merge sets no status field merge_agentic_results / _find_and_replace_task (ir_serde.py) replaces the matched task with the authored task definition verbatim and does not set or mutate any `status` field. Reword the flowx-route merge note to describe the in-place replacement accurately and drop the false "status becomes translated" claim, consistent with the name-match description above it. Skill-doc edit only; no src/ changes. make fmt + make test green. Co-authored-by: Isaac --- skills/flowx-route/SKILL.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/skills/flowx-route/SKILL.md b/skills/flowx-route/SKILL.md index 73bb1f1f..b6eb4bcd 100644 --- a/skills/flowx-route/SKILL.md +++ b/skills/flowx-route/SKILL.md @@ -190,7 +190,8 @@ verify the target is a `PlaceholderActivity`, so make sure each `activity_name` agentic gap. Other pipelines and unmatched tasks are left untouched. MCP: `flowx(command="merge_agentic", parameters={"source": "adf", "report_path": ..., "agentic_results_dir": ..., "output_path": ...})`. -The matched task's status becomes `translated`; the command exits non-zero if any result can't be +The matched task is replaced in place by the authored task definition (carrying whatever fields that +task defines; the merge itself sets no `status`); the command exits non-zero if any result can't be matched. ### 3b — Cross-pipeline COMBINE (N pipelines → M) From 3c74b276caf7227ef859c933c21ecd91388ca750 Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Thu, 17 Sep 2026 16:11:35 +0100 Subject: [PATCH 005/122] Scope routing/agentic-conversion as ADF-only; fix Airflow control-edge claim Airflow discovery does not yet emit control lineage or motifs (sources/airflow/discover.py), so it cannot form routing components, and convert --merge-agentic --source airflow is disabled (sources/airflow/convert.py). The route -> edit -> fill agentic-conversion flow is therefore ADF-only today; Airflow aligns via #63. - flowx-route: removed the false "Airflow (RunJob) emit control edges, source-neutral routing" claim; added an "ADF only (current scope)" note near the top directing Airflow agentic gaps to flowx-resolve-airflow-gaps. - flowx-migrate: added the ADF-only scope caveat to the Context intro and Step 7 (route); deterministic discover/convert/package still described as working for both sources. Skill-doc edits only; no src/ changes. make fmt + make test green. Co-authored-by: Isaac --- skills/flowx-migrate/SKILL.md | 11 +++++++++++ skills/flowx-route/SKILL.md | 12 ++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/skills/flowx-migrate/SKILL.md b/skills/flowx-migrate/SKILL.md index b09d2b50..aa1567ab 100644 --- a/skills/flowx-migrate/SKILL.md +++ b/skills/flowx-migrate/SKILL.md @@ -24,6 +24,12 @@ via Declarative Automation Bundles. This skill runs the full flow in sequence: d This is the top-level orchestration skill. It runs the full migration pipeline: +> **Scope:** deterministic discover → convert → package works for **both ADF and Airflow**. The +> **routing + agentic-conversion path (enrich → route → fill)** applies to **ADF today** — Airflow +> discovery does not yet emit control lineage or motifs, so it cannot form routing components, and +> `convert --merge-agentic --source airflow` is disabled. Airflow follows its own +> `flowx-resolve-airflow-gaps` path and aligns with routing via #63. + 1. **Discover** — Parse the source's definitions into a typed inventory 2. **Enrich** *(default)* — Author the agentic `insights` layer over the inventory and merge it (`flowx-enrich`); skippable for a deterministic-only pass @@ -277,6 +283,11 @@ For failures, suggest: ### Step 7 — Route the conversion (deterministic vs. agentic) +**ADF only today.** Routing (and the agentic fill in Step 8) applies to **ADF**. For **Airflow**, skip +Steps 7–8 and take the deterministic report straight to configuration/package; resolve any Airflow +agentic gaps with the **`flowx-resolve-airflow-gaps`** skill instead (Airflow discovery does not yet +emit control lineage or motifs — routing aligns via #63). + Routing reads the deterministic baseline report `.work/translation_report.json` (built in Step 5) and edits it. Invoke the **`flowx:flowx-route`** skill to present the per-connected-component recommendation, take the **customer's** decision (interactive, or an authored plan — the customer diff --git a/skills/flowx-route/SKILL.md b/skills/flowx-route/SKILL.md index b6eb4bcd..c1018aa4 100644 --- a/skills/flowx-route/SKILL.md +++ b/skills/flowx-route/SKILL.md @@ -25,6 +25,13 @@ Connect pipeline). It records the decision as a fingerprint-bound `metadata/conv edits the translation report so routed-agentic groups become placeholder gaps, and then you author the fill. +**ADF only (current scope).** The discover → enrich → route → edit → fill agentic-conversion flow is +supported for **ADF today**. Airflow is **not** yet wired for routing: its discovery does not emit +control lineage or motifs, so it cannot form multi-DAG routing components, and `convert +--merge-agentic --source airflow` is disabled. Airflow aligns with routing once #63 maps it onto the +shared discovery AST. For **Airflow agentic gaps today, use the `flowx-resolve-airflow-gaps` skill** +(the strict per-gap resolver) — not this routing flow. + **There is no LLM inside flowx.** The library computes the recommendation deterministically and only *validates and records* the decision and the authored fill — the same author → validate → merge contract `enrich` uses. This is additive and non-breaking: with **no recorded plan**, `convert` and @@ -49,8 +56,9 @@ placeholders, the only convert is the additive `convert --merge-agentic` fill Pipelines are grouped into weak/undirected **connected components** over the inventory's control lineage (`lineage.control_edges`), so mutually-referencing pipelines are decided together and a -caller/callee reference is never split across incompatible routes. Both ADF (`ExecutePipeline`) and -Airflow (`RunJob`) emit control edges, so routing is source-neutral. +caller/callee reference is never split across incompatible routes. ADF emits these control edges +(from `ExecutePipeline`) in discovery; Airflow discovery does not emit control lineage yet, which is +why routing is ADF-only today (see the scope note above). Run the **`setup`** skill first if you haven't. Everything below has an MCP-tool path (Genie Code, or a local stdio registration — call the single **`flowx`** tool, run no `python3`/`$PY`) and a venv-CLI From 05e2570a16468ba47044da694cb261d8b3ff1289 Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Fri, 18 Sep 2026 11:40:18 +0100 Subject: [PATCH 006/122] Routing/combine: source-tag check, idempotent combine, route audit, GA-grounded channel FIX 2: apply_combine_fill now asserts every agent-authored pipeline carries tags.source == 'adf' (routing/agentic is ADF-only). A missing/wrong tag fails the combine closed (nothing written) with a clear message, instead of being accepted at combine and only rejected later at package preflight. The required tag is now documented in the flowx-route AgenticComponentActivity / combine authoring example. FIX 3: Make combine idempotent. The recorded plan still lists a component's members after a prior combine collapsed them out of the report, so the plan/membership check passes on a re-run; a naive second run then re-appends the authored pipeline (duplicate). Detect the already-combined state (members gone from the report, authored pipeline(s) already present) and no-op (already_combined=true). Regression test: running combine twice with the same members + authored pipelines yields a single authored pipeline. FIX 5: Persist a compact metadata/route_audit.json (routed components + decisions + gaps introduced) before package prunes .work/, so the "what did routing change?" trail survives the default prune. Additive and non-breaking: it writes nothing when no conversion_plan.json was recorded, so the no-route path stays byte-identical. .work/ pruning is unchanged. FIX 8b: In flowx-route's fill authoring guidance, default the Lakeflow pipeline/Connect resource channel to 'current' (stable/GA); only use 'preview' with a cited GA-status justification + confirmed workspace availability. Require the same GA-vs-Preview verification/citation as flowx-enrich (FIX 8a) before recommending any connector / Lakeflow Connect pattern. Co-authored-by: Isaac --- skills/flowx-route/SKILL.md | 46 ++++++++++---- src/flowx/bundler/dab_writer.py | 70 +++++++++++++++++++++ src/flowx/route_agentic.py | 63 ++++++++++++++++++- tests/unit/test_cli_route_agentic.py | 1 + tests/unit/test_route_agentic_flow.py | 47 ++++++++++++++ tests/unit/test_route_audit.py | 91 +++++++++++++++++++++++++++ 6 files changed, 303 insertions(+), 15 deletions(-) create mode 100644 tests/unit/test_route_audit.py diff --git a/skills/flowx-route/SKILL.md b/skills/flowx-route/SKILL.md index c1018aa4..766a2466 100644 --- a/skills/flowx-route/SKILL.md +++ b/skills/flowx-route/SKILL.md @@ -227,6 +227,10 @@ MCP: `flowx(command="fill_agentic", parameters={"output_dir": ..., "members": [. — you can't swap pipelines the plan didn't route agentic. - `--pipelines-path` is a JSON **list** of pipeline IR dicts (the authored replacements), typically carrying `AgenticComponentActivity` nodes (see below). +- Each authored pipeline **must** carry the source tag `"tags": {"source": "adf"}` (routing/agentic + conversion is ADF-only). Combine asserts this up front and **fails closed** (nothing written, with + a clear message) on a missing or non-`adf` tag, so a mis-tagged pipeline is caught here rather than + surviving to the package preflight. - The merged report is **always** validated with the structural bundle invariants (a real `prepare → write_bundle` pass) before it is written — no bypass — so a duplicate key, dangling dependency, cycle, or dangling pipeline/run_job reference can never land on disk. On any violation, @@ -235,25 +239,34 @@ MCP: `flowx(command="fill_agentic", parameters={"output_dir": ..., "members": [. ### Authoring an `AgenticComponentActivity` (the escape hatch) When the target can't be expressed by the typed engine (e.g. a managed Lakeflow Connect ingestion -pipeline), emit an `AgenticComponentActivity` task inside the authored pipeline. It carries the raw -bundle components the package phase writes verbatim: +pipeline), emit an `AgenticComponentActivity` task inside the authored pipeline. The authored +**pipeline** carries the required `tags.source == "adf"`; the task carries the raw bundle components +the package phase writes verbatim: ```json { "name": "IngestAll", - "task_key": "ingest_all", - "type": "AgenticComponentActivity", - "files": [ - {"path": "src/ingest/lakeflow_connect.py", "content": "# authored pipeline source ..."} - ], - "resources": [ - {"resource_key": "ingest_all_pipeline", "definition": { ...raw pipeline resource... }} - ], - "task": {"pipeline_task": {"pipeline_id": "${resources.pipelines.ingest_all_pipeline.id}"}}, - "raw_definition": { ...original source definitions, retained for auditing... } + "tags": {"source": "adf"}, + "tasks": [ + { + "name": "IngestAll", + "task_key": "ingest_all", + "type": "AgenticComponentActivity", + "files": [ + {"path": "src/ingest/lakeflow_connect.py", "content": "# authored pipeline source ..."} + ], + "resources": [ + {"resource_key": "ingest_all_pipeline", "definition": { "channel": "current", "...": "raw pipeline resource" }} + ], + "task": {"pipeline_task": {"pipeline_id": "${resources.pipelines.ingest_all_pipeline.id}"}}, + "raw_definition": { "...": "original source definitions, retained for auditing" } + } + ] } ``` +- `tags` — the authored pipeline **must** set `tags.source` to `"adf"` (routing/agentic is ADF-only); + combine fails closed otherwise (see the combine guarantees above). - `files` — files written below the bundle `src/`; each is `path` + either UTF-8 `content` or base64 `binary_content`. - `resources` — bundle resources in the `resource_key` + raw `definition` shape the bundle writer @@ -262,6 +275,15 @@ bundle components the package phase writes verbatim: authored resource or file. - `raw_definition` — the original source definition, retained for provenance. +**Default the Lakeflow pipeline / Connect resource `channel` to `current` (stable/GA).** Do **not** +emit `channel: preview` by default. Only use `preview` with a cited GA-status justification and +confirmed workspace availability — and **verify GA-vs-Preview status before recommending any +connector or Lakeflow Connect pattern**: do not hardcode GA/Preview status or dates (release state +changes), check the feature's current release state **and** target-workspace availability against the +current public Databricks docs, and cite the source. Treat a **Private Preview** connector as +`doNotSuggest` unless the workspace has confirmed enrollment/entitlement. (Same grounding rule the +`flowx-enrich` skill applies when authoring recommended patterns.) + ## Step 4 — Continue to package Once the routed-agentic groups are filled and the report validates, continue with `flowx-convert`'s diff --git a/src/flowx/bundler/dab_writer.py b/src/flowx/bundler/dab_writer.py index 93b89335..d71c7fe0 100644 --- a/src/flowx/bundler/dab_writer.py +++ b/src/flowx/bundler/dab_writer.py @@ -378,6 +378,70 @@ def _default_report_path(output_dir: Path) -> Path: return work / "translation_report.json" +def _write_route_audit(output_dir: Path) -> Path | None: + """Persist a compact routing/gaps audit to ``metadata/route_audit.json`` before the prune. + + Package prunes the transient ``.work/`` folder (translation report + ``gaps.json``) by default, + which erases the "what did routing change?" trail. When a routing decision was recorded + (``metadata/conversion_plan.json`` exists), summarise the routed components, their decisions, and + the gaps routing introduced into an additive ``metadata/`` artifact that survives the prune. + + Returns the written path, or ``None`` when there is no recorded plan (no routing happened) -- so + the no-route path writes nothing and stays byte-identical. + """ + metadata_dir = Path(output_dir) / "metadata" + plan_path = metadata_dir / "conversion_plan.json" + if not plan_path.is_file(): + return None + try: + plan = json.loads(plan_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + if not isinstance(plan, dict): + return None + + components: list[dict[str, Any]] = [] + agentic_pipelines: list[str] = [] + for component in plan.get("components", []): + if not isinstance(component, dict): + continue + members = [str(member) for member in component.get("members", []) if isinstance(member, str)] + decision = component.get("decision") + components.append({"component_id": component.get("component_id"), "members": members, "decision": decision}) + if decision == "agentic": + agentic_pipelines.extend(members) + + gaps_introduced: list[dict[str, Any]] = [] + gaps_path = Path(output_dir) / ".work" / "gaps.json" + if gaps_path.is_file(): + try: + gaps = json.loads(gaps_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + gaps = [] + for gap in gaps if isinstance(gaps, list) else []: + if isinstance(gap, dict): + gaps_introduced.append( + { + "pipeline": gap.get("pipeline"), + "activity_name": gap.get("activity_name"), + "activity_type": gap.get("activity_type"), + } + ) + + audit = { + "schema": "flowx.route_audit/v1", + "recorded_against_inventory_sha256": plan.get("inventory_sha256"), + "components": components, + "agentic_pipelines": sorted(set(agentic_pipelines)), + "gaps_count": len(gaps_introduced), + "gaps_introduced": gaps_introduced, + } + metadata_dir.mkdir(parents=True, exist_ok=True) + audit_path = metadata_dir / "route_audit.json" + audit_path.write_text(json.dumps(audit, indent=2), encoding="utf-8") + return audit_path + + def main(argv: list[str] | None = None) -> int: """Package-phase entry point for DAB bundle generation. @@ -584,6 +648,12 @@ def main(argv: list[str] | None = None) -> int: print(format_result(result), file=sys.stderr) invariant_violations += len(result.violations) + # Persist the routing/gaps audit to metadata/ before pruning .work/, so the "what did routing + # change?" trail survives the default prune. No-ops (writes nothing) when no plan was recorded. + audit_path = _write_route_audit(args.output_dir) + if audit_path is not None: + print(f"Wrote routing audit to {audit_path}") + if not args.keep_intermediates: work_dir = args.output_dir / ".work" if work_dir.is_dir(): diff --git a/src/flowx/route_agentic.py b/src/flowx/route_agentic.py index 09cc7989..5772e8d8 100644 --- a/src/flowx/route_agentic.py +++ b/src/flowx/route_agentic.py @@ -50,6 +50,11 @@ METADATA_DIRNAME = "metadata" INVENTORY_FILENAME = "inventory.json" +# Routing / in-engine agentic conversion is ADF-only, so every agent-authored combine pipeline must +# carry this source tag. The package preflight enforces it too, but combine asserts it up front so a +# mis-tagged authored pipeline fails closed here (nothing written) instead of surviving to package. +REQUIRED_COMBINE_SOURCE_TAG = "adf" + # Guidance stamped onto every placeholder the alteration produces. _PLACEHOLDER_COMMENT = ( "Routed agentic by the conversion plan; author a replacement task (per-pipeline fill) or replace " @@ -354,6 +359,27 @@ def _resolve_agentic_component(output_dir: Path, members: set[str]) -> tuple[str ) +def _authored_source_tag_violations(authored_pipelines: list[dict[str, Any]]) -> list[str]: + """Reports each authored combine pipeline that is missing the required ``tags.source == 'adf'``. + + Routing / in-engine agentic conversion is ADF-only, so an authored combine pipeline that omits or + mis-sets the source tag is an authoring error. Catching it here fails the combine closed (nothing + written) with a clear message, rather than letting the mis-tagged pipeline reach the package + preflight where it is only rejected much later. + """ + violations: list[str] = [] + for index, pipeline in enumerate(authored_pipelines): + label = pipeline.get("name") if isinstance(pipeline, dict) and pipeline.get("name") else f"pipeline[{index}]" + tags = pipeline.get("tags") if isinstance(pipeline, dict) else None + source = tags.get("source") if isinstance(tags, dict) else None + if source != REQUIRED_COMBINE_SOURCE_TAG: + violations.append( + f"{label}: authored combine pipeline must carry tags.source == " + f"{REQUIRED_COMBINE_SOURCE_TAG!r}, got {source!r}" + ) + return violations + + def apply_combine_fill( output_dir: Path, group_members: Iterable[str], @@ -364,13 +390,21 @@ def apply_combine_fill( The group's membership is **bound to the recorded plan**: ``group_members`` must exactly match a routed-agentic component in ``metadata/conversion_plan.json`` (whose fingerprint must still match the current inventory), so a caller cannot swap deterministic pipelines or a partial/typoed group. + Every authored pipeline must carry ``tags.source == 'adf'`` (routing/agentic is ADF-only); a + mis-tagged pipeline fails the combine closed here rather than surviving to the package preflight. The merged report is then **always** validated with the structural bundle invariants (a real ``prepare -> write_bundle`` pass over :func:`validate_report_structurally`) -- there is no bypass -- and written back only when it passes, so a dangling reference or duplicate key never lands on disk. - Returns ``{"ok", "violations", "error", "component_id", "pipelines"}``. ``ok`` is ``False`` (and - nothing written) on a plan/membership error (``error`` set) or any structural violation - (``violations`` set). + The combine is **idempotent**: because the merged report no longer contains the collapsed members + (only the recorded plan still lists them), re-running with the same ``group_members`` and authored + pipeline(s) detects the already-combined state -- the members are gone from the report and the + authored pipeline(s) are already present -- and no-ops (``already_combined`` true) instead of + appending the authored pipeline(s) a second time. + + Returns ``{"ok", "violations", "error", "component_id", "pipelines", "already_combined"}``. ``ok`` + is ``False`` (and nothing written) on a plan/membership error (``error`` set), a missing source tag, + or any structural violation (``violations`` set). Raises: FileNotFoundError: when the translation report is missing (run convert first). @@ -380,12 +414,34 @@ def apply_combine_fill( if error is not None: return {"ok": False, "error": error, "violations": [], "pipelines": 0} + tag_violations = _authored_source_tag_violations(authored_pipelines) + if tag_violations: + return {"ok": False, "error": None, "violations": tag_violations, "pipelines": 0} + work = Path(output_dir) / WORK_DIRNAME report_path = work / REPORT_FILENAME if not report_path.exists(): raise FileNotFoundError(f"No {REPORT_FILENAME} under {work}; run the convert phase first.") report = json.loads(report_path.read_text(encoding="utf-8")) + + # Idempotency: the recorded plan still lists the members even after a prior combine collapsed them + # out of the report, so the plan/membership check above passes on a re-run. Detect the + # already-combined state (members gone from the report, authored pipeline(s) already present) and + # no-op, so a second run cannot append a duplicate authored pipeline. + report_pipelines = _report_pipelines(report) + report_names = {pipeline.get("name") for pipeline in report_pipelines} + authored_names = {pipeline.get("name") for pipeline in authored_pipelines} + if authored_names and not (members & report_names) and authored_names <= report_names: + return { + "ok": True, + "error": None, + "violations": [], + "component_id": component_id, + "pipelines": len(report_pipelines), + "already_combined": True, + } + merged = combine_group_fill(report, members, authored_pipelines) result = validate_report_structurally(merged) @@ -400,6 +456,7 @@ def apply_combine_fill( "violations": [], "component_id": component_id, "pipelines": len(merged["pipelines"]), + "already_combined": False, } diff --git a/tests/unit/test_cli_route_agentic.py b/tests/unit/test_cli_route_agentic.py index 302d0bb3..a43081eb 100644 --- a/tests/unit/test_cli_route_agentic.py +++ b/tests/unit/test_cli_route_agentic.py @@ -196,6 +196,7 @@ def _lfc_pipeline() -> dict[str, Any]: definition = {"name": "orders_ingestion", "catalog": "${var.catalog}", "target": "${var.schema}"} return { "name": "orders_lfc", + "tags": {"source": "adf"}, "tasks": [ { "name": "Ingest orders", diff --git a/tests/unit/test_route_agentic_flow.py b/tests/unit/test_route_agentic_flow.py index 29f9ea09..583843bb 100644 --- a/tests/unit/test_route_agentic_flow.py +++ b/tests/unit/test_route_agentic_flow.py @@ -108,6 +108,7 @@ def _lfc_pipeline() -> dict[str, Any]: } return { "name": "orders_lfc", + "tags": {"source": "adf"}, "tasks": [ { "name": "Ingest orders", @@ -463,6 +464,52 @@ def test_apply_combine_fill_rejects_and_does_not_write_on_dangling_reference(tmp assert (tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_bytes() == report_before +def test_combine_rejects_an_authored_pipeline_missing_the_source_tag(tmp_path: Path) -> None: + """FIX 2: an authored pipeline without tags.source == 'adf' fails closed at combine (nothing written).""" + _setup_routed_agentic(tmp_path, decision="agentic") + report_before = (tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_bytes() + untagged = _lfc_pipeline() + del untagged["tags"] + + result = apply_combine_fill(tmp_path, ["parent", "child"], [untagged]) + + assert result["ok"] is False + assert any("tags.source" in violation and "adf" in violation for violation in result["violations"]) + # Nothing is written when the source tag is missing. + assert (tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_bytes() == report_before + + +def test_combine_rejects_an_authored_pipeline_with_wrong_source_tag(tmp_path: Path) -> None: + """A non-'adf' source tag is rejected the same way (routing/agentic is ADF-only).""" + _setup_routed_agentic(tmp_path, decision="agentic") + mistagged = _lfc_pipeline() + mistagged["tags"] = {"source": "airflow"} + + result = apply_combine_fill(tmp_path, ["parent", "child"], [mistagged]) + + assert result["ok"] is False + assert any("tags.source" in violation for violation in result["violations"]) + + +def test_combine_is_idempotent_running_twice_yields_no_duplicate(tmp_path: Path) -> None: + """FIX 3: re-running combine with the same members + authored pipeline does not duplicate it.""" + _setup_routed_agentic(tmp_path, decision="agentic") + + first = apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()]) + assert first["ok"] is True + assert first["already_combined"] is False + + # The recorded plan still lists {parent, child}, so the plan/membership check passes again; the + # report, however, now contains only the authored pipeline. A naive re-run would re-append it. + second = apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()]) + assert second["ok"] is True + assert second["already_combined"] is True + + report = json.loads((tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_text(encoding="utf-8")) + names = [pipeline["name"] for pipeline in report["pipelines"]] + assert names == ["orders_lfc"] # exactly one authored pipeline, no duplicate + + # --------------------------------------------------------------------------- # # Combine membership is bound to the recorded, fingerprint-bound plan. # --------------------------------------------------------------------------- # diff --git a/tests/unit/test_route_audit.py b/tests/unit/test_route_audit.py new file mode 100644 index 00000000..09074d3d --- /dev/null +++ b/tests/unit/test_route_audit.py @@ -0,0 +1,91 @@ +"""Tests for the routing/gaps audit artifact the package phase persists (FIX 5). + +Package prunes the transient ``.work/`` folder by default, which erases the routing trail +(translation report + ``gaps.json``). ``_write_route_audit`` summarises the recorded plan's routed +components + decisions + the gaps routing introduced into ``metadata/route_audit.json`` so the trail +survives the prune. It must no-op (write nothing) when no plan was recorded, keeping the no-route +path byte-identical. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from flowx.bundler.dab_writer import _write_route_audit + + +def _write_plan(output_dir: Path) -> None: + metadata = output_dir / "metadata" + metadata.mkdir(parents=True, exist_ok=True) + plan = { + "schema_version": "1", + "inventory_sha256": "abc123", + "components": [ + {"component_id": "component-1", "members": ["parent", "child"], "decision": "agentic"}, + {"component_id": "component-2", "members": ["solo"], "decision": "deterministic"}, + ], + } + (metadata / "conversion_plan.json").write_text(json.dumps(plan, indent=2), encoding="utf-8") + + +def _write_gaps(output_dir: Path) -> None: + work = output_dir / ".work" + work.mkdir(parents=True, exist_ok=True) + gaps = [ + { + "activity_name": "Extract", + "activity_type": "CopyActivity", + "raw_definition": {"name": "Extract", "type": "CopyActivity"}, + "pipeline": "parent", + }, + { + "activity_name": "Load", + "activity_type": "NotebookActivity", + "raw_definition": {"name": "Load"}, + "pipeline": "child", + }, + ] + (work / "gaps.json").write_text(json.dumps(gaps, indent=2), encoding="utf-8") + + +def test_route_audit_summarises_components_decisions_and_gaps(tmp_path: Path) -> None: + _write_plan(tmp_path) + _write_gaps(tmp_path) + + audit_path = _write_route_audit(tmp_path) + + assert audit_path == tmp_path / "metadata" / "route_audit.json" + audit = json.loads(audit_path.read_text(encoding="utf-8")) + assert audit["recorded_against_inventory_sha256"] == "abc123" + # Both routed components and their decisions are recorded. + decisions = {component["component_id"]: component["decision"] for component in audit["components"]} + assert decisions == {"component-1": "agentic", "component-2": "deterministic"} + # Only the agentic component's members are surfaced as agentic pipelines. + assert audit["agentic_pipelines"] == ["child", "parent"] + # The gaps introduced survive as a compact summary (no verbose raw_definition). + assert audit["gaps_count"] == 2 + assert {gap["pipeline"] for gap in audit["gaps_introduced"]} == {"parent", "child"} + assert all("raw_definition" not in gap for gap in audit["gaps_introduced"]) + + +def test_route_audit_noops_without_a_recorded_plan(tmp_path: Path) -> None: + # No conversion_plan.json -> no routing happened -> nothing is written (no-route path stays clean). + (tmp_path / "metadata").mkdir(parents=True, exist_ok=True) + + audit_path = _write_route_audit(tmp_path) + + assert audit_path is None + assert not (tmp_path / "metadata" / "route_audit.json").exists() + + +def test_route_audit_handles_missing_gaps_file(tmp_path: Path) -> None: + # A recorded plan but no gaps.json (e.g. all-deterministic route) still writes an audit with zero gaps. + _write_plan(tmp_path) + + audit_path = _write_route_audit(tmp_path) + + assert audit_path is not None + audit = json.loads(audit_path.read_text(encoding="utf-8")) + assert audit["gaps_count"] == 0 + assert audit["gaps_introduced"] == [] From cbcd2156d5cbdb3b90f40773ef87e99bafbd1c0c Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Fri, 18 Sep 2026 12:42:33 +0100 Subject: [PATCH 007/122] Combine idempotency: key on recorded report state + fingerprint, not name sets Cross-review BLOCKING 1: the prior FIX 3 detection inferred "already combined" from pipeline-name sets, which broke two ways: (a) re-running combine with the SAME --members but a differently-named authored replacement fell through to combine_group_fill (members already gone, so nothing dropped) and appended a second authored pipeline -> duplicate; (b) an authored pipeline whose name collided with a former member defeated the detection entirely. Fix: each successful combine now stamps an additive `_combine_provenance` marker onto the report recording (component_id, inventory_sha256). Idempotency keys off that recorded state -- never off comparing authored names to member names -- so a re-run is detected regardless of how the authored replacement is named or whether a name collides. The marker lives in the report, so a fresh convert (which rewrites the report) clears it and combine re-applies after a genuine re-convert. The marker is an extra top-level report key: ignored by the package phase (not a recognized report shape key) and by ir_serde (which reads per-pipeline IR, not the report wrapper), so the convert->package contract shape is unchanged. Regression tests cover both failure modes: (a) a differently-named authored re-run does not duplicate, and (b) an authored name colliding with a former member is still detected as already-combined. Co-authored-by: Isaac --- src/flowx/route_agentic.py | 117 ++++++++++++++++++-------- tests/unit/test_route_agentic_flow.py | 50 +++++++++++ 2 files changed, 133 insertions(+), 34 deletions(-) diff --git a/src/flowx/route_agentic.py b/src/flowx/route_agentic.py index 5772e8d8..d21f30b0 100644 --- a/src/flowx/route_agentic.py +++ b/src/flowx/route_agentic.py @@ -55,6 +55,15 @@ # mis-tagged authored pipeline fails closed here (nothing written) instead of surviving to package. REQUIRED_COMBINE_SOURCE_TAG = "adf" +# Additive top-level marker stamped onto the report each time a combine is applied, recording the +# (component_id, inventory fingerprint) of every applied combine. Idempotency keys off this recorded +# state -- not off comparing authored pipeline names to member names -- so a re-run is detected no +# matter what the authored replacement is named or whether a name collides with a former member. It +# lives *in* the report, so a fresh convert (which rewrites the report) naturally clears it, and it is +# ignored by the package phase (not one of the recognized report shape keys) and by ir_serde (which +# reads per-pipeline IR, not the report wrapper). +_COMBINE_PROVENANCE_KEY = "_combine_provenance" + # Guidance stamped onto every placeholder the alteration produces. _PLACEHOLDER_COMMENT = ( "Routed agentic by the conversion plan; author a replacement task (per-pipeline fill) or replace " @@ -310,16 +319,17 @@ def combine_group_fill( return {"pipelines": kept} -def _resolve_agentic_component(output_dir: Path, members: set[str]) -> tuple[str | None, str | None]: +def _resolve_agentic_component(output_dir: Path, members: set[str]) -> tuple[str | None, str | None, str | None]: """Bind ``members`` to a routed-**agentic** component in the recorded, fingerprint-bound plan. Reads ``metadata/conversion_plan.json`` and ``metadata/inventory.json`` and returns - ``(component_id, error)``. The error is set (and ``component_id`` is ``None``) when: the plan or - inventory is missing; the plan's ``inventory_sha256`` no longer matches the current inventory (a - stale plan); ``members`` do not exactly equal one component's members (a partial, superset, or - mistyped group); or the exactly-matching component is routed deterministic rather than agentic. - Requiring an exact match to a routed-agentic component stops a caller from swapping deterministic - pipelines or a partial group. + ``(component_id, inventory_fingerprint, error)``. The error is set (and the other two are ``None``) + when: the plan or inventory is missing; the plan's ``inventory_sha256`` no longer matches the + current inventory (a stale plan); ``members`` do not exactly equal one component's members (a + partial, superset, or mistyped group); or the exactly-matching component is routed deterministic + rather than agentic. Requiring an exact match to a routed-agentic component stops a caller from + swapping deterministic pipelines or a partial group. The fingerprint is returned so the combine can + record its provenance keyed on the exact inventory it was applied against. """ from flowx.discovery_insights import inventory_fingerprint @@ -327,18 +337,22 @@ def _resolve_agentic_component(output_dir: Path, members: set[str]) -> tuple[str plan_path = metadata / "conversion_plan.json" inventory_path = metadata / INVENTORY_FILENAME if not plan_path.exists(): - return None, "No metadata/conversion_plan.json; record a routing decision with `route` first." + return None, None, "No metadata/conversion_plan.json; record a routing decision with `route` first." if not inventory_path.exists(): - return None, "No metadata/inventory.json; run the discover phase first." + return None, None, "No metadata/inventory.json; run the discover phase first." plan = json.loads(plan_path.read_text(encoding="utf-8")) inventory = json.loads(inventory_path.read_text(encoding="utf-8")) recorded_fingerprint = plan.get("inventory_sha256") current_fingerprint = inventory_fingerprint(inventory) if recorded_fingerprint != current_fingerprint: - return None, ( - "conversion_plan.json is stale: it was recorded against a different inventory " - f"({recorded_fingerprint!r} != {current_fingerprint!r}); re-run `route` before filling." + return ( + None, + None, + ( + "conversion_plan.json is stale: it was recorded against a different inventory " + f"({recorded_fingerprint!r} != {current_fingerprint!r}); re-run `route` before filling." + ), ) for component in plan.get("components", []): @@ -348,14 +362,22 @@ def _resolve_agentic_component(output_dir: Path, members: set[str]) -> tuple[str if component_members != members: continue if component.get("decision") == DECISION_AGENTIC: - return str(component.get("component_id")), None - return None, ( - f"members {sorted(members)} match component {component.get('component_id')!r}, which is " - f"routed {component.get('decision')!r}, not agentic; only routed-agentic components can be combined." + return str(component.get("component_id")), current_fingerprint, None + return ( + None, + None, + ( + f"members {sorted(members)} match component {component.get('component_id')!r}, which is " + f"routed {component.get('decision')!r}, not agentic; only routed-agentic components can be combined." + ), ) - return None, ( - f"members {sorted(members)} do not exactly match any component in the recorded plan " - "(partial, superset, or mistyped); pass the exact member set of one routed-agentic component." + return ( + None, + None, + ( + f"members {sorted(members)} do not exactly match any component in the recorded plan " + "(partial, superset, or mistyped); pass the exact member set of one routed-agentic component." + ), ) @@ -380,6 +402,25 @@ def _authored_source_tag_violations(authored_pipelines: list[dict[str, Any]]) -> return violations +def _combine_already_applied(report: dict[str, Any], component_id: str, fingerprint: str) -> bool: + """Report whether this component's combine (at this inventory fingerprint) is already recorded. + + Reads the additive ``_combine_provenance`` marker stamped onto the report by a prior combine. + Detection is keyed purely on ``(component_id, fingerprint)`` -- never on comparing authored + pipeline names to member names -- so a re-run is recognised as already-combined no matter what the + authored replacement is named, and even when an authored name collides with a former member. + """ + provenance = report.get(_COMBINE_PROVENANCE_KEY) if isinstance(report, dict) else None + if not isinstance(provenance, list): + return False + return any( + isinstance(entry, dict) + and entry.get("component_id") == component_id + and entry.get("inventory_sha256") == fingerprint + for entry in provenance + ) + + def apply_combine_fill( output_dir: Path, group_members: Iterable[str], @@ -396,11 +437,13 @@ def apply_combine_fill( ``prepare -> write_bundle`` pass over :func:`validate_report_structurally`) -- there is no bypass -- and written back only when it passes, so a dangling reference or duplicate key never lands on disk. - The combine is **idempotent**: because the merged report no longer contains the collapsed members - (only the recorded plan still lists them), re-running with the same ``group_members`` and authored - pipeline(s) detects the already-combined state -- the members are gone from the report and the - authored pipeline(s) are already present -- and no-ops (``already_combined`` true) instead of - appending the authored pipeline(s) a second time. + The combine is **idempotent**, keyed on recorded report state rather than pipeline names. Each + successful combine stamps a ``_combine_provenance`` entry -- ``(component_id, inventory + fingerprint)`` -- onto the report. A re-run detects that stamp and no-ops (``already_combined`` + true) instead of collapsing/appending again, so it never duplicates the authored pipeline(s) even + when the authored replacement is renamed, and it still recognises the already-combined state when + an authored pipeline's name collides with a former member. A fresh ``convert`` rewrites the report + without the marker, so the combine will re-apply after a genuine re-convert. Returns ``{"ok", "violations", "error", "component_id", "pipelines", "already_combined"}``. ``ok`` is ``False`` (and nothing written) on a plan/membership error (``error`` set), a missing source tag, @@ -410,9 +453,10 @@ def apply_combine_fill( FileNotFoundError: when the translation report is missing (run convert first). """ members = {str(member) for member in group_members} - component_id, error = _resolve_agentic_component(output_dir, members) + component_id, fingerprint, error = _resolve_agentic_component(output_dir, members) if error is not None: return {"ok": False, "error": error, "violations": [], "pipelines": 0} + assert component_id is not None and fingerprint is not None # guaranteed when error is None tag_violations = _authored_source_tag_violations(authored_pipelines) if tag_violations: @@ -425,24 +469,29 @@ def apply_combine_fill( report = json.loads(report_path.read_text(encoding="utf-8")) - # Idempotency: the recorded plan still lists the members even after a prior combine collapsed them - # out of the report, so the plan/membership check above passes on a re-run. Detect the - # already-combined state (members gone from the report, authored pipeline(s) already present) and - # no-op, so a second run cannot append a duplicate authored pipeline. - report_pipelines = _report_pipelines(report) - report_names = {pipeline.get("name") for pipeline in report_pipelines} - authored_names = {pipeline.get("name") for pipeline in authored_pipelines} - if authored_names and not (members & report_names) and authored_names <= report_names: + # Idempotency: a prior combine stamps _combine_provenance onto the report for this + # (component_id, fingerprint). Detecting that recorded state -- not the authored/member name sets + # -- means a re-run no-ops regardless of how the authored replacement is named or whether a name + # collides with a former member, so it can never append a duplicate authored pipeline. + if _combine_already_applied(report, component_id, fingerprint): return { "ok": True, "error": None, "violations": [], "component_id": component_id, - "pipelines": len(report_pipelines), + "pipelines": len(_report_pipelines(report)), "already_combined": True, } merged = combine_group_fill(report, members, authored_pipelines) + # Carry any prior provenance forward (combine_group_fill returns only ``pipelines``) and record + # this combine so a later re-run detects it. + prior_provenance = report.get(_COMBINE_PROVENANCE_KEY) if isinstance(report, dict) else None + provenance = ( + [entry for entry in prior_provenance if isinstance(entry, dict)] if isinstance(prior_provenance, list) else [] + ) + provenance.append({"component_id": component_id, "inventory_sha256": fingerprint, "members": sorted(members)}) + merged[_COMBINE_PROVENANCE_KEY] = provenance result = validate_report_structurally(merged) if not result.ok: diff --git a/tests/unit/test_route_agentic_flow.py b/tests/unit/test_route_agentic_flow.py index 583843bb..65e5c0ac 100644 --- a/tests/unit/test_route_agentic_flow.py +++ b/tests/unit/test_route_agentic_flow.py @@ -122,6 +122,13 @@ def _lfc_pipeline() -> dict[str, Any]: } +def _named_lfc_pipeline(name: str) -> dict[str, Any]: + """A correctly-tagged authored combine pipeline with a caller-chosen ``name``.""" + pipeline = _lfc_pipeline() + pipeline["name"] = name + return pipeline + + def _write_work(output_dir: Path, report: dict[str, Any], gaps: list[dict[str, Any]] | None = None) -> None: work = output_dir / WORK_DIRNAME work.mkdir(parents=True, exist_ok=True) @@ -510,6 +517,49 @@ def test_combine_is_idempotent_running_twice_yields_no_duplicate(tmp_path: Path) assert names == ["orders_lfc"] # exactly one authored pipeline, no duplicate +def test_combine_idempotent_with_a_differently_named_authored_pipeline(tmp_path: Path) -> None: + """FIX 3 (a): a second combine whose authored replacement is renamed must NOT duplicate. + + Name-set matching would fail here (the new name is absent from the report, the old members are + already gone), fall through, and append a second authored pipeline. Provenance keyed on + (component_id, fingerprint) catches the re-run regardless of the authored name. + """ + _setup_routed_agentic(tmp_path, decision="agentic") + + first = apply_combine_fill(tmp_path, ["parent", "child"], [_named_lfc_pipeline("orders_lfc")]) + assert first["ok"] is True and first["already_combined"] is False + + # Same members, but the authored replacement is named differently this time. + second = apply_combine_fill(tmp_path, ["parent", "child"], [_named_lfc_pipeline("orders_lfc_v2")]) + assert second["ok"] is True + assert second["already_combined"] is True + + report = json.loads((tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_text(encoding="utf-8")) + names = [pipeline["name"] for pipeline in report["pipelines"]] + assert names == ["orders_lfc"] # first authored pipeline kept; the renamed re-run added nothing + + +def test_combine_idempotent_when_authored_name_collides_with_a_former_member(tmp_path: Path) -> None: + """FIX 3 (b): an authored name colliding with a former member is still detected as already-combined. + + After the first combine the report holds a pipeline named 'parent' (a former member). Name-based + detection would see 'parent' present and conclude the combine had not happened; provenance keeps + the detection correct and independent of names. + """ + _setup_routed_agentic(tmp_path, decision="agentic") + + first = apply_combine_fill(tmp_path, ["parent", "child"], [_named_lfc_pipeline("parent")]) + assert first["ok"] is True and first["already_combined"] is False + + second = apply_combine_fill(tmp_path, ["parent", "child"], [_named_lfc_pipeline("parent")]) + assert second["ok"] is True + assert second["already_combined"] is True + + report = json.loads((tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_text(encoding="utf-8")) + names = [pipeline["name"] for pipeline in report["pipelines"]] + assert names == ["parent"] # exactly one authored pipeline, no duplicate + + # --------------------------------------------------------------------------- # # Combine membership is bound to the recorded, fingerprint-bound plan. # --------------------------------------------------------------------------- # From c061f5d54aacf3fbe7855baf40aec700ce81d544 Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Fri, 18 Sep 2026 16:28:12 +0100 Subject: [PATCH 008/122] Routing: surface recommended-pattern release_state as a neutral disclosure Mirror the per-pattern release_state / release_state_source onto AgenticPattern (they already ride into the agentic option via the {**pattern} spread; this makes the parity explicit and typed). Add a neutral, per-pattern release_disclosures list to AgenticOption: one entry per recommended pattern whose state is disclosed (public_preview / private_preview / beta), carrying the pipeline, pattern, state, a factual label, and a human-readable message. It is pure factual labelling, not a warning: - ga / unknown -> silent (no entry; unknown is treated exactly like ga); - public_preview -> label "Public Preview (production-ready)"; - private_preview -> label "Private Preview"; - beta -> label "Beta". The route skill points its channel guidance at the structured release_state and reflects the neutral labels. The separate recommendation-eligibility rule (treat Private Preview as doNotSuggest without confirmed enrollment) and the verify-before-recommend grounding rule are unchanged. Co-authored-by: Isaac --- skills/flowx-route/SKILL.md | 28 ++++-- src/flowx/models/conversion_plan.py | 24 ++++- src/flowx/routing.py | 58 +++++++++++- tests/unit/test_routing_recommend.py | 127 ++++++++++++++++++++++++++- 4 files changed, 225 insertions(+), 12 deletions(-) diff --git a/skills/flowx-route/SKILL.md b/skills/flowx-route/SKILL.md index 766a2466..70fdd130 100644 --- a/skills/flowx-route/SKILL.md +++ b/skills/flowx-route/SKILL.md @@ -276,13 +276,27 @@ the package phase writes verbatim: - `raw_definition` — the original source definition, retained for provenance. **Default the Lakeflow pipeline / Connect resource `channel` to `current` (stable/GA).** Do **not** -emit `channel: preview` by default. Only use `preview` with a cited GA-status justification and -confirmed workspace availability — and **verify GA-vs-Preview status before recommending any -connector or Lakeflow Connect pattern**: do not hardcode GA/Preview status or dates (release state -changes), check the feature's current release state **and** target-workspace availability against the -current public Databricks docs, and cite the source. Treat a **Private Preview** connector as -`doNotSuggest` unless the workspace has confirmed enrollment/entitlement. (Same grounding rule the -`flowx-enrich` skill applies when authoring recommended patterns.) +emit `channel: preview` by default. Let the enriched insight's **structured `release_state`** drive +the channel — the routing recommendation exposes each agentic option's states as a neutral +`release_disclosures` list (factual labels, no alarm). Surface the state as disclosure, tiered as: + +- `ga` — `channel: current`; **silent** (not surfaced). +- `unknown` — **silent**, treated exactly like `ga` (no separate label — we can't distinguish them). +- `public_preview` — the informational label **"Public Preview (production-ready)"**: generally + production-ready and supported per Databricks; still confirm workspace availability. Emit + `channel: preview` only with the cited `release_state_source`. +- `private_preview` — the plain factual label **"Private Preview"**. Emit `channel: preview` only + with the cited `release_state_source`. +- `beta` — the plain factual label **"Beta"**. Emit `channel: preview` only with the cited + `release_state_source`. + +Separately from *how the state is surfaced*, keep the recommendation-eligibility rule: treat a +**Private Preview** connector as `doNotSuggest` unless the workspace has confirmed +enrollment/entitlement. And **verify GA-vs-Preview status before recommending any connector or +Lakeflow Connect pattern**: do not hardcode GA/Preview status or dates (release state changes), check +the feature's current release state **and** target-workspace availability against the current public +Databricks docs, and cite the source. (Same structured grounding rule the `flowx-enrich` skill +applies when authoring recommended patterns.) ## Step 4 — Continue to package diff --git a/src/flowx/models/conversion_plan.py b/src/flowx/models/conversion_plan.py index bd0dc7ae..9f568576 100644 --- a/src/flowx/models/conversion_plan.py +++ b/src/flowx/models/conversion_plan.py @@ -18,7 +18,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Any +from typing import Any, Literal # The plan schema version stamped onto the recorded artifact. Bump on any backwards-incompatible # change to the recorded shape. @@ -29,6 +29,12 @@ DECISION_AGENTIC = "agentic" DECISIONS: tuple[str, ...] = (DECISION_DETERMINISTIC, DECISION_AGENTIC) +# Recommended-pattern release states surfaced as a neutral disclosure label on the agentic option +# (:attr:`AgenticOption.release_disclosures`). ``"ga"`` and ``"unknown"`` are deliberately **silent** +# -- they contribute no entry, and ``"unknown"`` is treated exactly like ``"ga"`` (we do not surface +# or distinguish it). This is factual labelling, never a warning or an alarm. +DISCLOSED_RELEASE_STATES: tuple[str, ...] = ("public_preview", "private_preview", "beta") + @dataclass(slots=True, kw_only=True) class DeterministicOption: @@ -62,12 +68,20 @@ class AgenticPattern: fit: One line on why it fits / what it replaces. simplification_pattern: ``True`` when the pattern uses a distinctive capability that collapses a whole legacy pattern (e.g. a multi-pipeline -> Lakeflow Connect re-architecture). + release_state: The pattern's verified Databricks GA/Preview release state, carried verbatim + from the insight (one of :data:`~flowx.models.insights.RELEASE_STATES`, or ``None`` when + the insight left it unstated). Drives the neutral release-state disclosure on + :class:`AgenticOption`. + release_state_source: The doc URL / citation grounding :attr:`release_state`, carried verbatim + from the insight; ``None`` when unstated. """ pipeline: str pattern: str fit: str simplification_pattern: bool + release_state: Literal["ga", "public_preview", "private_preview", "beta", "unknown"] | None = None + release_state_source: str | None = None @dataclass(slots=True, kw_only=True) @@ -79,10 +93,18 @@ class AgenticOption: each tagged with its pipeline. Empty when the inventory carries no insights. has_simplification: ``True`` when any recommended pattern is a ``simplification_pattern`` -- surfaced prominently so the user sees a re-architecture option, not a buried sub-key. + release_disclosures: A neutral, per-pattern **disclosure** of any non-silent ``release_state`` + -- one entry per recommended pattern whose state is in :data:`DISCLOSED_RELEASE_STATES`, + carrying the ``pipeline``, ``pattern``, ``release_state``, and a factual ``label`` + (``public_preview`` labelled "Public Preview (production-ready)"; ``private_preview`` / + ``beta`` stated as the plain labels "Private Preview" / "Beta"). ``"ga"`` and ``"unknown"`` + are **silent** -- they add nothing (``"unknown"`` is treated exactly like ``"ga"``). Empty + when nothing needs disclosing. Factual labelling, never a warning or an alarm. """ recommended_patterns: list[AgenticPattern] = field(default_factory=list) has_simplification: bool = False + release_disclosures: list[dict[str, Any]] = field(default_factory=list) @dataclass(slots=True, kw_only=True) diff --git a/src/flowx/routing.py b/src/flowx/routing.py index a6f318cf..be2ce9d3 100644 --- a/src/flowx/routing.py +++ b/src/flowx/routing.py @@ -236,12 +236,55 @@ def _deterministic_option(members: list[str], inventory: dict[str, Any]) -> dict } +# Neutral disclosure labels for the agentic option, keyed by release state. ``"ga"`` and ``"unknown"`` +# carry NO entry -- they are silent (we do not surface them, and ``"unknown"`` is treated exactly like +# ``"ga"``). ``"public_preview"`` is labelled production-ready per Databricks; ``"private_preview"`` / +# ``"beta"`` are stated as plain factual labels. This is disclosure, not a warning. +_RELEASE_STATE_LABELS: dict[str, str] = { + "public_preview": "Public Preview (production-ready)", + "private_preview": "Private Preview", + "beta": "Beta", +} + + +def _release_disclosure(pattern: dict[str, Any]) -> dict[str, Any] | None: + """Build a neutral release-state disclosure for one recommended pattern, or ``None``. + + Returns ``None`` for ``"ga"`` / ``"unknown"`` (both silent -- ``"unknown"`` is treated exactly like + ``"ga"``) and for a pattern with no ``release_state``. ``"public_preview"`` / ``"private_preview"`` + / ``"beta"`` each yield a factual state ``label`` -- ``public_preview`` noted as production-ready -- + with no warning framing. The cited source is appended when the insight carried one. + """ + release_state = pattern.get("release_state") + label = _RELEASE_STATE_LABELS.get(release_state) if isinstance(release_state, str) else None + if label is None: + return None + message = f"'{pattern.get('pattern')}' (pipeline '{pattern.get('pipeline')}'): {label}." + source = pattern.get("release_state_source") + if isinstance(source, str) and source.strip(): + message = f"{message} Source: {source.strip()}" + return { + "pipeline": pattern.get("pipeline"), + "pattern": pattern.get("pattern"), + "release_state": release_state, + "label": label, + "message": message, + } + + def _agentic_option(members: list[str], inventory: dict[str, Any]) -> dict[str, Any]: """Surface the agent-authored recommended patterns for a component as a first-class option. Draws every ``recommended_patterns`` entry from the member pipelines' insights, tagging each with - its pipeline, and flags whether any is a ``simplification_pattern`` (a distinctive re-architecture - such as a multi-pipeline -> Lakeflow Connect collapse) so the user sees it prominently. + its pipeline (the per-pattern ``release_state`` / ``release_state_source`` ride along in the + ``{**pattern}`` spread), and computes two signals for the user: + + * ``has_simplification`` -- any pattern is a ``simplification_pattern`` (a distinctive + re-architecture such as a multi-pipeline -> Lakeflow Connect collapse), surfaced prominently. + * ``release_disclosures`` -- a neutral, per-pattern disclosure of any non-silent ``release_state`` + (:data:`~flowx.models.conversion_plan.DISCLOSED_RELEASE_STATES`): ``public_preview`` labelled + production-ready, ``private_preview`` / ``beta`` stated as plain labels. ``ga`` and ``unknown`` + add nothing (silent). Factual labelling, not a warning. """ insights_by_pipeline = _pipeline_insights(inventory) recommended_patterns: list[dict[str, Any]] = [] @@ -253,7 +296,16 @@ def _agentic_option(members: list[str], inventory: dict[str, Any]) -> dict[str, if isinstance(pattern, dict): recommended_patterns.append({"pipeline": pipeline, **pattern}) has_simplification = any(pattern.get("simplification_pattern") for pattern in recommended_patterns) - return {"recommended_patterns": recommended_patterns, "has_simplification": has_simplification} + release_disclosures: list[dict[str, Any]] = [] + for pattern in recommended_patterns: + disclosure = _release_disclosure(pattern) + if disclosure is not None: + release_disclosures.append(disclosure) + return { + "recommended_patterns": recommended_patterns, + "has_simplification": has_simplification, + "release_disclosures": release_disclosures, + } def recommend_component(members: list[str], inventory: dict[str, Any]) -> tuple[str, dict[str, Any]]: diff --git a/tests/unit/test_routing_recommend.py b/tests/unit/test_routing_recommend.py index 43369b7d..a2725386 100644 --- a/tests/unit/test_routing_recommend.py +++ b/tests/unit/test_routing_recommend.py @@ -167,7 +167,132 @@ def test_agentic_option_surfaces_insight_patterns_with_simplification_prominent( def test_agentic_option_empty_when_no_insights() -> None: graphs = [SourceGraph(name="a", source="unit", tasks=[_node("load", "Notebook")])] _, options = recommend_component(["a"], _inventory(graphs)) - assert options["agentic"] == {"recommended_patterns": [], "has_simplification": False} + assert options["agentic"] == { + "recommended_patterns": [], + "has_simplification": False, + "release_disclosures": [], + } + + +# --------------------------------------------------------------------------- # +# Agentic option: neutral GA/Preview release-state disclosure (no warnings). +# --------------------------------------------------------------------------- # + + +def _insights_one_pattern(pattern: dict[str, Any]) -> dict[str, Any]: + """An insights block with a single recommended pattern on pipeline 'a'.""" + return {"pipeline_insights": [{"pipeline": "a", "recommended_patterns": [pattern]}]} + + +def test_agentic_option_ga_and_unknown_are_silent() -> None: + # 'ga' and 'unknown' both contribute NO disclosure entry -- 'unknown' is treated exactly like 'ga'. + for state in ("ga", "unknown"): + graphs = [SourceGraph(name="a", source="unit", tasks=[_node("load", "Notebook")])] + insights = _insights_one_pattern( + { + "pattern": "Lakeflow Job", + "fit": "Native orchestration", + "simplification_pattern": False, + "release_state": state, + } + ) + _, options = recommend_component(["a"], _inventory(graphs, insights=insights)) + assert options["agentic"]["release_disclosures"] == [], f"{state!r} must be silent" + + +def test_agentic_option_public_preview_is_disclosed_as_production_ready() -> None: + graphs = [SourceGraph(name="a", source="unit", tasks=[_node("load", "Notebook")])] + insights = _insights_one_pattern( + { + "pattern": "Lakeflow Connect", + "fit": "Managed ingestion", + "simplification_pattern": True, + "release_state": "public_preview", + "release_state_source": "https://docs.databricks.com/ingestion/lakeflow-connect/", + } + ) + _, options = recommend_component(["a"], _inventory(graphs, insights=insights)) + disclosures = options["agentic"]["release_disclosures"] + assert len(disclosures) == 1 + disclosure = disclosures[0] + assert disclosure["release_state"] == "public_preview" + assert disclosure["label"] == "Public Preview (production-ready)" + assert disclosure["pipeline"] == "a" + # A neutral disclosure -- no warning/severity framing. + assert "severity" not in disclosure + # The cited source rides along into the human-readable disclosure. + assert "docs.databricks.com" in disclosure["message"] + + +def test_agentic_option_private_preview_and_beta_are_plain_labels() -> None: + for state, expected_label in (("private_preview", "Private Preview"), ("beta", "Beta")): + graphs = [SourceGraph(name="a", source="unit", tasks=[_node("load", "Notebook")])] + insights = _insights_one_pattern( + { + "pattern": "Some capability", + "fit": "A capability", + "simplification_pattern": False, + "release_state": state, + "release_state_source": "https://docs.databricks.com/some-feature/", + } + ) + _, options = recommend_component(["a"], _inventory(graphs, insights=insights)) + disclosures = options["agentic"]["release_disclosures"] + assert len(disclosures) == 1 + assert disclosures[0]["label"] == expected_label + assert disclosures[0]["release_state"] == state + # Plain factual label -- no warning/severity field. + assert "severity" not in disclosures[0] + + +def test_agentic_option_discloses_each_non_silent_pattern() -> None: + # A component with a public-preview pattern and a private-preview pattern discloses BOTH as plain + # neutral labels (public preview noted production-ready); a ga/unknown pattern would add nothing. + graphs = [ + SourceGraph( + name="parent", + source="unit", + tasks=[_node("call_child", "ExecutePipeline")], + lineage=Lineage( + control_edges=[ + ControlEdge(source_workflow="parent", target_workflow="child", via_task_key="call_child") + ] + ), + ), + SourceGraph(name="child", source="unit", tasks=[_node("load", "Notebook")]), + ] + insights = { + "pipeline_insights": [ + { + "pipeline": "parent", + "recommended_patterns": [ + { + "pattern": "Public thing", + "fit": "x", + "simplification_pattern": False, + "release_state": "public_preview", + "release_state_source": "https://docs.databricks.com/a/", + } + ], + }, + { + "pipeline": "child", + "recommended_patterns": [ + { + "pattern": "Gated thing", + "fit": "y", + "simplification_pattern": False, + "release_state": "private_preview", + "release_state_source": "https://docs.databricks.com/b/", + } + ], + }, + ] + } + _, options = recommend_component(["child", "parent"], _inventory(graphs, insights=insights)) + disclosures = options["agentic"]["release_disclosures"] + labels = {d["release_state"]: d["label"] for d in disclosures} + assert labels == {"public_preview": "Public Preview (production-ready)", "private_preview": "Private Preview"} def test_build_recommendation_emits_both_options_and_a_default_plan() -> None: From 1643c41ded5faf823ebc9718cbfded3585636a8a Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Mon, 21 Sep 2026 11:39:10 +0100 Subject: [PATCH 009/122] Refresh AGENTS.md module map for the sources/ layout The Module Descriptions table and the "Adding a New Deterministic Translator" guide still pointed at modules removed in the source-split refactor (parser/adf_loader.py, translator/engine.py, translator/activity_translators/). Repoint them at the current sources// layout (sources/adf/loader.py, sources/adf/translate.py, sources/adf/translators/) and add the new modules this branch introduces (sources/adf/discovery_mapping.py, sources/airflow/, route_agentic.py, models/discovery.py). Co-authored-by: Isaac --- AGENTS.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7f2da251..cd2e02d6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -73,10 +73,13 @@ intermediates under `.work/` (pruned by `package`). | `models/adf_ast.py` | Typed AST nodes for ADF definitions | | `models/ir.py` | Databricks intermediate representation | | `models/dab.py` | DAB output schema types | -| `parser/adf_loader.py` | Parses ADF exports, produces `metadata/inventory.json` + `metadata/profile_report.csv` | +| `models/discovery.py` | Source-faithful shared discovery AST (`SourceGraph`) that both ADF and Airflow map onto | +| `sources/adf/loader.py` | Parses ADF exports, produces `metadata/inventory.json` + `metadata/profile_report.csv` | +| `sources/adf/discovery_mapping.py` | Maps the ADF AST onto the shared `SourceGraph` discovery model (1:1, lossless) | +| `sources/adf/translate.py` | Registry dispatch, topological sort, context threading | +| `sources/adf/translators/` | One module per deterministic activity type (16 total) | +| `sources/airflow/` | Airflow source: loader, discover, and convert (mirrors the ADF source layout) | | `parser/expression_parser.py` | Translates ADF expressions (@activity, @pipeline, @variables) | -| `translator/engine.py` | Registry dispatch, topological sort, context threading | -| `translator/activity_translators/` | One module per deterministic activity type (16 total) | | `preparer/workflow_preparer.py` | Orchestrates activity preparers | | `preparer/code_generator.py` | Notebook code generation for activity types | | `preparer/activity_preparers/` | One module per activity type | @@ -87,6 +90,7 @@ intermediates under `.work/` (pruned by `package`). | `reporting/results.py` | Writes per-run coverage to a UC table (run_id/run_date/run_by) via the SDK | | `reporting/dashboard.py` | Installs + publishes an AI/BI coverage dashboard over the results table | | `routing.py` | Groups pipelines into connected components over control lineage; recommends deterministic/agentic per component (both options) and records the user's decision as `metadata/conversion_plan.json` (additive; convert untouched) | +| `route_agentic.py` | Applies a routing decision after convert: rewrites the translation report for agentic-routed groups (placeholder tasks + per-task `AgenticGap`), keeping the fill in-engine | | `models/conversion_plan.py` | Source-neutral conversion-plan artifact model (per-component decision + both options), the routing counterpart to `models/insights.py` | ## Activity Types @@ -129,11 +133,11 @@ ExecuteDataFlow, SqlServerStoredProcedure, AzureFunction, WebHook, Custom, Execu ## Adding a New Deterministic Translator 1. Add IR dataclass to `src/flowx/models/ir.py` -2. Create translator at `src/flowx/translator/activity_translators/.py` +2. Create translator at `src/flowx/sources/adf/translators/.py` 3. Create preparer at `src/flowx/preparer/activity_preparers/.py` 4. Add notebook generator to `src/flowx/preparer/code_generator.py` if needed -5. Register in engine.py (TRANSLATOR_REGISTRY for leaf, match statement for control-flow) -6. Move from AGENTIC_TYPES to DETERMINISTIC_TYPES in adf_loader.py +5. Register in `src/flowx/sources/adf/translate.py` (TRANSLATOR_REGISTRY for leaf, match statement for control-flow) +6. Move from AGENTIC_TYPES to DETERMINISTIC_TYPES in `src/flowx/sources/adf/loader.py` 7. Update activity-mapping.md reference 8. Add test fixtures and unit tests From 9182cac534878ee8d429126d22340877a9d9f5f0 Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Mon, 21 Sep 2026 11:39:25 +0100 Subject: [PATCH 010/122] Make CLAUDE.md a symlink to AGENTS.md CLAUDE.md previously re-imported AGENTS.md via the `@AGENTS.md` line. Replace it with a real relative symlink (git mode 120000) so AGENTS.md is the single canonical onboarding doc and the two never drift. Co-authored-by: Isaac --- CLAUDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 120000 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 43c994c2..00000000 --- a/CLAUDE.md +++ /dev/null @@ -1 +0,0 @@ -@AGENTS.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 00000000..47dc3e3d --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file From d113cc03137437cb0a3163c6a45efe34b9eb744c Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Mon, 21 Sep 2026 11:42:01 +0100 Subject: [PATCH 011/122] Add self-governance note to AGENTS.md Record the bar for maintaining AGENTS.md now that it is the single canonical onboarding doc, so future edits keep it concise and pointer-first. Co-authored-by: Isaac --- AGENTS.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index cd2e02d6..2a48b9e1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -159,3 +159,10 @@ only one-line pointers): proxy the SDK sees the workspace `Origin` and a proxied `Host: localhost:`, so its Host/Origin allowlist misfires (403/421) while adding nothing on top of the proxy's authentication. Browser CORS is a separate concern configured via `FLOWX_ALLOWED_ORIGINS`. + +## Maintaining this file + +Keep this file for knowledge useful to almost every future agent session in this project. +Do not repeat what the codebase already shows; point to the authoritative file or command instead. +Prefer rewriting or pruning existing entries over appending new ones. +When updating this file, preserve this bar for all agents and keep entries concise. From c2fb754e659e47dbfa32879c2eb53844d7c0c826 Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Mon, 5 Oct 2026 12:07:10 +0100 Subject: [PATCH 012/122] Refuse agentic routing decisions for non-ADF inventories Agentic routing is ADF-only in Phase 1, but only fill-agentic combine checked the source: route with an agentic decision on an Airflow inventory recorded the plan and rewrote the DAG's deterministic tasks into placeholder gaps, outside the Airflow per-gap resolver. validate_plan now rejects an agentic decision unless the inventory source is adf (an inventory with no source predates the unified emitter and counts as ADF), and the recommendation for such an inventory is deterministic with a finding that says why. The route and migrate skills say this instead of promising Airflow routing once #63 lands. Route test fixtures now declare source adf. Co-authored-by: Isaac --- skills/flowx-migrate/SKILL.md | 8 +- skills/flowx-route/SKILL.md | 18 +++-- src/flowx/routing.py | 42 ++++++++-- tests/unit/test_cli_route_agentic.py | 2 +- tests/unit/test_route_agentic_flow.py | 2 +- tests/unit/test_routing_components.py | 2 +- tests/unit/test_routing_recommend.py | 2 +- tests/unit/test_routing_record.py | 2 +- tests/unit/test_routing_source_scope.py | 102 ++++++++++++++++++++++++ 9 files changed, 157 insertions(+), 23 deletions(-) create mode 100644 tests/unit/test_routing_source_scope.py diff --git a/skills/flowx-migrate/SKILL.md b/skills/flowx-migrate/SKILL.md index aa1567ab..cc2a90ee 100644 --- a/skills/flowx-migrate/SKILL.md +++ b/skills/flowx-migrate/SKILL.md @@ -25,10 +25,10 @@ via Declarative Automation Bundles. This skill runs the full flow in sequence: d This is the top-level orchestration skill. It runs the full migration pipeline: > **Scope:** deterministic discover → convert → package works for **both ADF and Airflow**. The -> **routing + agentic-conversion path (enrich → route → fill)** applies to **ADF today** — Airflow -> discovery does not yet emit control lineage or motifs, so it cannot form routing components, and -> `convert --merge-agentic --source airflow` is disabled. Airflow follows its own -> `flowx-resolve-airflow-gaps` path and aligns with routing via #63. +> **routing + agentic-conversion path (enrich → route → fill)** applies to **ADF today**, and +> `route` enforces it: an Airflow inventory is routed `deterministic` only, and +> `convert --merge-agentic --source airflow` is disabled. Airflow gaps follow their own +> `flowx-resolve-airflow-gaps` path. 1. **Discover** — Parse the source's definitions into a typed inventory 2. **Enrich** *(default)* — Author the agentic `insights` layer over the inventory and merge it diff --git a/skills/flowx-route/SKILL.md b/skills/flowx-route/SKILL.md index 70fdd130..42109330 100644 --- a/skills/flowx-route/SKILL.md +++ b/skills/flowx-route/SKILL.md @@ -26,11 +26,12 @@ edits the translation report so routed-agentic groups become placeholder gaps, a the fill. **ADF only (current scope).** The discover → enrich → route → edit → fill agentic-conversion flow is -supported for **ADF today**. Airflow is **not** yet wired for routing: its discovery does not emit -control lineage or motifs, so it cannot form multi-DAG routing components, and `convert ---merge-agentic --source airflow` is disabled. Airflow aligns with routing once #63 maps it onto the -shared discovery AST. For **Airflow agentic gaps today, use the `flowx-resolve-airflow-gaps` skill** -(the strict per-gap resolver) — not this routing flow. +supported for **ADF today**, and `route` enforces it: for an Airflow inventory it recommends and +records only `deterministic` decisions and rejects an `agentic` one (the plan is not recorded and the +report is left untouched). `convert --merge-agentic --source airflow` is disabled too. Airflow +DAGs are often one connected component, so Airflow agentic conversion follows the Airflow track's own +per-gap and patch contract rather than this whole-component flow. For **Airflow agentic gaps today, +use the `flowx-resolve-airflow-gaps` skill** (the strict per-gap resolver), not this routing flow. **There is no LLM inside flowx.** The library computes the recommendation deterministically and only *validates and records* the decision and the authored fill — the same author → validate → merge @@ -56,9 +57,10 @@ placeholders, the only convert is the additive `convert --merge-agentic` fill Pipelines are grouped into weak/undirected **connected components** over the inventory's control lineage (`lineage.control_edges`), so mutually-referencing pipelines are decided together and a -caller/callee reference is never split across incompatible routes. ADF emits these control edges -(from `ExecutePipeline`) in discovery; Airflow discovery does not emit control lineage yet, which is -why routing is ADF-only today (see the scope note above). +caller/callee reference is never split across incompatible routes. Only resolved control edges join +a component; an unresolved call is reported as a finding. ADF emits these control edges (from +`ExecutePipeline`) in discovery. Agentic routing is ADF-only today whatever lineage an inventory +carries (see the scope note above). Run the **`setup`** skill first if you haven't. Everything below has an MCP-tool path (Genie Code, or a local stdio registration — call the single **`flowx`** tool, run no `python3`/`$PY`) and a venv-CLI diff --git a/src/flowx/routing.py b/src/flowx/routing.py index be2ce9d3..e409cfc5 100644 --- a/src/flowx/routing.py +++ b/src/flowx/routing.py @@ -31,8 +31,10 @@ re-recording the same decision against the same inventory rewrites byte-identical bytes. This is additive, opt-in routing metadata only: with no recorded plan, ``convert`` and ``package`` -behave exactly as today. Component computation reads ``lineage.control_edges``, which both ADF -(``ExecutePipeline``) and Airflow (``RunJob``) emit, so the artifact is source-neutral. +behave exactly as today. Component computation reads ``lineage.control_edges``, so the artifact's +shape is source-neutral, but agentic decisions are accepted only for ADF inventories in Phase 1 (see +:data:`AGENTIC_ROUTING_SOURCES`). Airflow gaps stay with the fingerprint-bound per-gap resolver, so +for an Airflow inventory route recommends and records deterministic decisions only. """ from __future__ import annotations @@ -55,6 +57,11 @@ # The recorded conversion-plan artifact lives beside inventory.json under metadata/. PLAN_FILENAME = "conversion_plan.json" +# Sources whose inventories route may send agentic in Phase 1. An inventory with no top-level +# ``source`` predates the unified emitter and is ADF. +AGENTIC_ROUTING_SOURCES = frozenset({"adf"}) +_LEGACY_INVENTORY_SOURCE = "adf" + # Authored top-level keys (everything else the library owns and rejects on input). _PLAN_TOP_KEYS = {"components"} _LIBRARY_TOP_KEYS = {"schema_version", "inventory_sha256", "findings"} @@ -63,6 +70,19 @@ _COMPONENT_LIBRARY_KEYS = {"recommended", "options"} +def agentic_routing_supported(inventory: dict[str, Any]) -> bool: + """Whether route may take an agentic decision for this inventory's source (ADF only in Phase 1).""" + return inventory.get("source", _LEGACY_INVENTORY_SOURCE) in AGENTIC_ROUTING_SOURCES + + +def _agentic_routing_unsupported_note(inventory: dict[str, Any]) -> str: + """The finding / violation text that explains why a non-ADF inventory routes deterministic only.""" + return ( + f"agentic routing is ADF-only in Phase 1 (inventory source {inventory.get('source')!r}); " + "route every component 'deterministic' and resolve its gaps with the flowx-resolve-airflow-gaps skill" + ) + + def _pipeline_names(inventory: dict[str, Any]) -> set[str]: """The set of real pipeline names in the inventory (the foreign-key domain). @@ -317,12 +337,14 @@ def recommend_component(members: list[str], inventory: dict[str, Any]) -> tuple[ Returns: ``(recommended, options)`` where ``recommended`` is ``"deterministic"`` when the whole - component is engine-capable, else ``"agentic"``; and ``options`` carries the ``deterministic`` - and ``agentic`` peers as first-class entries. + component is engine-capable or the inventory's source cannot be routed agentic, else + ``"agentic"``; and ``options`` carries the ``deterministic`` and ``agentic`` peers as + first-class entries. """ deterministic = _deterministic_option(members, inventory) agentic = _agentic_option(members, inventory) - recommended = _DETERMINISTIC_STRATEGY if deterministic["capable"] else "agentic" + route_deterministic = deterministic["capable"] or not agentic_routing_supported(inventory) + recommended = _DETERMINISTIC_STRATEGY if route_deterministic else "agentic" return recommended, {"deterministic": deterministic, "agentic": agentic} @@ -336,6 +358,8 @@ def build_recommendation(inventory: dict[str, Any]) -> dict[str, Any]: per component for overrides. """ components, findings = build_components(inventory) + if not agentic_routing_supported(inventory): + findings = [*findings, _agentic_routing_unsupported_note(inventory)] component_entries: list[dict[str, Any]] = [] default_plan_components: list[dict[str, Any]] = [] for index, members in enumerate(components, start=1): @@ -377,7 +401,8 @@ def validate_plan(raw: Any, inventory: dict[str, Any]) -> list[str]: * every member must be a real inventory pipeline, and a component's ``members`` must exactly match one computed connected component -- so a decision can never split a component or span two; * the plan is a **bijection** over components: every component is decided exactly once (no - partial plan, no duplicate/conflicting decisions). + partial plan, no duplicate/conflicting decisions); + * an ``agentic`` decision is only accepted for an ADF inventory (:func:`agentic_routing_supported`). """ if not isinstance(raw, dict): return [f"conversion plan must be a JSON object, got {type(raw).__name__}"] @@ -403,6 +428,11 @@ def validate_plan(raw: Any, inventory: dict[str, Any]) -> list[str]: if matched_id is not None: decided_ids.append(matched_id) + if not agentic_routing_supported(inventory): + for index, component in enumerate(components): + if isinstance(component, dict) and component.get("decision") == "agentic": + violations.append(f"components[{index}]: {_agentic_routing_unsupported_note(inventory)}") + for component_id, count in Counter(decided_ids).items(): if count > 1: violations.append( diff --git a/tests/unit/test_cli_route_agentic.py b/tests/unit/test_cli_route_agentic.py index a43081eb..2d907dd3 100644 --- a/tests/unit/test_cli_route_agentic.py +++ b/tests/unit/test_cli_route_agentic.py @@ -48,7 +48,7 @@ def _inventory() -> dict[str, Any]: ) child = SourceGraph(name="child", source="unit", tasks=[_node("copy_orders", "Copy")]) solo = SourceGraph(name="solo", source="unit", tasks=[_node("load", "Notebook")]) - return build_source_inventory([parent, child, solo], source="unit", source_dir="/tmp/src") + return build_source_inventory([parent, child, solo], source="adf", source_dir="/tmp/src") def _report() -> dict[str, Any]: diff --git a/tests/unit/test_route_agentic_flow.py b/tests/unit/test_route_agentic_flow.py index 65e5c0ac..489cace5 100644 --- a/tests/unit/test_route_agentic_flow.py +++ b/tests/unit/test_route_agentic_flow.py @@ -161,7 +161,7 @@ def _routed_inventory() -> dict[str, Any]: ), ) child = SourceGraph(name="child", source="unit", tasks=[_node("copy_orders", "Copy")]) - return build_source_inventory([parent, child], source="unit", source_dir="/tmp/src") + return build_source_inventory([parent, child], source="adf", source_dir="/tmp/src") def _setup_routed_agentic(output_dir: Path, *, decision: str = "agentic") -> None: diff --git a/tests/unit/test_routing_components.py b/tests/unit/test_routing_components.py index d5b458e2..53c6f964 100644 --- a/tests/unit/test_routing_components.py +++ b/tests/unit/test_routing_components.py @@ -46,7 +46,7 @@ def _edge(source: str, target: str, via: str, *, resolved: bool = True) -> Contr def _inventory(graphs: list[SourceGraph]) -> dict[str, Any]: - return build_source_inventory(graphs, source="unit", source_dir="/tmp/src") + return build_source_inventory(graphs, source="adf", source_dir="/tmp/src") def test_isolated_pipelines_each_form_their_own_component() -> None: diff --git a/tests/unit/test_routing_recommend.py b/tests/unit/test_routing_recommend.py index a2725386..dea42867 100644 --- a/tests/unit/test_routing_recommend.py +++ b/tests/unit/test_routing_recommend.py @@ -37,7 +37,7 @@ def _inventory( motifs: dict[str, list[DetectedMotif]] | None = None, insights: dict[str, Any] | None = None, ) -> dict[str, Any]: - inventory = build_source_inventory(graphs, source="unit", source_dir="/tmp/src", motifs_by_pipeline=motifs) + inventory = build_source_inventory(graphs, source="adf", source_dir="/tmp/src", motifs_by_pipeline=motifs) if insights is not None: inventory["insights"] = insights return inventory diff --git a/tests/unit/test_routing_record.py b/tests/unit/test_routing_record.py index c8ea44fc..1a8c9fd3 100644 --- a/tests/unit/test_routing_record.py +++ b/tests/unit/test_routing_record.py @@ -64,7 +64,7 @@ def _inventory() -> dict[str, Any]: ) child = SourceGraph(name="child", source="unit", tasks=[_node("copy_orders", "Copy")]) solo = SourceGraph(name="solo", source="unit", tasks=[_node("load", "Notebook")]) - inventory = build_source_inventory([parent, child, solo], source="unit", source_dir="/tmp/src") + inventory = build_source_inventory([parent, child, solo], source="adf", source_dir="/tmp/src") inventory["insights"] = { "pipeline_insights": [ { diff --git a/tests/unit/test_routing_source_scope.py b/tests/unit/test_routing_source_scope.py new file mode 100644 index 00000000..a52ba688 --- /dev/null +++ b/tests/unit/test_routing_source_scope.py @@ -0,0 +1,102 @@ +"""Agentic routing is ADF-only in Phase 1, and ``route`` enforces it. + +Before this guard only ``fill-agentic combine`` checked the source, so ``route`` with an agentic +decision on an Airflow inventory recorded the plan and rewrote the DAG's deterministic tasks into +placeholder gaps, outside the Airflow per-gap resolver. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from flowx import routing +from flowx.adapter.__main__ import main as adapter_cli_main +from flowx.discovery_inventory import STRATEGY_PROPERTY, build_source_inventory +from flowx.models.discovery import CONCEPT_NOTEBOOK, SourceGraph, SourceNode +from flowx.route_agentic import GAPS_FILENAME, REPORT_FILENAME, WORK_DIRNAME + + +def _node(task_key: str, strategy: str) -> SourceNode: + return SourceNode( + source_id=task_key, + task_key=task_key, + concept=CONCEPT_NOTEBOOK, + source="airflow", + name=task_key, + native_type="PythonOperator", + properties={STRATEGY_PROPERTY: strategy}, + ) + + +def _inventory(source: str | None) -> dict[str, Any]: + """One fully deterministic DAG and one DAG with an agentic gap.""" + graphs = [ + SourceGraph( + name="chain_dag", source="airflow", tasks=[_node("a", "deterministic"), _node("b", "deterministic")] + ), + SourceGraph(name="gap_dag", source="airflow", tasks=[_node("c", "agentic")]), + ] + inventory = build_source_inventory(graphs, source=source or "adf", source_dir="/tmp/dags") + if source is None: + del inventory["source"] + return inventory + + +def _setup(output_dir: Path, inventory: dict[str, Any]) -> bytes: + metadata = output_dir / "metadata" + metadata.mkdir(parents=True, exist_ok=True) + (metadata / "inventory.json").write_text(json.dumps(inventory, indent=2), encoding="utf-8") + work = output_dir / WORK_DIRNAME + work.mkdir(parents=True, exist_ok=True) + report = { + "pipelines": [ + {"name": "chain_dag", "tasks": [{"name": "a", "task_key": "a", "type": "NotebookActivity"}]}, + {"name": "gap_dag", "tasks": [{"name": "c", "task_key": "c", "type": "PlaceholderActivity"}]}, + ] + } + report_bytes = json.dumps(report, indent=2).encode("utf-8") + (work / REPORT_FILENAME).write_bytes(report_bytes) + (work / GAPS_FILENAME).write_text("[]", encoding="utf-8") + return report_bytes + + +def _plan(decision_for_chain: str) -> dict[str, Any]: + return { + "components": [ + {"component_id": "component-1", "members": ["chain_dag"], "decision": decision_for_chain}, + {"component_id": "component-2", "members": ["gap_dag"], "decision": "deterministic"}, + ] + } + + +def test_route_refuses_an_agentic_decision_on_an_airflow_inventory(tmp_path: Path) -> None: + report_bytes = _setup(tmp_path, _inventory("airflow")) + plan_path = tmp_path / "plan.json" + plan_path.write_text(json.dumps(_plan("agentic")), encoding="utf-8") + + code = adapter_cli_main(["route", "--output-dir", str(tmp_path), "--plan-path", str(plan_path)]) + + assert code == 1 + assert not (tmp_path / "metadata" / "conversion_plan.json").exists() + assert (tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_bytes() == report_bytes + + +def test_airflow_inventory_is_recommended_and_recorded_deterministic_only(tmp_path: Path) -> None: + inventory = _inventory("airflow") + + recommendation = routing.build_recommendation(inventory) + + assert {component["recommended"] for component in recommendation["components"]} == {"deterministic"} + assert any("ADF-only" in finding for finding in recommendation["findings"]) + _setup(tmp_path, inventory) + recorded = routing.record_plan(tmp_path, plan=recommendation["default_plan"]) + assert recorded["ok"] is True + + +@pytest.mark.parametrize("source", ["adf", None]) +def test_adf_and_legacy_unsourced_inventories_still_accept_agentic(source: str | None) -> None: + assert routing.validate_plan(_plan("agentic"), _inventory(source)) == [] From 5e0a2baa1a07beaa5f37c7d9d102a9f46d90e1ab Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Mon, 5 Oct 2026 12:07:12 +0100 Subject: [PATCH 013/122] Fail package closed on a stale routing plan and hash the route audit Package now refuses to write a bundle when metadata/conversion_plan.json was recorded against a different inventory fingerprint than the current one, or when the report's combine provenance does not match an agentic component in the plan. route records the hash of the report it first edited under _route_provenance, and route_audit.json records the current inventory fingerprint plus the plan, baseline report and packaged report hashes. Co-authored-by: Isaac --- src/flowx/bundler/dab_writer.py | 114 ++++++++++++++++++- src/flowx/route_agentic.py | 24 ++-- tests/unit/test_package_routing_preflight.py | 111 ++++++++++++++++++ 3 files changed, 239 insertions(+), 10 deletions(-) create mode 100644 tests/unit/test_package_routing_preflight.py diff --git a/src/flowx/bundler/dab_writer.py b/src/flowx/bundler/dab_writer.py index d71c7fe0..b59fd6a8 100644 --- a/src/flowx/bundler/dab_writer.py +++ b/src/flowx/bundler/dab_writer.py @@ -4,6 +4,7 @@ import argparse import copy +import hashlib import json import re import sys @@ -378,17 +379,95 @@ def _default_report_path(output_dir: Path) -> Path: return work / "translation_report.json" -def _write_route_audit(output_dir: Path) -> Path | None: +def _routing_plan_failures(output_dir: Path, report_path: Path) -> list[str]: + """Check a recorded routing plan still matches what is about to be packaged. + + A plan is bound to the inventory it was recorded against. If discover ran again since route, + or a combine fill was applied under a different plan, the report no longer reflects the user's + decision, so package refuses rather than shipping it. Returns one message per problem; an empty + list means there is no recorded plan or it still matches. + """ + from flowx.discovery_insights import inventory_fingerprint + from flowx.models.conversion_plan import DECISION_AGENTIC + from flowx.route_agentic import COMBINE_PROVENANCE_KEY + + metadata_dir = Path(output_dir) / "metadata" + plan_path = metadata_dir / "conversion_plan.json" + if not plan_path.is_file(): + return [] + try: + plan = json.loads(plan_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + return [f"metadata/conversion_plan.json is unreadable: {error}"] + if not isinstance(plan, dict): + return ["metadata/conversion_plan.json must contain a JSON object"] + + inventory_path = metadata_dir / "inventory.json" + if not inventory_path.is_file(): + return ["metadata/conversion_plan.json was recorded but metadata/inventory.json is missing"] + try: + inventory = json.loads(inventory_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + return [f"metadata/inventory.json is unreadable: {error}"] + if not isinstance(inventory, dict): + return ["metadata/inventory.json must contain a JSON object"] + + recorded = plan.get("inventory_sha256") + current = inventory_fingerprint(inventory) + if recorded != current: + return [ + f"metadata/conversion_plan.json was recorded against inventory {recorded!r} but the current " + f"inventory is {current!r}; re-run route against the current inventory" + ] + + try: + report = json.loads(Path(report_path).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return [] + provenance = report.get(COMBINE_PROVENANCE_KEY) if isinstance(report, dict) else None + if not isinstance(provenance, list): + return [] + agentic_components = { + (component.get("component_id"), tuple(sorted(component.get("members") or []))) + for component in plan.get("components", []) + if isinstance(component, dict) and component.get("decision") == DECISION_AGENTIC + } + failures: list[str] = [] + for entry in provenance: + if not isinstance(entry, dict): + continue + component_id = entry.get("component_id") + if entry.get("inventory_sha256") != current: + failures.append(f"combine for {component_id!r} was applied against a different inventory") + elif (component_id, tuple(sorted(entry.get("members") or []))) not in agentic_components: + failures.append(f"combine for {component_id!r} does not match an agentic component in the recorded plan") + return failures + + +def _file_sha256(path: Path) -> str | None: + """SHA-256 of a file's bytes, or ``None`` when it cannot be read.""" + try: + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + except OSError: + return None + + +def _write_route_audit(output_dir: Path, report_path: Path | None = None) -> Path | None: """Persist a compact routing/gaps audit to ``metadata/route_audit.json`` before the prune. Package prunes the transient ``.work/`` folder (translation report + ``gaps.json``) by default, which erases the "what did routing change?" trail. When a routing decision was recorded (``metadata/conversion_plan.json`` exists), summarise the routed components, their decisions, and - the gaps routing introduced into an additive ``metadata/`` artifact that survives the prune. + the gaps routing introduced into an additive ``metadata/`` artifact that survives the prune. It + also records hashes of the inventory, the plan, and the packaged report, plus the report route + started from when route recorded one, so the trail can be checked against those files later. Returns the written path, or ``None`` when there is no recorded plan (no routing happened) -- so the no-route path writes nothing and stays byte-identical. """ + from flowx.discovery_insights import inventory_fingerprint + from flowx.route_agentic import ROUTE_PROVENANCE_KEY + metadata_dir = Path(output_dir) / "metadata" plan_path = metadata_dir / "conversion_plan.json" if not plan_path.is_file(): @@ -400,6 +479,24 @@ def _write_route_audit(output_dir: Path) -> Path | None: if not isinstance(plan, dict): return None + inventory_sha256: str | None = None + try: + inventory = json.loads((metadata_dir / "inventory.json").read_text(encoding="utf-8")) + if isinstance(inventory, dict): + inventory_sha256 = inventory_fingerprint(inventory) + except (OSError, json.JSONDecodeError): + pass + + baseline_report_sha256: str | None = None + if report_path is not None: + try: + report = json.loads(Path(report_path).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + report = None + route_provenance = report.get(ROUTE_PROVENANCE_KEY) if isinstance(report, dict) else None + if isinstance(route_provenance, dict): + baseline_report_sha256 = route_provenance.get("baseline_report_sha256") + components: list[dict[str, Any]] = [] agentic_pipelines: list[str] = [] for component in plan.get("components", []): @@ -431,6 +528,10 @@ def _write_route_audit(output_dir: Path) -> Path | None: audit = { "schema": "flowx.route_audit/v1", "recorded_against_inventory_sha256": plan.get("inventory_sha256"), + "inventory_sha256": inventory_sha256, + "conversion_plan_sha256": _file_sha256(plan_path), + "baseline_report_sha256": baseline_report_sha256, + "translation_report_sha256": _file_sha256(report_path) if report_path is not None else None, "components": components, "agentic_pipelines": sorted(set(agentic_pipelines)), "gaps_count": len(gaps_introduced), @@ -525,6 +626,13 @@ def main(argv: list[str] | None = None) -> int: print(f" - {failure}", file=sys.stderr) return 1 + plan_failures = _routing_plan_failures(args.output_dir, args.report) + if plan_failures: + print("Error: routing plan preflight failed; no bundle files were written.", file=sys.stderr) + for failure in plan_failures: + print(f" - {failure}", file=sys.stderr) + return 1 + if args.profile: set_profile(args.profile) @@ -650,7 +758,7 @@ def main(argv: list[str] | None = None) -> int: # Persist the routing/gaps audit to metadata/ before pruning .work/, so the "what did routing # change?" trail survives the default prune. No-ops (writes nothing) when no plan was recorded. - audit_path = _write_route_audit(args.output_dir) + audit_path = _write_route_audit(args.output_dir, args.report) if audit_path is not None: print(f"Wrote routing audit to {audit_path}") diff --git a/src/flowx/route_agentic.py b/src/flowx/route_agentic.py index d21f30b0..745754b8 100644 --- a/src/flowx/route_agentic.py +++ b/src/flowx/route_agentic.py @@ -32,6 +32,7 @@ from __future__ import annotations import copy +import hashlib import json import os import sys @@ -62,7 +63,11 @@ # lives *in* the report, so a fresh convert (which rewrites the report) naturally clears it, and it is # ignored by the package phase (not one of the recognized report shape keys) and by ir_serde (which # reads per-pipeline IR, not the report wrapper). -_COMBINE_PROVENANCE_KEY = "_combine_provenance" +COMBINE_PROVENANCE_KEY = "_combine_provenance" + +# Top-level report key where route records the hash of the report it edited, so package can show +# what the routing edit started from after .work/ is pruned. Written once, on the first edit. +ROUTE_PROVENANCE_KEY = "_route_provenance" # Guidance stamped onto every placeholder the alteration produces. _PLACEHOLDER_COMMENT = ( @@ -267,8 +272,10 @@ def apply_plan_to_report(output_dir: Path, plan: dict[str, Any]) -> dict[str, An """Apply a routing decision to the report on disk: placeholder the routed-agentic groups. Reads ``/.work/translation_report.json`` (and ``gaps.json`` when present), rewrites - them for the plan's agentic components, and writes them back atomically. When no component is - routed agentic the files are left untouched, so the non-breaking guarantee holds. + them for the plan's agentic components, and writes them back atomically. The first edit also + records the SHA-256 of the report it started from under ``_route_provenance`` so the package + audit can name it. When no component is routed agentic the files are left untouched, so the + non-breaking guarantee holds. Returns a summary dict with the altered pipeline names and the resulting gap count. @@ -285,12 +292,15 @@ def apply_plan_to_report(output_dir: Path, plan: dict[str, Any]) -> dict[str, An if not agentic: return {"agentic_pipelines": [], "gaps": 0, "altered": False} - report = json.loads(report_path.read_text(encoding="utf-8")) + baseline_bytes = report_path.read_bytes() + report = json.loads(baseline_bytes) gaps = json.loads(gaps_path.read_text(encoding="utf-8")) if gaps_path.exists() else [] if not isinstance(gaps, list): gaps = [] new_report, new_gaps = alter_report(report, gaps, agentic) + if isinstance(new_report, dict) and ROUTE_PROVENANCE_KEY not in new_report: + new_report[ROUTE_PROVENANCE_KEY] = {"baseline_report_sha256": hashlib.sha256(baseline_bytes).hexdigest()} _write_json_atomic(report_path, new_report) _write_json_atomic(gaps_path, new_gaps) return {"agentic_pipelines": sorted(agentic), "gaps": len(new_gaps), "altered": True} @@ -410,7 +420,7 @@ def _combine_already_applied(report: dict[str, Any], component_id: str, fingerpr pipeline names to member names -- so a re-run is recognised as already-combined no matter what the authored replacement is named, and even when an authored name collides with a former member. """ - provenance = report.get(_COMBINE_PROVENANCE_KEY) if isinstance(report, dict) else None + provenance = report.get(COMBINE_PROVENANCE_KEY) if isinstance(report, dict) else None if not isinstance(provenance, list): return False return any( @@ -486,12 +496,12 @@ def apply_combine_fill( merged = combine_group_fill(report, members, authored_pipelines) # Carry any prior provenance forward (combine_group_fill returns only ``pipelines``) and record # this combine so a later re-run detects it. - prior_provenance = report.get(_COMBINE_PROVENANCE_KEY) if isinstance(report, dict) else None + prior_provenance = report.get(COMBINE_PROVENANCE_KEY) if isinstance(report, dict) else None provenance = ( [entry for entry in prior_provenance if isinstance(entry, dict)] if isinstance(prior_provenance, list) else [] ) provenance.append({"component_id": component_id, "inventory_sha256": fingerprint, "members": sorted(members)}) - merged[_COMBINE_PROVENANCE_KEY] = provenance + merged[COMBINE_PROVENANCE_KEY] = provenance result = validate_report_structurally(merged) if not result.ok: diff --git a/tests/unit/test_package_routing_preflight.py b/tests/unit/test_package_routing_preflight.py new file mode 100644 index 00000000..80411c68 --- /dev/null +++ b/tests/unit/test_package_routing_preflight.py @@ -0,0 +1,111 @@ +"""Package refuses to ship a translation report whose routing plan has gone stale. + +A recorded ``metadata/conversion_plan.json`` is bound to the inventory fingerprint it was recorded +against. When discover runs again, or a combine fill was applied under a different plan, the report +no longer reflects the user's decision, so package fails closed before writing any bundle file. +When the plan still matches, the route audit records hashes of what was packaged. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +import pytest + +from flowx import routing +from flowx.bundler.dab_writer import main as package_main +from flowx.discovery_inventory import STRATEGY_PROPERTY, build_source_inventory +from flowx.ir_serde import pipeline_to_dict +from flowx.models.discovery import CONCEPT_NOTEBOOK, SourceGraph, SourceNode +from flowx.models.ir import NotebookActivity, Pipeline +from flowx.route_agentic import COMBINE_PROVENANCE_KEY, REPORT_FILENAME, WORK_DIRNAME, apply_plan_to_report + + +def _inventory(strategy: str = "deterministic") -> dict[str, Any]: + node = SourceNode( + source_id="load", + task_key="load", + concept=CONCEPT_NOTEBOOK, + source="adf", + name="load", + native_type="DatabricksNotebook", + properties={STRATEGY_PROPERTY: strategy}, + ) + return build_source_inventory([SourceGraph(name="solo", source="adf", tasks=[node])], source="adf", source_dir="/x") + + +def _setup(output_dir: Path, inventory: dict[str, Any]) -> Path: + metadata = output_dir / "metadata" + metadata.mkdir(parents=True, exist_ok=True) + (metadata / "inventory.json").write_text(json.dumps(inventory, indent=2), encoding="utf-8") + work = output_dir / WORK_DIRNAME + work.mkdir(parents=True, exist_ok=True) + pipeline = Pipeline( + name="solo", + tasks=[NotebookActivity(name="load", task_key="load", notebook_path="/Workspace/Shared/load")], + tags={"source": "adf"}, + ) + report_path = work / REPORT_FILENAME + report_path.write_text(json.dumps({"pipelines": [pipeline_to_dict(pipeline)]}, indent=2), encoding="utf-8") + return report_path + + +def _package(output_dir: Path) -> int: + return package_main(["--output-dir", str(output_dir), "--no-download-workspace-files", "--keep-intermediates"]) + + +def _record(output_dir: Path, decision: str) -> None: + plan = {"components": [{"component_id": "component-1", "members": ["solo"], "decision": decision}]} + assert routing.record_plan(output_dir, plan=plan)["ok"] is True + + +def test_package_refuses_a_plan_recorded_against_a_different_inventory( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + _setup(tmp_path, _inventory()) + _record(tmp_path, "deterministic") + # Discover runs again and the inventory changes after route recorded the plan. + (tmp_path / "metadata" / "inventory.json").write_text(json.dumps(_inventory("agentic")), encoding="utf-8") + + assert _package(tmp_path) == 1 + assert "re-run route against the current inventory" in capsys.readouterr().err + assert not (tmp_path / "databricks.yml").exists() + assert not (tmp_path / "metadata" / "route_audit.json").exists() + + +def test_package_refuses_a_combine_that_does_not_match_the_plan( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + report_path = _setup(tmp_path, _inventory()) + _record(tmp_path, "deterministic") + report = json.loads(report_path.read_text(encoding="utf-8")) + current = json.loads((tmp_path / "metadata" / "conversion_plan.json").read_text())["inventory_sha256"] + report[COMBINE_PROVENANCE_KEY] = [{"component_id": "component-1", "inventory_sha256": current, "members": ["solo"]}] + report_path.write_text(json.dumps(report), encoding="utf-8") + + assert _package(tmp_path) == 1 + assert "does not match an agentic component in the recorded plan" in capsys.readouterr().err + assert not (tmp_path / "databricks.yml").exists() + + +def test_package_with_a_current_plan_writes_an_audit_with_hashes(tmp_path: Path) -> None: + report_path = _setup(tmp_path, _inventory("agentic")) + baseline_sha256 = hashlib.sha256(report_path.read_bytes()).hexdigest() + _record(tmp_path, "agentic") + plan = json.loads((tmp_path / "metadata" / "conversion_plan.json").read_text(encoding="utf-8")) + apply_plan_to_report(tmp_path, plan) + + assert _package(tmp_path) in (0, 1) # 1 only signals bundle-invariant warnings, not the preflight + assert (tmp_path / "databricks.yml").exists() + + audit = json.loads((tmp_path / "metadata" / "route_audit.json").read_text(encoding="utf-8")) + assert audit["inventory_sha256"] == audit["recorded_against_inventory_sha256"] == plan["inventory_sha256"] + assert ( + audit["conversion_plan_sha256"] + == hashlib.sha256((tmp_path / "metadata" / "conversion_plan.json").read_bytes()).hexdigest() + ) + assert audit["baseline_report_sha256"] == baseline_sha256 + assert audit["translation_report_sha256"] == hashlib.sha256(report_path.read_bytes()).hexdigest() From bcba3caf89f120ebe194f2a045c515d99eea32cc Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Mon, 5 Oct 2026 12:07:14 +0100 Subject: [PATCH 014/122] Describe models/discovery.py as the source-neutral discovery graph contract Co-authored-by: Isaac --- AGENTS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 2a48b9e1..5f7b3729 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -73,9 +73,9 @@ intermediates under `.work/` (pruned by `package`). | `models/adf_ast.py` | Typed AST nodes for ADF definitions | | `models/ir.py` | Databricks intermediate representation | | `models/dab.py` | DAB output schema types | -| `models/discovery.py` | Source-faithful shared discovery AST (`SourceGraph`) that both ADF and Airflow map onto | +| `models/discovery.py` | Source-neutral discovery graph contract (`SourceGraph`); each source keeps its own parser and projects into it | | `sources/adf/loader.py` | Parses ADF exports, produces `metadata/inventory.json` + `metadata/profile_report.csv` | -| `sources/adf/discovery_mapping.py` | Maps the ADF AST onto the shared `SourceGraph` discovery model (1:1, lossless) | +| `sources/adf/discovery_mapping.py` | Maps the ADF AST onto the `SourceGraph` discovery graph contract (1:1, lossless) | | `sources/adf/translate.py` | Registry dispatch, topological sort, context threading | | `sources/adf/translators/` | One module per deterministic activity type (16 total) | | `sources/airflow/` | Airflow source: loader, discover, and convert (mirrors the ADF source layout) | From 4a79c0959aad68183f00fef7b0f7e9764371a634 Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Mon, 5 Oct 2026 17:10:05 +0100 Subject: [PATCH 015/122] Build routing test inventories with motifs on the graphs The shared emitter now reads motifs from each graph's lineage, so the routing recommendation tests attach them with the ADF loader's helper. Co-authored-by: Isaac --- tests/unit/test_routing_recommend.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_routing_recommend.py b/tests/unit/test_routing_recommend.py index dea42867..7919c713 100644 --- a/tests/unit/test_routing_recommend.py +++ b/tests/unit/test_routing_recommend.py @@ -16,6 +16,7 @@ from flowx.models.ir import ControlEdge, Lineage from flowx.models.motifs import MOTIF_METADATA_DRIVEN_BULK_COPY, DetectedMotif from flowx.routing import build_recommendation, recommend_component +from flowx.sources.adf.loader import attach_motifs_to_graphs def _node(task_key: str, native_type: str, *, strategy: str = "deterministic") -> SourceNode: @@ -37,7 +38,8 @@ def _inventory( motifs: dict[str, list[DetectedMotif]] | None = None, insights: dict[str, Any] | None = None, ) -> dict[str, Any]: - inventory = build_source_inventory(graphs, source="adf", source_dir="/tmp/src", motifs_by_pipeline=motifs) + attach_motifs_to_graphs(graphs, motifs or {}) + inventory = build_source_inventory(graphs, source="adf", source_dir="/tmp/src") if insights is not None: inventory["insights"] = insights return inventory From 235e5279e6d9ee4b1b7b23f17863f4ecf23f3737 Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Mon, 5 Oct 2026 17:20:43 +0100 Subject: [PATCH 016/122] Make the typed ConversionPlan (schema 2) the library's plan API conversion_plan.json was built and read as raw dicts; the dataclasses only backed tests. ConversionPlan now loads, round-trips and writes the plan and every reader goes through it: route records it, apply_plan applies it to the IR after convert, the combine fill and package check it, and the route audit reads it. Schema 2 binds the plan to what it was decided on: the inventory fingerprint, the saved source_graphs.json hash the inventory records, and the saved source_insights.json hash when enrich ran. Package refuses to run when any of them no longer match. Decisions stay per component; each component carries a reserved assignments list for per-node or subgraph routing (deterministic, agentic with a named pattern, or mixed at a boundary), which must stay empty for now. Co-authored-by: Isaac --- skills/flowx-route/SKILL.md | 10 +- src/flowx/adapter/__main__.py | 8 +- src/flowx/bundler/dab_writer.py | 74 +++++------ src/flowx/models/conversion_plan.py | 178 ++++++++++++++++++++++++-- src/flowx/route_agentic.py | 49 +++++-- src/flowx/routing.py | 107 ++++++++++------ tests/unit/test_conversion_plan_v2.py | 152 ++++++++++++++++++++++ tests/unit/test_route_audit.py | 21 +-- 8 files changed, 480 insertions(+), 119 deletions(-) create mode 100644 tests/unit/test_conversion_plan_v2.py diff --git a/skills/flowx-route/SKILL.md b/skills/flowx-route/SKILL.md index 42109330..e4af4688 100644 --- a/skills/flowx-route/SKILL.md +++ b/skills/flowx-route/SKILL.md @@ -224,8 +224,8 @@ MCP: `flowx(command="fill_agentic", parameters={"output_dir": ..., "members": [. `combine` is the only action. Its guarantees: - `--members` (comma-separated; MCP accepts a list) must **exactly match** a routed-**agentic** - component in the recorded `metadata/conversion_plan.json`, whose `inventory_sha256` must still match - the current inventory. A partial group, a superset, a typo, or a deterministic component is refused + component in the recorded `metadata/conversion_plan.json`, whose `inventory_sha256`, + `source_graphs_sha256` and `source_insights_sha256` must still match the current inventory. A partial group, a superset, a typo, or a deterministic component is refused — you can't swap pipelines the plan didn't route agentic. - `--pipelines-path` is a JSON **list** of pipeline IR dicts (the authored replacements), typically carrying `AgenticComponentActivity` nodes (see below). @@ -304,7 +304,11 @@ applies when authoring recommended patterns.) Once the routed-agentic groups are filled and the report validates, continue with `flowx-convert`'s just-in-time configuration (`inspect`/`modify`) as usual, then `flowx-package`. The recorded -`metadata/conversion_plan.json` is kept alongside `inventory.json` as the routing record. +`metadata/conversion_plan.json` is kept alongside `inventory.json` as the routing record. It is the +library's typed `ConversionPlan` (schema 2): one decision per component, bound to the inventory +fingerprint, the saved `source_graphs.json` and the saved `source_insights.json` it was decided on, +with a reserved, empty `assignments` list per component for per-node routing later. Package refuses +to run when any of those no longer match. ## Reference diff --git a/src/flowx/adapter/__main__.py b/src/flowx/adapter/__main__.py index 64ca32b8..3b2693e6 100644 --- a/src/flowx/adapter/__main__.py +++ b/src/flowx/adapter/__main__.py @@ -186,7 +186,8 @@ def _run_route(args: argparse.Namespace) -> int: that fails validation (report + plan left untouched). """ from flowx import routing - from flowx.route_agentic import REPORT_FILENAME, WORK_DIRNAME, apply_plan_to_report + from flowx.models.conversion_plan import ConversionPlan + from flowx.route_agentic import REPORT_FILENAME, WORK_DIRNAME, apply_plan inventory_path = args.output_dir / "metadata" / "inventory.json" if not inventory_path.exists(): @@ -231,7 +232,10 @@ def _run_route(args: argparse.Namespace) -> int: _emit_json(result, args.out) return 1 try: - edit = apply_plan_to_report(args.output_dir, plan) + recorded = ConversionPlan.load(args.output_dir) + if recorded is None: + raise FileNotFoundError("route recorded no conversion_plan.json") + edit = apply_plan(args.output_dir, recorded) except (FileNotFoundError, OSError, ValueError, json.JSONDecodeError) as error: print(f"Failed to edit translation report: {error}", file=sys.stderr) return 1 diff --git a/src/flowx/bundler/dab_writer.py b/src/flowx/bundler/dab_writer.py index b59fd6a8..4a7a9a42 100644 --- a/src/flowx/bundler/dab_writer.py +++ b/src/flowx/bundler/dab_writer.py @@ -382,25 +382,24 @@ def _default_report_path(output_dir: Path) -> Path: def _routing_plan_failures(output_dir: Path, report_path: Path) -> list[str]: """Check a recorded routing plan still matches what is about to be packaged. - A plan is bound to the inventory it was recorded against. If discover ran again since route, - or a combine fill was applied under a different plan, the report no longer reflects the user's - decision, so package refuses rather than shipping it. Returns one message per problem; an empty - list means there is no recorded plan or it still matches. + A plan is bound to the inventory, the saved source graphs and the saved source insights it was + decided on. If discover or enrich ran again since route, or a combine fill was applied under a + different plan, the report no longer reflects the user's decision, so package refuses rather + than shipping it. Returns one message per problem; an empty list means there is no recorded plan + or it still matches. """ from flowx.discovery_insights import inventory_fingerprint - from flowx.models.conversion_plan import DECISION_AGENTIC + from flowx.models.conversion_plan import DECISION_AGENTIC, ConversionPlan from flowx.route_agentic import COMBINE_PROVENANCE_KEY + from flowx.routing import plan_binding_violations metadata_dir = Path(output_dir) / "metadata" - plan_path = metadata_dir / "conversion_plan.json" - if not plan_path.is_file(): - return [] try: - plan = json.loads(plan_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as error: + plan = ConversionPlan.load(Path(output_dir)) + except (OSError, ValueError) as error: return [f"metadata/conversion_plan.json is unreadable: {error}"] - if not isinstance(plan, dict): - return ["metadata/conversion_plan.json must contain a JSON object"] + if plan is None: + return [] inventory_path = metadata_dir / "inventory.json" if not inventory_path.is_file(): @@ -412,13 +411,9 @@ def _routing_plan_failures(output_dir: Path, report_path: Path) -> list[str]: if not isinstance(inventory, dict): return ["metadata/inventory.json must contain a JSON object"] - recorded = plan.get("inventory_sha256") - current = inventory_fingerprint(inventory) - if recorded != current: - return [ - f"metadata/conversion_plan.json was recorded against inventory {recorded!r} but the current " - f"inventory is {current!r}; re-run route against the current inventory" - ] + stale = plan_binding_violations(plan, inventory) + if stale: + return [f"metadata/{message}" if message.startswith("conversion_plan.json") else message for message in stale] try: report = json.loads(Path(report_path).read_text(encoding="utf-8")) @@ -427,10 +422,11 @@ def _routing_plan_failures(output_dir: Path, report_path: Path) -> list[str]: provenance = report.get(COMBINE_PROVENANCE_KEY) if isinstance(report, dict) else None if not isinstance(provenance, list): return [] + current = inventory_fingerprint(inventory) agentic_components = { - (component.get("component_id"), tuple(sorted(component.get("members") or []))) - for component in plan.get("components", []) - if isinstance(component, dict) and component.get("decision") == DECISION_AGENTIC + (component.component_id, tuple(sorted(component.members))) + for component in plan.components + if component.decision == DECISION_AGENTIC } failures: list[str] = [] for entry in provenance: @@ -459,24 +455,24 @@ def _write_route_audit(output_dir: Path, report_path: Path | None = None) -> Pat which erases the "what did routing change?" trail. When a routing decision was recorded (``metadata/conversion_plan.json`` exists), summarise the routed components, their decisions, and the gaps routing introduced into an additive ``metadata/`` artifact that survives the prune. It - also records hashes of the inventory, the plan, and the packaged report, plus the report route - started from when route recorded one, so the trail can be checked against those files later. + also records hashes of the inventory, the saved source graphs and source insights the plan was + bound to, the plan, and the packaged report, plus the report route started from when route + recorded one, so the trail can be checked against those files later. Returns the written path, or ``None`` when there is no recorded plan (no routing happened) -- so the no-route path writes nothing and stays byte-identical. """ from flowx.discovery_insights import inventory_fingerprint + from flowx.models.conversion_plan import DECISION_AGENTIC, PLAN_FILENAME, ConversionPlan from flowx.route_agentic import ROUTE_PROVENANCE_KEY metadata_dir = Path(output_dir) / "metadata" - plan_path = metadata_dir / "conversion_plan.json" - if not plan_path.is_file(): - return None + plan_path = metadata_dir / PLAN_FILENAME try: - plan = json.loads(plan_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): + plan = ConversionPlan.load(Path(output_dir)) + except (OSError, ValueError): return None - if not isinstance(plan, dict): + if plan is None: return None inventory_sha256: str | None = None @@ -499,14 +495,12 @@ def _write_route_audit(output_dir: Path, report_path: Path | None = None) -> Pat components: list[dict[str, Any]] = [] agentic_pipelines: list[str] = [] - for component in plan.get("components", []): - if not isinstance(component, dict): - continue - members = [str(member) for member in component.get("members", []) if isinstance(member, str)] - decision = component.get("decision") - components.append({"component_id": component.get("component_id"), "members": members, "decision": decision}) - if decision == "agentic": - agentic_pipelines.extend(members) + for component in plan.components: + components.append( + {"component_id": component.component_id, "members": list(component.members), "decision": component.decision} + ) + if component.decision == DECISION_AGENTIC: + agentic_pipelines.extend(component.members) gaps_introduced: list[dict[str, Any]] = [] gaps_path = Path(output_dir) / ".work" / "gaps.json" @@ -527,8 +521,10 @@ def _write_route_audit(output_dir: Path, report_path: Path | None = None) -> Pat audit = { "schema": "flowx.route_audit/v1", - "recorded_against_inventory_sha256": plan.get("inventory_sha256"), + "recorded_against_inventory_sha256": plan.inventory_sha256, "inventory_sha256": inventory_sha256, + "source_graphs_sha256": plan.source_graphs_sha256, + "source_insights_sha256": plan.source_insights_sha256, "conversion_plan_sha256": _file_sha256(plan_path), "baseline_report_sha256": baseline_report_sha256, "translation_report_sha256": _file_sha256(report_path) if report_path is not None else None, diff --git a/src/flowx/models/conversion_plan.py b/src/flowx/models/conversion_plan.py index 9f568576..a56e882c 100644 --- a/src/flowx/models/conversion_plan.py +++ b/src/flowx/models/conversion_plan.py @@ -3,32 +3,50 @@ The routing step (:mod:`flowx.routing`) groups pipelines into connected components over control lineage, presents each component's two conversion options as first-class peers, and records the user's per-component choice as ``metadata/conversion_plan.json``. These models are **source-neutral** -and document the shape of that artifact; the validate/record engine works on the raw dict form and -these dataclasses back the unit tests, mirroring the split in :mod:`flowx.models.insights`. +and are the library's typed form of that artifact: :class:`ConversionPlan` loads, round-trips and +writes it, and every reader (route, the fill, package) goes through it. Validation of the authored +decision against the inventory lives in :mod:`flowx.routing`. The agent authors **only** :attr:`ComponentPlan.decision` (and an optional :attr:`ComponentPlan.rationale`). Everything else -- ``component_id``, ``members``, ``recommended``, and both :class:`ComponentOptions` -- is recomputed by the library on record so the recorded facts can never drift from the inventory or be faked. The library also owns :attr:`ConversionPlan.schema_version` -and :attr:`ConversionPlan.inventory_sha256` (the fingerprint that binds the plan to the inventory). +and the hashes that bind the plan to what it was decided on: :attr:`ConversionPlan.inventory_sha256` +(the deterministic inventory), :attr:`ConversionPlan.source_graphs_sha256` (the saved +``source_graphs.json``) and :attr:`ConversionPlan.source_insights_sha256` (the saved +``source_insights.json``, when enrich ran). -This is Phase-1, descriptive-only routing metadata: recording a plan does not alter ``convert``. +Phase 1 records one decision per component and applies it to the IR after convert. Each component +also carries :attr:`ComponentPlan.assignments`, reserved for per-node or per-subgraph routing +(deterministic, agentic with a named pattern, or mixed at a boundary); Phase 1 requires it empty. """ from __future__ import annotations +import json +import os from dataclasses import dataclass, field +from pathlib import Path from typing import Any, Literal # The plan schema version stamped onto the recorded artifact. Bump on any backwards-incompatible -# change to the recorded shape. -SCHEMA_VERSION = "1" +# change to the recorded shape. Version 2 adds the source graph and source insights hashes and the +# reserved per-node assignments. +SCHEMA_VERSION = "2" + +# Where the recorded plan lives, beside inventory.json under the output's metadata/ folder. +PLAN_FILENAME = "conversion_plan.json" # The two conversion routes a component can take. DECISION_DETERMINISTIC = "deterministic" DECISION_AGENTIC = "agentic" DECISIONS: tuple[str, ...] = (DECISION_DETERMINISTIC, DECISION_AGENTIC) +# Routes a reserved per-node assignment may name once Phase 2 uses them; "mixed" joins a +# deterministic and an agentic part at explicit boundary edges. +ROUTE_MIXED = "mixed" +ASSIGNMENT_ROUTES: tuple[str, ...] = (DECISION_DETERMINISTIC, DECISION_AGENTIC, ROUTE_MIXED) + # Recommended-pattern release states surfaced as a neutral disclosure label on the agentic option # (:attr:`AgenticOption.release_disclosures`). ``"ga"`` and ``"unknown"`` are deliberately **silent** # -- they contribute no entry, and ``"unknown"`` is treated exactly like ``"ga"`` (we do not surface @@ -115,6 +133,47 @@ class ComponentOptions: agentic: AgenticOption +@dataclass(slots=True, kw_only=True) +class NodeAssignment: + """A routing choice for one node or a bounded subgraph inside a component (reserved for Phase 2). + + Attributes: + pipeline: The pipeline the nodes belong to. + task_keys: The node, or the nodes of a bounded subgraph, this assignment covers. + route: One of :data:`ASSIGNMENT_ROUTES`. + pattern: The named pattern an agentic part is converted to, for example a recommended + pattern from the insights. + boundary: For ``"mixed"``, the edges where the deterministic and agentic parts meet. + """ + + pipeline: str + task_keys: list[str] = field(default_factory=list) + route: str + pattern: str | None = None + boundary: list[dict[str, Any]] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + """Serialise to the recorded JSON shape.""" + return { + "pipeline": self.pipeline, + "task_keys": list(self.task_keys), + "route": self.route, + "pattern": self.pattern, + "boundary": list(self.boundary), + } + + @classmethod + def from_dict(cls, raw: dict[str, Any]) -> NodeAssignment: + """Rehydrate from the recorded JSON shape.""" + return cls( + pipeline=str(raw.get("pipeline", "")), + task_keys=[str(key) for key in raw.get("task_keys") or []], + route=str(raw.get("route", "")), + pattern=raw.get("pattern"), + boundary=list(raw.get("boundary") or []), + ) + + @dataclass(slots=True, kw_only=True) class ComponentPlan: """One component's routing decision. @@ -125,17 +184,47 @@ class ComponentPlan: recommended: The library's starting suggestion -- ``"deterministic"`` when the component is engine-capable, else ``"agentic"``. decision: The user's authored per-component choice (may override :attr:`recommended`). - options: Both conversion options with their evidence (library-computed). Optional here so the - authored input -- which carries only the decision -- can round-trip through this model. + options: Both conversion options with their evidence (library-computed), in the recorded + JSON shape that :class:`ComponentOptions` documents. Optional so the authored input -- + which carries only the decision -- can round-trip through this model. rationale: Optional author note on why this decision was chosen. + assignments: Reserved per-node or per-subgraph routing (Phase 2); empty in Phase 1. """ component_id: str members: list[str] = field(default_factory=list) recommended: str | None = None decision: str - options: ComponentOptions | None = None + options: dict[str, Any] | None = None rationale: str | None = None + assignments: list[NodeAssignment] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + """Serialise to the recorded JSON shape (``rationale`` only when set).""" + result: dict[str, Any] = { + "component_id": self.component_id, + "members": list(self.members), + "recommended": self.recommended, + "decision": self.decision, + "options": self.options, + } + if self.rationale is not None: + result["rationale"] = self.rationale + result["assignments"] = [assignment.to_dict() for assignment in self.assignments] + return result + + @classmethod + def from_dict(cls, raw: dict[str, Any]) -> ComponentPlan: + """Rehydrate from the recorded JSON shape.""" + return cls( + component_id=str(raw.get("component_id", "")), + members=[str(member) for member in raw.get("members") or []], + recommended=raw.get("recommended"), + decision=str(raw.get("decision", "")), + options=raw.get("options"), + rationale=raw.get("rationale"), + assignments=[NodeAssignment.from_dict(item) for item in raw.get("assignments") or []], + ) @dataclass(slots=True, kw_only=True) @@ -149,9 +238,80 @@ class ConversionPlan: schema_version: Library-owned plan schema version. inventory_sha256: Library-owned fingerprint binding the plan to the deterministic inventory base (see :func:`flowx.discovery_insights.inventory_fingerprint`). + source_graphs_sha256: Hash of the saved ``source_graphs.json`` the inventory was projected + from, or ``None`` when the source does not persist one. + source_insights_sha256: Hash of the saved ``source_insights.json``, or ``None`` when enrich + did not run. """ components: list[ComponentPlan] = field(default_factory=list) findings: list[str] = field(default_factory=list) schema_version: str = SCHEMA_VERSION inventory_sha256: str | None = None + source_graphs_sha256: str | None = None + source_insights_sha256: str | None = None + + def agentic_members(self) -> list[list[str]]: + """The member lists of every component routed agentic.""" + return [list(component.members) for component in self.components if component.decision == DECISION_AGENTIC] + + def to_dict(self) -> dict[str, Any]: + """Serialise to the recorded ``conversion_plan.json`` shape.""" + return { + "schema_version": self.schema_version, + "inventory_sha256": self.inventory_sha256, + "source_graphs_sha256": self.source_graphs_sha256, + "source_insights_sha256": self.source_insights_sha256, + "components": [component.to_dict() for component in self.components], + "findings": list(self.findings), + } + + @classmethod + def from_dict(cls, raw: Any) -> ConversionPlan: + """Rehydrate a recorded plan. + + Raises: + ValueError: The document is not an object, or was recorded under another schema version + (re-run ``route`` to record it again). + """ + if not isinstance(raw, dict): + raise ValueError(f"{PLAN_FILENAME} must contain a JSON object, got {type(raw).__name__}") + if raw.get("schema_version") != SCHEMA_VERSION: + raise ValueError( + f"{PLAN_FILENAME} has schema_version {raw.get('schema_version')!r}; expected {SCHEMA_VERSION!r}. " + "Re-run route to record the decision again." + ) + components = [item for item in raw.get("components") or [] if isinstance(item, dict)] + return cls( + schema_version=SCHEMA_VERSION, + inventory_sha256=raw.get("inventory_sha256"), + source_graphs_sha256=raw.get("source_graphs_sha256"), + source_insights_sha256=raw.get("source_insights_sha256"), + components=[ComponentPlan.from_dict(item) for item in components], + findings=[str(finding) for finding in raw.get("findings") or []], + ) + + @classmethod + def load(cls, output_dir: Path) -> ConversionPlan | None: + """Load the recorded plan from ``/metadata``, or ``None`` when none was recorded. + + Raises: + ValueError: The file is not valid JSON or not a version this library reads. + """ + path = Path(output_dir) / "metadata" / PLAN_FILENAME + if not path.is_file(): + return None + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as error: + raise ValueError(f"{PLAN_FILENAME} is not valid JSON: {error}") from error + return cls.from_dict(raw) + + def write(self, output_dir: Path) -> Path: + """Write the plan to ``/metadata`` atomically; the same plan rewrites identical bytes.""" + path = Path(output_dir) / "metadata" / PLAN_FILENAME + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.tmp") + temporary.write_text(json.dumps(self.to_dict(), indent=2), encoding="utf-8") + os.replace(temporary, path) + return path diff --git a/src/flowx/route_agentic.py b/src/flowx/route_agentic.py index 745754b8..3742bad9 100644 --- a/src/flowx/route_agentic.py +++ b/src/flowx/route_agentic.py @@ -41,7 +41,7 @@ from pathlib import Path from typing import Any -from flowx.models.conversion_plan import DECISION_AGENTIC +from flowx.models.conversion_plan import DECISION_AGENTIC, ConversionPlan # The report + gaps live under the shared output dir's transient .work/ folder, beside the pipeline IR. WORK_DIRNAME = ".work" @@ -306,6 +306,30 @@ def apply_plan_to_report(output_dir: Path, plan: dict[str, Any]) -> dict[str, An return {"agentic_pipelines": sorted(agentic), "gaps": len(new_gaps), "altered": True} +def apply_plan(output_dir: Path, plan: ConversionPlan) -> dict[str, Any]: + """Apply a recorded, typed plan to the IR after convert: the library's one routing entry point. + + Checks the plan still matches the inventory, source graphs and source insights it was decided + on, then placeholders the routed-agentic components in the translation report (see + :func:`apply_plan_to_report`); the fills (``convert --merge-agentic`` per pipeline, + ``fill-agentic combine`` across pipelines) then replace those placeholders. Phase 1 decides whole + components, so the reserved per-node assignments are not read here. + + Raises: + FileNotFoundError: The inventory or translation report is missing. + ValueError: The plan is stale against the current discovery outputs. + """ + from flowx.routing import plan_binding_violations + + inventory_path = Path(output_dir) / METADATA_DIRNAME / INVENTORY_FILENAME + if not inventory_path.exists(): + raise FileNotFoundError(f"No {INVENTORY_FILENAME} under {inventory_path.parent}; run the discover phase first.") + stale = plan_binding_violations(plan, json.loads(inventory_path.read_text(encoding="utf-8"))) + if stale: + raise ValueError("; ".join(stale)) + return apply_plan_to_report(output_dir, plan.to_dict()) + + # --------------------------------------------------------------------------- # # Cross-pipeline COMBINE: the pipeline-grain fill. # --------------------------------------------------------------------------- # @@ -342,28 +366,25 @@ def _resolve_agentic_component(output_dir: Path, members: set[str]) -> tuple[str record its provenance keyed on the exact inventory it was applied against. """ from flowx.discovery_insights import inventory_fingerprint + from flowx.routing import plan_binding_violations metadata = Path(output_dir) / METADATA_DIRNAME - plan_path = metadata / "conversion_plan.json" inventory_path = metadata / INVENTORY_FILENAME - if not plan_path.exists(): + try: + recorded = ConversionPlan.load(Path(output_dir)) + except ValueError as error: + return None, None, str(error) + if recorded is None: return None, None, "No metadata/conversion_plan.json; record a routing decision with `route` first." if not inventory_path.exists(): return None, None, "No metadata/inventory.json; run the discover phase first." - plan = json.loads(plan_path.read_text(encoding="utf-8")) inventory = json.loads(inventory_path.read_text(encoding="utf-8")) - recorded_fingerprint = plan.get("inventory_sha256") current_fingerprint = inventory_fingerprint(inventory) - if recorded_fingerprint != current_fingerprint: - return ( - None, - None, - ( - "conversion_plan.json is stale: it was recorded against a different inventory " - f"({recorded_fingerprint!r} != {current_fingerprint!r}); re-run `route` before filling." - ), - ) + stale = plan_binding_violations(recorded, inventory) + if stale: + return None, None, f"conversion_plan.json is stale: {stale[0]}; re-run `route` before filling." + plan = recorded.to_dict() for component in plan.get("components", []): if not isinstance(component, dict): diff --git a/src/flowx/routing.py b/src/flowx/routing.py index e409cfc5..abb97c8d 100644 --- a/src/flowx/routing.py +++ b/src/flowx/routing.py @@ -40,22 +40,19 @@ from __future__ import annotations import json -import os from collections import Counter from collections.abc import Iterable from pathlib import Path from typing import Any from flowx.discovery_insights import inventory_fingerprint -from flowx.models.conversion_plan import DECISIONS, SCHEMA_VERSION +from flowx.models.conversion_plan import DECISIONS, PLAN_FILENAME, SCHEMA_VERSION, ComponentPlan, ConversionPlan # The strategy value that marks an activity as individually engine-capable (a 1:1 deterministic # translation). Any other value (``"agentic"`` / ``"unsupported"`` / missing) is a gap unless the # activity is claimed by a detected motif -- the multi-activity capability signal from #64. _DETERMINISTIC_STRATEGY = "deterministic" -# The recorded conversion-plan artifact lives beside inventory.json under metadata/. -PLAN_FILENAME = "conversion_plan.json" # Sources whose inventories route may send agentic in Phase 1. An inventory with no top-level # ``source`` predates the unified emitter and is ADF. @@ -64,9 +61,9 @@ # Authored top-level keys (everything else the library owns and rejects on input). _PLAN_TOP_KEYS = {"components"} -_LIBRARY_TOP_KEYS = {"schema_version", "inventory_sha256", "findings"} +_LIBRARY_TOP_KEYS = {"schema_version", "inventory_sha256", "source_graphs_sha256", "source_insights_sha256", "findings"} # Authored per-component keys vs the fields the library recomputes and rejects on input. -_COMPONENT_AUTHORED_KEYS = {"component_id", "members", "decision", "rationale"} +_COMPONENT_AUTHORED_KEYS = {"component_id", "members", "decision", "rationale", "assignments"} _COMPONENT_LIBRARY_KEYS = {"recommended", "options"} @@ -474,6 +471,12 @@ def _validate_component_entry( if rationale is not None and (not isinstance(rationale, str) or not rationale.strip()): violations.append(f"{loc}: 'rationale' must be a non-empty string when present") + if component.get("assignments"): + violations.append( + f"{loc}: 'assignments' (per-node or subgraph routing) is reserved for Phase 2; " + "decide the whole component and leave it empty" + ) + component_id = component.get("component_id") if not isinstance(component_id, str) or not component_id: violations.append(f"{loc}: 'component_id' must be a non-empty string") @@ -528,13 +531,14 @@ def load_plan(*, plan: dict[str, Any] | None = None, plan_path: Path | None = No return json.loads(Path(plan_path).read_text(encoding="utf-8")) -def build_plan_document(inventory: dict[str, Any], raw: dict[str, Any]) -> dict[str, Any]: - """Build the recorded plan document from a validated authored plan. +def build_plan(inventory: dict[str, Any], raw: dict[str, Any]) -> ConversionPlan: + """Build the typed plan from a validated authored plan. The library recomputes ``members`` / ``recommended`` / both ``options`` and overlays only the authored ``decision`` (and optional ``rationale``) per component, so the recorded facts cannot - drift from the inventory. Stamps the library-owned ``schema_version`` and ``inventory_sha256``. - Does not mutate the inputs and performs no I/O. + drift from the inventory. It stamps the schema version and the hashes that bind the plan to what + it was decided on: the inventory fingerprint, the saved source graphs the inventory records, and + the saved source insights when enrich ran. Does not mutate the inputs and performs no I/O. """ recommendation = build_recommendation(inventory) computed_by_id = _components_by_id(inventory) @@ -545,36 +549,53 @@ def build_plan_document(inventory: dict[str, Any], raw: dict[str, Any]) -> dict[ component_id = members_to_id[frozenset(component["members"])] authored_by_id[component_id] = component - components_out: list[dict[str, Any]] = [] - for entry in recommendation["components"]: - component_id = entry["component_id"] - authored = authored_by_id[component_id] - recorded: dict[str, Any] = { - "component_id": component_id, - "members": entry["members"], - "recommended": entry["recommended"], - "decision": authored["decision"], - "options": entry["options"], - } - rationale = authored.get("rationale") - if rationale is not None: - recorded["rationale"] = rationale - components_out.append(recorded) + components = [ + ComponentPlan( + component_id=entry["component_id"], + members=entry["members"], + recommended=entry["recommended"], + decision=authored_by_id[entry["component_id"]]["decision"], + options=entry["options"], + rationale=authored_by_id[entry["component_id"]].get("rationale"), + ) + for entry in recommendation["components"] + ] + insights = inventory.get("insights") + return ConversionPlan( + schema_version=SCHEMA_VERSION, + inventory_sha256=inventory_fingerprint(inventory), + source_graphs_sha256=inventory.get("source_graphs_sha256"), + source_insights_sha256=insights.get("source_insights_sha256") if isinstance(insights, dict) else None, + components=components, + findings=recommendation["findings"], + ) - return { - "schema_version": SCHEMA_VERSION, - "inventory_sha256": inventory_fingerprint(inventory), - "components": components_out, - "findings": recommendation["findings"], - } +def build_plan_document(inventory: dict[str, Any], raw: dict[str, Any]) -> dict[str, Any]: + """The recorded JSON form of :func:`build_plan`.""" + return build_plan(inventory, raw).to_dict() + + +def plan_binding_violations(plan: ConversionPlan, inventory: dict[str, Any]) -> list[str]: + """Check a recorded plan still matches the inventory, source graphs and insights it was decided on. -def _write_plan_atomic(path: Path, document: dict[str, Any]) -> None: - """Write the plan JSON atomically (temp file + ``os.replace``), matching the inventory formatting.""" - path.parent.mkdir(parents=True, exist_ok=True) - temporary = path.with_name(f".{path.name}.tmp") - temporary.write_text(json.dumps(document, indent=2), encoding="utf-8") - os.replace(temporary, path) + Returns one message per mismatch; an empty list means the plan is current. A plan is stale when + discover or enrich ran again after it was recorded. + """ + violations: list[str] = [] + current = inventory_fingerprint(inventory) + if plan.inventory_sha256 != current: + violations.append( + f"{PLAN_FILENAME} was recorded against inventory {plan.inventory_sha256!r} but the current " + f"inventory is {current!r}; re-run route against the current inventory" + ) + if plan.source_graphs_sha256 != inventory.get("source_graphs_sha256"): + violations.append(f"{PLAN_FILENAME} was recorded against different source graphs; re-run route") + insights = inventory.get("insights") + current_insights = insights.get("source_insights_sha256") if isinstance(insights, dict) else None + if plan.source_insights_sha256 != current_insights: + violations.append(f"{PLAN_FILENAME} was recorded against different source insights; re-run route") + return violations def record_plan( @@ -610,12 +631,14 @@ def record_plan( if violations: return {"ok": False, "violations": violations, "components": 0} - document = build_plan_document(inventory, raw) - _write_plan_atomic(inventory_path.with_name(PLAN_FILENAME), document) + recorded = build_plan(inventory, raw) + recorded.write(Path(output_dir)) return { "ok": True, "violations": [], - "inventory_sha256": document["inventory_sha256"], - "components": len(document["components"]), - "findings": len(document["findings"]), + "inventory_sha256": recorded.inventory_sha256, + "source_graphs_sha256": recorded.source_graphs_sha256, + "source_insights_sha256": recorded.source_insights_sha256, + "components": len(recorded.components), + "findings": len(recorded.findings), } diff --git a/tests/unit/test_conversion_plan_v2.py b/tests/unit/test_conversion_plan_v2.py new file mode 100644 index 00000000..82debb6c --- /dev/null +++ b/tests/unit/test_conversion_plan_v2.py @@ -0,0 +1,152 @@ +"""The typed conversion plan (schema 2): the library's load, validate and record path. + +The plan is bound to what it was decided on: the deterministic inventory fingerprint, the saved +source graphs hash the inventory records (H0) and the saved source insights hash (H1). Phase 1 +decides whole components; per-node assignments are reserved and must stay empty. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from flowx import routing +from flowx.bundler.dab_writer import main as package_main +from flowx.discovery_inventory import STRATEGY_PROPERTY, build_source_inventory +from flowx.ir_serde import pipeline_to_dict +from flowx.models.conversion_plan import SCHEMA_VERSION, ComponentPlan, ConversionPlan, NodeAssignment +from flowx.models.discovery import CONCEPT_NOTEBOOK, SourceGraph, SourceNode +from flowx.models.ir import NotebookActivity, Pipeline +from flowx.route_agentic import REPORT_FILENAME, WORK_DIRNAME, apply_plan + + +def _inventory(insights_hash: str | None = "insights-1") -> dict[str, Any]: + node = SourceNode( + source_id="load", + task_key="load", + concept=CONCEPT_NOTEBOOK, + source="adf", + name="load", + native_type="DatabricksNotebook", + properties={STRATEGY_PROPERTY: "agentic"}, + ) + inventory = build_source_inventory( + [SourceGraph(name="solo", source="adf", tasks=[node])], + source="adf", + source_dir="/src", + source_graphs_sha256="graphs-1", + ) + if insights_hash is not None: + inventory["insights"] = {"schema_version": "1", "source_insights_sha256": insights_hash} + return inventory + + +def _setup(output_dir: Path, inventory: dict[str, Any]) -> None: + metadata = output_dir / "metadata" + metadata.mkdir(parents=True, exist_ok=True) + (metadata / "inventory.json").write_text(json.dumps(inventory, indent=2), encoding="utf-8") + work = output_dir / WORK_DIRNAME + work.mkdir(parents=True, exist_ok=True) + pipeline = Pipeline( + name="solo", + tasks=[NotebookActivity(name="load", task_key="load", notebook_path="/Workspace/Shared/load")], + tags={"source": "adf"}, + ) + (work / REPORT_FILENAME).write_text(json.dumps({"pipelines": [pipeline_to_dict(pipeline)]}), encoding="utf-8") + + +def _decide(decision: str, **extra: Any) -> dict[str, Any]: + return {"components": [{"component_id": "component-1", "members": ["solo"], "decision": decision, **extra}]} + + +def test_plan_round_trips_including_reserved_assignments(tmp_path: Path) -> None: + plan = ConversionPlan( + inventory_sha256="inv", + source_graphs_sha256="h0", + source_insights_sha256="h1", + components=[ + ComponentPlan( + component_id="component-1", + members=["solo"], + decision="agentic", + assignments=[NodeAssignment(pipeline="solo", task_keys=["load"], route="agentic", pattern="lfc")], + ) + ], + ) + + plan.write(tmp_path) + + assert ConversionPlan.load(tmp_path) == plan + assert json.loads((tmp_path / "metadata" / "conversion_plan.json").read_text())["schema_version"] == "2" + + +def test_a_plan_from_another_schema_version_is_refused(tmp_path: Path) -> None: + metadata = tmp_path / "metadata" + metadata.mkdir() + (metadata / "conversion_plan.json").write_text(json.dumps({"schema_version": "1", "components": []})) + + with pytest.raises(ValueError, match="Re-run route"): + ConversionPlan.load(tmp_path) + + +def test_recorded_plan_is_bound_to_the_source_graphs_and_insights(tmp_path: Path) -> None: + _setup(tmp_path, _inventory()) + + result = routing.record_plan(tmp_path, plan=_decide("agentic")) + recorded = ConversionPlan.load(tmp_path) + + assert result["ok"] is True + assert recorded is not None + assert recorded.schema_version == SCHEMA_VERSION + assert recorded.source_graphs_sha256 == "graphs-1" + assert recorded.source_insights_sha256 == "insights-1" + assert result["source_graphs_sha256"] == "graphs-1" + + +def test_per_node_assignments_are_reserved_for_phase_2() -> None: + inventory = _inventory() + assignment = {"pipeline": "solo", "task_keys": ["load"], "route": "agentic"} + + assert routing.validate_plan(_decide("agentic", assignments=[]), inventory) == [] + violations = routing.validate_plan(_decide("agentic", assignments=[assignment]), inventory) + assert any("reserved for Phase 2" in violation for violation in violations) + + +def test_authored_plans_cannot_set_the_bound_hashes() -> None: + plan = {**_decide("agentic"), "source_graphs_sha256": "x", "source_insights_sha256": "y"} + + violations = routing.validate_plan(plan, _inventory()) + + assert any("source_graphs_sha256" in violation and "library" in violation for violation in violations) + assert any("source_insights_sha256" in violation and "library" in violation for violation in violations) + + +def test_package_refuses_a_plan_when_the_insights_changed_after_route( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """Only the H1 binding catches this: the inventory fingerprint ignores the insights block.""" + _setup(tmp_path, _inventory("insights-1")) + assert routing.record_plan(tmp_path, plan=_decide("deterministic"))["ok"] is True + (tmp_path / "metadata" / "inventory.json").write_text(json.dumps(_inventory("insights-2")), encoding="utf-8") + + code = package_main(["--output-dir", str(tmp_path), "--no-download-workspace-files", "--keep-intermediates"]) + + assert code == 1 + assert "different source insights" in capsys.readouterr().err + assert not (tmp_path / "databricks.yml").exists() + + +def test_apply_plan_refuses_a_stale_plan(tmp_path: Path) -> None: + _setup(tmp_path, _inventory()) + assert routing.record_plan(tmp_path, plan=_decide("agentic"))["ok"] is True + recorded = ConversionPlan.load(tmp_path) + assert recorded is not None + report_bytes = (tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_bytes() + (tmp_path / "metadata" / "inventory.json").write_text(json.dumps(_inventory("insights-2")), encoding="utf-8") + + with pytest.raises(ValueError, match="different source insights"): + apply_plan(tmp_path, recorded) + assert (tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_bytes() == report_bytes diff --git a/tests/unit/test_route_audit.py b/tests/unit/test_route_audit.py index 09074d3d..b478ef48 100644 --- a/tests/unit/test_route_audit.py +++ b/tests/unit/test_route_audit.py @@ -13,20 +13,19 @@ from pathlib import Path from flowx.bundler.dab_writer import _write_route_audit +from flowx.models.conversion_plan import ComponentPlan, ConversionPlan def _write_plan(output_dir: Path) -> None: - metadata = output_dir / "metadata" - metadata.mkdir(parents=True, exist_ok=True) - plan = { - "schema_version": "1", - "inventory_sha256": "abc123", - "components": [ - {"component_id": "component-1", "members": ["parent", "child"], "decision": "agentic"}, - {"component_id": "component-2", "members": ["solo"], "decision": "deterministic"}, + ConversionPlan( + inventory_sha256="abc123", + source_graphs_sha256="graphs123", + source_insights_sha256="insights123", + components=[ + ComponentPlan(component_id="component-1", members=["parent", "child"], decision="agentic"), + ComponentPlan(component_id="component-2", members=["solo"], decision="deterministic"), ], - } - (metadata / "conversion_plan.json").write_text(json.dumps(plan, indent=2), encoding="utf-8") + ).write(output_dir) def _write_gaps(output_dir: Path) -> None: @@ -58,6 +57,8 @@ def test_route_audit_summarises_components_decisions_and_gaps(tmp_path: Path) -> assert audit_path == tmp_path / "metadata" / "route_audit.json" audit = json.loads(audit_path.read_text(encoding="utf-8")) assert audit["recorded_against_inventory_sha256"] == "abc123" + assert audit["source_graphs_sha256"] == "graphs123" + assert audit["source_insights_sha256"] == "insights123" # Both routed components and their decisions are recorded. decisions = {component["component_id"]: component["decision"] for component in audit["components"]} assert decisions == {"component-1": "agentic", "component-2": "deterministic"} From b839e96ba887f75e13ca39160988e75c38b948b2 Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Mon, 5 Oct 2026 21:13:31 +0100 Subject: [PATCH 017/122] Bring docs, AGENTS.md and the migrate and route skills in line with the saved graphs, insights and plan Wording only. The migrate workflow and route skill still said enrich merges into inventory.json, and the migrate skill still said Airflow routing aligns via #63; they now describe source_insights.json and that route refuses agentic decisions for Airflow inventories. AGENTS.md lists source_graphs.json in discover's outputs, adds rows for discovery_serde, discovery_inventory and discovery_insights, and describes the typed ConversionPlan and the ADF-only route. The README and docs output trees gain source_insights.json and conversion_plan.json, placed away from the lines #108 changes. Co-authored-by: Isaac --- AGENTS.md | 11 +++++++---- README.md | 2 ++ docs/content/docs/architecture.mdx | 2 +- docs/content/docs/guide.mdx | 2 ++ skills/flowx-migrate/SKILL.md | 5 +++-- skills/flowx-migrate/references/workflow.md | 3 ++- skills/flowx-route/SKILL.md | 2 +- 7 files changed, 18 insertions(+), 9 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 5f7b3729..112ce32e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -56,7 +56,7 @@ All three phases write into one shared `` (default `./flowx_output`) the DAB bundle at the top level, kept artifacts under `metadata/`, and transient intermediates under `.work/` (pruned by `package`). -1. **Discover** -- Parse ADF JSON from UC volumes -> typed AST -> `metadata/inventory.json` + `metadata/profile_report.csv` + verbatim `metadata/.arm.json` +1. **Discover** -- Parse ADF JSON from UC volumes -> typed AST -> `SourceGraph` -> `metadata/source_graphs.json` (versioned, hashed, motifs inside) + `metadata/inventory.json` + `metadata/profile_report.csv` + verbatim `metadata/.arm.json`; optional enrich adds `metadata/source_insights.json` and rebuilds the inventory with it 2. **Convert** -- Registry dispatch + topological sort -> Pipeline IR (deterministic + agentic gaps); transient report at `.work/translation_report.json` 3. **Package** -- IR -> DAB YAML + generated notebooks + setup scripts; prunes `.work/` @@ -74,7 +74,10 @@ intermediates under `.work/` (pruned by `package`). | `models/ir.py` | Databricks intermediate representation | | `models/dab.py` | DAB output schema types | | `models/discovery.py` | Source-neutral discovery graph contract (`SourceGraph`); each source keeps its own parser and projects into it | -| `sources/adf/loader.py` | Parses ADF exports, produces `metadata/inventory.json` + `metadata/profile_report.csv` | +| `discovery_serde.py` | SourceGraph JSON round trip and the persisted `source_graphs.json` envelope (contract version, per-graph and document hashes) | +| `discovery_inventory.py` | The one source-agnostic emitter that projects source graphs into `inventory.json` | +| `discovery_insights.py` | `enrich`: validates agent-authored insights, records `source_insights.json` and rebuilds `inventory.json` with it | +| `sources/adf/loader.py` | Parses ADF exports, produces `metadata/source_graphs.json` + `metadata/inventory.json` + `metadata/profile_report.csv` | | `sources/adf/discovery_mapping.py` | Maps the ADF AST onto the `SourceGraph` discovery graph contract (1:1, lossless) | | `sources/adf/translate.py` | Registry dispatch, topological sort, context threading | | `sources/adf/translators/` | One module per deterministic activity type (16 total) | @@ -89,9 +92,9 @@ intermediates under `.work/` (pruned by `package`). | `reporting/coverage.py` | Builds per-pipeline coverage rows from `metadata/` | | `reporting/results.py` | Writes per-run coverage to a UC table (run_id/run_date/run_by) via the SDK | | `reporting/dashboard.py` | Installs + publishes an AI/BI coverage dashboard over the results table | -| `routing.py` | Groups pipelines into connected components over control lineage; recommends deterministic/agentic per component (both options) and records the user's decision as `metadata/conversion_plan.json` (additive; convert untouched) | +| `routing.py` | Groups pipelines into connected components over control lineage; recommends deterministic/agentic per component (both options) and records the user's decision as `metadata/conversion_plan.json` (additive; convert untouched). Agentic decisions are ADF-only | | `route_agentic.py` | Applies a routing decision after convert: rewrites the translation report for agentic-routed groups (placeholder tasks + per-task `AgenticGap`), keeping the fill in-engine | -| `models/conversion_plan.py` | Source-neutral conversion-plan artifact model (per-component decision + both options), the routing counterpart to `models/insights.py` | +| `models/conversion_plan.py` | Typed `ConversionPlan` (schema 2): the library's load/validate/record API for `conversion_plan.json`, one decision per component, bound to the inventory, `source_graphs.json` and `source_insights.json` hashes; package fails closed on a mismatch | ## Activity Types diff --git a/README.md b/README.md index ea3e7e8e..e78156c1 100644 --- a/README.md +++ b/README.md @@ -259,6 +259,8 @@ flowx_output/ inventory.json # discover: activity inventory profile_report.csv # discover: per-pipeline complexity report .arm.json # discover: verbatim original ADF/ARM source + source_insights.json # enrich: agent-authored insights, bound to source_graphs.json + conversion_plan.json # route: the recorded per-component conversion decision configuration.json # modify: collected configuration answers .work/ # transient intermediates (translation report, IR, gaps.json); pruned by package ``` diff --git a/docs/content/docs/architecture.mdx b/docs/content/docs/architecture.mdx index 8bb2e3ac..14c122d9 100644 --- a/docs/content/docs/architecture.mdx +++ b/docs/content/docs/architecture.mdx @@ -19,7 +19,7 @@ All three phases share one `output_dir` (default `./flowx_output`): | Phase | Module | Reads | Writes | |---------------|-------------------------|------------------------|------------------------------------------------------------------------------------------| -| **profile** | `parser/adf_loader.py` | ADF JSON exports | `metadata/inventory.json`, `metadata/profile_report.csv`, `metadata/.arm.json` | +| **profile** | `parser/adf_loader.py` | ADF JSON exports | `metadata/source_graphs.json`, `metadata/inventory.json`, `metadata/profile_report.csv`, `metadata/.arm.json` | | **translate** | `translator/engine.py` | inventory + ADF source | `.work/translation_report.json` (transient IR) | | **prepare** | `bundler/dab_writer.py` | translation report | `databricks.yml`, `resources/`, `src/`, `setup/` (and prunes `.work/`) | diff --git a/docs/content/docs/guide.mdx b/docs/content/docs/guide.mdx index 821e9da1..b3121c37 100644 --- a/docs/content/docs/guide.mdx +++ b/docs/content/docs/guide.mdx @@ -80,6 +80,8 @@ flowx_output/ │ ├── inventory.json # discover: activity inventory │ ├── profile_report.csv # profile: per-pipeline complexity report │ ├── .arm.json # discover: verbatim original ADF/ARM pipeline source +│ ├── source_insights.json # enrich: agent-authored insights, bound to source_graphs.json +│ ├── conversion_plan.json # route: the recorded per-component conversion decision │ └── configuration.json # modify: the collected configuration answers └── .work/ # transient intermediates (translation report, IR, gaps.json); pruned by prepare ``` diff --git a/skills/flowx-migrate/SKILL.md b/skills/flowx-migrate/SKILL.md index cc2a90ee..617c454e 100644 --- a/skills/flowx-migrate/SKILL.md +++ b/skills/flowx-migrate/SKILL.md @@ -285,8 +285,9 @@ For failures, suggest: **ADF only today.** Routing (and the agentic fill in Step 8) applies to **ADF**. For **Airflow**, skip Steps 7–8 and take the deterministic report straight to configuration/package; resolve any Airflow -agentic gaps with the **`flowx-resolve-airflow-gaps`** skill instead (Airflow discovery does not yet -emit control lineage or motifs — routing aligns via #63). +agentic gaps with the **`flowx-resolve-airflow-gaps`** skill instead. `route` refuses an agentic +decision for an Airflow inventory; Airflow agentic conversion follows the Airflow track's own per-gap +contract. Routing reads the deterministic baseline report `.work/translation_report.json` (built in Step 5) and edits it. Invoke the **`flowx:flowx-route`** skill to present the per-connected-component diff --git a/skills/flowx-migrate/references/workflow.md b/skills/flowx-migrate/references/workflow.md index cb345ef5..29b83822 100644 --- a/skills/flowx-migrate/references/workflow.md +++ b/skills/flowx-migrate/references/workflow.md @@ -70,7 +70,8 @@ merges: - **Enrich (default):** the agent authors an `insights` layer (factory-wide recommendation, per-pipeline intent + recommended Databricks patterns, cross-pipeline relationships) and `enrich` - merges it into `metadata/inventory.json` under a single additive `insights` key, leaving every + records it in `metadata/source_insights.json` (bound to the saved `source_graphs.json`) and + rebuilds `metadata/inventory.json` with it under a single additive `insights` key, leaving every deterministic key byte-identical. Skippable for a deterministic-only, headless pass. - **Route:** groups pipelines into connected components over control lineage and, per component, records a `deterministic` or `agentic` decision as the fingerprint-bound diff --git a/skills/flowx-route/SKILL.md b/skills/flowx-route/SKILL.md index e4af4688..4f7ccb7a 100644 --- a/skills/flowx-route/SKILL.md +++ b/skills/flowx-route/SKILL.md @@ -163,7 +163,7 @@ Exit 1 (nothing written) on a missing report it cannot produce, or a plan that f Every routed-agentic pipeline's tasks are now `PlaceholderActivity` nodes with one pipeline-tagged `AgenticGap` each. Two fills exist depending on the grain; you author the replacement (no LLM in the -library — it validates and merges). +library, which validates and records). ### 3a — Per-pipeline fill (keep the pipeline 1:1) From 1e7cbe73577ec5db8c63ff6f98f5c77489afc369 Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Wed, 7 Oct 2026 09:53:38 +0100 Subject: [PATCH 018/122] no-mistakes(review): Replace routing markers with a per-component routing record --- skills/flowx-migrate/references/workflow.md | 7 +- skills/flowx-route/SKILL.md | 25 +- src/flowx/adapter/__main__.py | 32 ++- src/flowx/bundler/dab_writer.py | 88 +++---- src/flowx/ir_serde.py | 6 + src/flowx/mcp/server.py | 10 +- src/flowx/models/conversion_plan.py | 95 +------- src/flowx/route_agentic.py | 243 +++++++++++++------ src/flowx/routing.py | 5 - tests/unit/test_package_routing_preflight.py | 126 +++++++++- tests/unit/test_route_agentic_flow.py | 107 +++++++- tests/unit/test_route_audit.py | 4 +- 12 files changed, 508 insertions(+), 240 deletions(-) diff --git a/skills/flowx-migrate/references/workflow.md b/skills/flowx-migrate/references/workflow.md index 29b83822..7b0d4fd1 100644 --- a/skills/flowx-migrate/references/workflow.md +++ b/skills/flowx-migrate/references/workflow.md @@ -76,8 +76,11 @@ merges: - **Route:** groups pipelines into connected components over control lineage and, per component, records a `deterministic` or `agentic` decision as the fingerprint-bound `metadata/conversion_plan.json`. Recording an agentic decision edits `.work/translation_report.json` - so those groups' tasks become placeholder gaps; a fully-deterministic plan (or no plan) leaves - convert/package behaving exactly as before — the non-breaking guarantee. + so those groups' tasks become placeholder gaps and stamps a routing record onto it (per component: + members, decision and outcome, plus the plan and baseline report hashes); a fully-deterministic + plan (or no plan) leaves convert/package behaving exactly as before — the non-breaking guarantee. + The fills and `modify` carry the record forward; re-routing under different decisions is refused + (re-run convert, then route), and package refuses a report whose record no longer matches the plan. Routed-agentic groups are then filled during convert: per-pipeline via `convert --merge-agentic`, or cross-pipeline (N→M, e.g. a Lakeflow Connect collapse) via `fill-agentic combine` using authored diff --git a/skills/flowx-route/SKILL.md b/skills/flowx-route/SKILL.md index 4f7ccb7a..ade1afe3 100644 --- a/skills/flowx-route/SKILL.md +++ b/skills/flowx-route/SKILL.md @@ -99,6 +99,15 @@ Take the per-component decision and record it. `route` validates the plan, write routed-**agentic** group's tasks become placeholder gaps (deterministic groups are left byte-identical; when nothing is routed agentic the report is untouched — the non-breaking guarantee). +When it edits the report, route also stamps a **routing record** onto it (the top-level +`_routing_record` key): the plan's hash, the hash of the deterministic report it started from, and one +entry per `component_id` with its `members`, `decision` and `outcome` — `deterministic`, +`agentic-not-viable` (nothing filled yet) or `agentic-applied` (a fill left none of its members with a +routed placeholder). Every later rewrite (the fills and `modify`) carries the record forward and only +the fills update outcomes. Re-running route under the **same** decisions only refreshes the record's +plan hash; under different components or decisions it refuses and writes nothing — re-run convert, +then route. + There are three ways to supply the decision: - **Interactive prompt (TTY).** Run `route` with no `--plan-path` on a real terminal and it asks, per @@ -157,7 +166,8 @@ the convert phase in-process first. Otherwise run `flowx-convert` before routing [--source adf --source-path ] # only needed to trigger convert when the report is missing ``` -Exit 1 (nothing written) on a missing report it cannot produce, or a plan that fails validation. +Exit 1 (nothing written) on a missing report it cannot produce, a plan that fails validation, or a +report already routed under different components or decisions. ## Step 3 — Fill the routed-agentic groups @@ -226,7 +236,10 @@ MCP: `flowx(command="fill_agentic", parameters={"output_dir": ..., "members": [. - `--members` (comma-separated; MCP accepts a list) must **exactly match** a routed-**agentic** component in the recorded `metadata/conversion_plan.json`, whose `inventory_sha256`, `source_graphs_sha256` and `source_insights_sha256` must still match the current inventory. A partial group, a superset, a typo, or a deterministic component is refused - — you can't swap pipelines the plan didn't route agentic. + — you can't swap pipelines the plan didn't route agentic. The report must carry a routing record + that matches that plan (route has applied it). +- Re-running combine once the component's outcome is `agentic-applied` is a no-op + (`already_combined: true`), so the authored pipelines are never appended twice. - `--pipelines-path` is a JSON **list** of pipeline IR dicts (the authored replacements), typically carrying `AgenticComponentActivity` nodes (see below). - Each authored pipeline **must** carry the source tag `"tags": {"source": "adf"}` (routing/agentic @@ -304,11 +317,15 @@ applies when authoring recommended patterns.) Once the routed-agentic groups are filled and the report validates, continue with `flowx-convert`'s just-in-time configuration (`inspect`/`modify`) as usual, then `flowx-package`. The recorded -`metadata/conversion_plan.json` is kept alongside `inventory.json` as the routing record. It is the +`metadata/conversion_plan.json` is kept alongside `inventory.json` as the decision of record. It is the library's typed `ConversionPlan` (schema 2): one decision per component, bound to the inventory fingerprint, the saved `source_graphs.json` and the saved `source_insights.json` it was decided on, with a reserved, empty `assignments` list per component for per-node routing later. Package refuses -to run when any of those no longer match. +to run when any of those no longer match, or when the packaged report's routing record no longer +matches the plan (a different plan hash, different components, members or decisions, or no record +although the plan routes a component agentic). `metadata/route_audit.json` keeps each component's +decision and outcome from the record, the baseline report hash, and the gaps in agentic-routed +pipelines. ## Reference diff --git a/src/flowx/adapter/__main__.py b/src/flowx/adapter/__main__.py index 933da9b0..aa79ed56 100644 --- a/src/flowx/adapter/__main__.py +++ b/src/flowx/adapter/__main__.py @@ -182,12 +182,13 @@ def _run_route(args: argparse.Namespace) -> int: groups untouched; nothing routed agentic leaves the report byte-identical). Triggers the convert phase in-process when the report is missing and ``--source`` / - ``--source-path`` are supplied. Returns 1 on a missing report it cannot produce, or on a plan - that fails validation (report + plan left untouched). + ``--source-path`` are supplied. Returns 1 on a missing report it cannot produce, on a plan that + fails validation, or when the report was already routed under different components or decisions + (report + plan left untouched in each case). """ from flowx import routing from flowx.models.conversion_plan import ConversionPlan - from flowx.route_agentic import REPORT_FILENAME, WORK_DIRNAME, apply_plan + from flowx.route_agentic import REPORT_FILENAME, WORK_DIRNAME, apply_plan, reroute_conflict inventory_path = args.output_dir / "metadata" / "inventory.json" if not inventory_path.exists(): @@ -223,6 +224,16 @@ def _run_route(args: argparse.Namespace) -> int: ) return 1 + if not routing.validate_plan(plan, inventory): + try: + conflict = reroute_conflict(args.output_dir, plan) + except (OSError, json.JSONDecodeError) as error: + print(f"Failed to read {report_path}: {error}", file=sys.stderr) + return 1 + if conflict is not None: + print(f"Refusing to route: {conflict}.", file=sys.stderr) + return 1 + try: result = routing.record_plan(args.output_dir, plan=plan) except (FileNotFoundError, OSError, ValueError, json.JSONDecodeError) as error: @@ -1247,13 +1258,18 @@ def _emit_json(payload: dict[str, Any], out: Path | None) -> None: def _write_modified_report(report_path: Path, pipelines: list[dict[str, Any]], out: Path) -> None: """Writes the configuration-stamped IR to *out* using the input report's shape. + A routing record on the input report is carried over unchanged, inside the ``pipelines`` + wrapper, so package can still check the stamped report against the recorded plan. + Args: report_path: Path the modified report was sourced from. Used - only to detect whether the input was a single pipeline IR - or an aggregated translation report. + to detect whether the input was a single pipeline IR or an + aggregated translation report, and to read its routing record. pipelines: Stamped pipeline IR dicts to write. out: Destination path for the modified report. """ + from flowx.route_agentic import ROUTING_RECORD_KEY, routing_record + raw = json.loads(report_path.read_text(encoding="utf-8")) if isinstance(raw, dict) and "translations" in raw: by_name = {pipeline["name"]: pipeline for pipeline in pipelines} @@ -1264,7 +1280,11 @@ def _write_modified_report(report_path: Path, pipelines: list[dict[str, Any]], o out.parent.mkdir(parents=True, exist_ok=True) out.write_text(json.dumps(raw, indent=2, default=str) + "\n", encoding="utf-8") return - payload = pipelines[0] if len(pipelines) == 1 else {"pipelines": pipelines} + record = routing_record(raw) + if record is not None: + payload: Any = {"pipelines": pipelines, ROUTING_RECORD_KEY: record} + else: + payload = pipelines[0] if len(pipelines) == 1 else {"pipelines": pipelines} out.parent.mkdir(parents=True, exist_ok=True) out.write_text(json.dumps(payload, indent=2, default=str) + "\n", encoding="utf-8") diff --git a/src/flowx/bundler/dab_writer.py b/src/flowx/bundler/dab_writer.py index 38c712bc..e45b1c8d 100644 --- a/src/flowx/bundler/dab_writer.py +++ b/src/flowx/bundler/dab_writer.py @@ -380,14 +380,15 @@ def _routing_plan_failures(output_dir: Path, report_path: Path) -> list[str]: """Check a recorded routing plan still matches what is about to be packaged. A plan is bound to the inventory, the saved source graphs and the saved source insights it was - decided on. If discover or enrich ran again since route, or a combine fill was applied under a - different plan, the report no longer reflects the user's decision, so package refuses rather - than shipping it. Returns one message per problem; an empty list means there is no recorded plan - or it still matches. + decided on. If discover or enrich ran again since route, package refuses rather than shipping a + report that no longer reflects the user's decision. It also refuses when the report's routing + record disagrees with the plan: a different plan hash, different components, members or + decisions, or no record although the plan routes a component agentic. Returns one message per + problem; an empty list means there is no recorded plan or it still matches. """ - from flowx.discovery_insights import inventory_fingerprint + from flowx.discovery_serde import canonical_sha256 from flowx.models.conversion_plan import DECISION_AGENTIC, ConversionPlan - from flowx.route_agentic import COMBINE_PROVENANCE_KEY + from flowx.route_agentic import routing_record, routing_record_mismatches from flowx.routing import plan_binding_violations metadata_dir = Path(output_dir) / "metadata" @@ -416,24 +417,18 @@ def _routing_plan_failures(output_dir: Path, report_path: Path) -> list[str]: report = json.loads(Path(report_path).read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): return [] - provenance = report.get(COMBINE_PROVENANCE_KEY) if isinstance(report, dict) else None - if not isinstance(provenance, list): + report_name = Path(report_path).name + remedy = "re-run convert, then route" + record = routing_record(report) + if record is None: + if any(component.decision == DECISION_AGENTIC for component in plan.components): + return [f"the plan routes a component agentic but {report_name} carries no routing record; {remedy}"] return [] - current = inventory_fingerprint(inventory) - agentic_components = { - (component.component_id, tuple(sorted(component.members))) - for component in plan.components - if component.decision == DECISION_AGENTIC - } + plan_document = plan.to_dict() failures: list[str] = [] - for entry in provenance: - if not isinstance(entry, dict): - continue - component_id = entry.get("component_id") - if entry.get("inventory_sha256") != current: - failures.append(f"combine for {component_id!r} was applied against a different inventory") - elif (component_id, tuple(sorted(entry.get("members") or []))) not in agentic_components: - failures.append(f"combine for {component_id!r} does not match an agentic component in the recorded plan") + if record.get("conversion_plan_sha256") != canonical_sha256(plan_document): + failures.append(f"{report_name} was routed under a different conversion plan; {remedy}") + failures.extend(f"{mismatch}; {remedy}" for mismatch in routing_record_mismatches(record, plan_document)) return failures @@ -450,18 +445,19 @@ def _write_route_audit(output_dir: Path, report_path: Path | None = None) -> Pat Package prunes the transient ``.work/`` folder (translation report + ``gaps.json``) by default, which erases the "what did routing change?" trail. When a routing decision was recorded - (``metadata/conversion_plan.json`` exists), summarise the routed components, their decisions, and - the gaps routing introduced into an additive ``metadata/`` artifact that survives the prune. It - also records hashes of the inventory, the saved source graphs and source insights the plan was - bound to, the plan, and the packaged report, plus the report route started from when route - recorded one, so the trail can be checked against those files later. + (``metadata/conversion_plan.json`` exists), summarise the routed components, their decisions and + their outcomes from the report's routing record, and the gaps routing introduced in agentic-routed + pipelines, into an additive ``metadata/`` artifact that survives the prune. It also records hashes + of the inventory, the saved source graphs and source insights the plan was bound to, the plan, and + the packaged report, plus the report route started from as the routing record names it, so the + trail can be checked against those files later. Returns the written path, or ``None`` when there is no recorded plan (no routing happened) -- so the no-route path writes nothing and stays byte-identical. """ from flowx.discovery_insights import inventory_fingerprint - from flowx.models.conversion_plan import DECISION_AGENTIC, PLAN_FILENAME, ConversionPlan - from flowx.route_agentic import ROUTE_PROVENANCE_KEY + from flowx.models.conversion_plan import DECISION_AGENTIC, DECISION_DETERMINISTIC, PLAN_FILENAME, ConversionPlan + from flowx.route_agentic import OUTCOME_DETERMINISTIC, routing_record metadata_dir = Path(output_dir) / "metadata" plan_path = metadata_dir / PLAN_FILENAME @@ -480,24 +476,34 @@ def _write_route_audit(output_dir: Path, report_path: Path | None = None) -> Pat except (OSError, json.JSONDecodeError): pass - baseline_report_sha256: str | None = None + record: dict[str, Any] | None = None if report_path is not None: try: - report = json.loads(Path(report_path).read_text(encoding="utf-8")) + record = routing_record(json.loads(Path(report_path).read_text(encoding="utf-8"))) except (OSError, json.JSONDecodeError): - report = None - route_provenance = report.get(ROUTE_PROVENANCE_KEY) if isinstance(report, dict) else None - if isinstance(route_provenance, dict): - baseline_report_sha256 = route_provenance.get("baseline_report_sha256") + record = None + recorded_components = record.get("components") if record is not None else None + if not isinstance(recorded_components, dict): + recorded_components = {} components: list[dict[str, Any]] = [] - agentic_pipelines: list[str] = [] + agentic_pipelines: set[str] = set() for component in plan.components: + recorded = recorded_components.get(component.component_id) + if isinstance(recorded, dict): + outcome = recorded.get("outcome") + else: + outcome = OUTCOME_DETERMINISTIC if component.decision == DECISION_DETERMINISTIC else None components.append( - {"component_id": component.component_id, "members": list(component.members), "decision": component.decision} + { + "component_id": component.component_id, + "members": list(component.members), + "decision": component.decision, + "outcome": outcome, + } ) if component.decision == DECISION_AGENTIC: - agentic_pipelines.extend(component.members) + agentic_pipelines.update(component.members) gaps_introduced: list[dict[str, Any]] = [] gaps_path = Path(output_dir) / ".work" / "gaps.json" @@ -507,7 +513,7 @@ def _write_route_audit(output_dir: Path, report_path: Path | None = None) -> Pat except (OSError, json.JSONDecodeError): gaps = [] for gap in gaps if isinstance(gaps, list) else []: - if isinstance(gap, dict): + if isinstance(gap, dict) and gap.get("pipeline") in agentic_pipelines: gaps_introduced.append( { "pipeline": gap.get("pipeline"), @@ -523,10 +529,10 @@ def _write_route_audit(output_dir: Path, report_path: Path | None = None) -> Pat "source_graphs_sha256": plan.source_graphs_sha256, "source_insights_sha256": plan.source_insights_sha256, "conversion_plan_sha256": _file_sha256(plan_path), - "baseline_report_sha256": baseline_report_sha256, + "baseline_report_sha256": record.get("baseline_report_sha256") if record is not None else None, "translation_report_sha256": _file_sha256(report_path) if report_path is not None else None, "components": components, - "agentic_pipelines": sorted(set(agentic_pipelines)), + "agentic_pipelines": sorted(agentic_pipelines), "gaps_count": len(gaps_introduced), "gaps_introduced": gaps_introduced, } diff --git a/src/flowx/ir_serde.py b/src/flowx/ir_serde.py index f41e69e0..9412137e 100644 --- a/src/flowx/ir_serde.py +++ b/src/flowx/ir_serde.py @@ -627,6 +627,9 @@ def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Pat ``NotebookActivity`` whose ``notebook_path`` points at a notebook the agent wrote to the workspace; the prepare phase then references it directly. + When the report carries a routing record, each agentic component's outcome is updated to show + whether the merge left any of its routed placeholders unfilled. + Args: report_path: ``translation_report.json`` produced by the translate phase. results_dir: Directory of per-activity result JSON files. @@ -658,6 +661,9 @@ def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Pat logger.warning("No placeholder named '%s' found for %s", activity_name, result_file.name) unmatched += 1 + from flowx.route_agentic import refresh_routing_outcomes + + refresh_routing_outcomes(report) destination = output_path or report_path destination.write_text(json.dumps(report, indent=2, default=str), encoding="utf-8") logger.info("Wrote merged report to %s (%d merged, %d unmatched)", destination, merged, unmatched) diff --git a/src/flowx/mcp/server.py b/src/flowx/mcp/server.py index d761aec5..6933c7c1 100644 --- a/src/flowx/mcp/server.py +++ b/src/flowx/mcp/server.py @@ -696,13 +696,17 @@ def flowx(command: str, parameters: dict[str, Any] | None = None) -> dict[str, A to read first. With a plan it validates + records metadata/conversion_plan.json AND edits the translation report so every routed-agentic group's tasks become placeholder gaps (deterministic groups untouched; convert's deterministic translation is never modified, and - with no agentic decision the report is byte-identical to today). + with no agentic decision the report is byte-identical to today). The edit stamps a routing + record onto the report (per component_id: members, decision, outcome; plus the plan and + baseline report hashes) that fills and apply_answers carry forward; re-routing under + different decisions is refused (re-run convert, then route). - "fill_agentic": output_dir(req), members(req: list of pipeline names or comma-separated string), one of pipelines(inline list of pipeline IR dicts) | pipelines_path — cross-pipeline COMBINE fill: replace a routed-agentic group's pipelines with the agent-authored pipeline(s) (typically AgenticComponentActivity nodes). `members` must exactly match a routed-agentic - component in the recorded, fingerprint-bound conversion_plan.json, and the merged report is - always validated structurally before writing (no bypass). Per-pipeline agentic fills use + component in the recorded, fingerprint-bound conversion_plan.json and the report's routing + record, and the merged report is always validated structurally before writing (no bypass); + the component's outcome becomes agentic-applied. Per-pipeline agentic fills use "merge_agentic" instead. - "inspect": report_path(req) — return the full translation-option schema (every option with a `show_when` condition) for the agent to walk locally. See "Collecting options" below. diff --git a/src/flowx/models/conversion_plan.py b/src/flowx/models/conversion_plan.py index a56e882c..7cd6fddb 100644 --- a/src/flowx/models/conversion_plan.py +++ b/src/flowx/models/conversion_plan.py @@ -9,7 +9,7 @@ The agent authors **only** :attr:`ComponentPlan.decision` (and an optional :attr:`ComponentPlan.rationale`). Everything else -- ``component_id``, ``members``, ``recommended``, -and both :class:`ComponentOptions` -- is recomputed by the library on record so the recorded facts +and both conversion options -- is recomputed by the library on record so the recorded facts can never drift from the inventory or be faked. The library also owns :attr:`ConversionPlan.schema_version` and the hashes that bind the plan to what it was decided on: :attr:`ConversionPlan.inventory_sha256` (the deterministic inventory), :attr:`ConversionPlan.source_graphs_sha256` (the saved @@ -27,7 +27,7 @@ import os from dataclasses import dataclass, field from pathlib import Path -from typing import Any, Literal +from typing import Any # The plan schema version stamped onto the recorded artifact. Bump on any backwards-incompatible # change to the recorded shape. Version 2 adds the source graph and source insights hashes and the @@ -47,92 +47,13 @@ ROUTE_MIXED = "mixed" ASSIGNMENT_ROUTES: tuple[str, ...] = (DECISION_DETERMINISTIC, DECISION_AGENTIC, ROUTE_MIXED) -# Recommended-pattern release states surfaced as a neutral disclosure label on the agentic option -# (:attr:`AgenticOption.release_disclosures`). ``"ga"`` and ``"unknown"`` are deliberately **silent** +# Recommended-pattern release states surfaced as a neutral disclosure label on the agentic option's +# ``release_disclosures``. ``"ga"`` and ``"unknown"`` are deliberately **silent** # -- they contribute no entry, and ``"unknown"`` is treated exactly like ``"ga"`` (we do not surface # or distinguish it). This is factual labelling, never a warning or an alarm. DISCLOSED_RELEASE_STATES: tuple[str, ...] = ("public_preview", "private_preview", "beta") -@dataclass(slots=True, kw_only=True) -class DeterministicOption: - """The deterministic (1:1 engine) conversion option for a component. - - Attributes: - capable: ``True`` when every activity in the component is engine-capable -- each either has a - ``"deterministic"`` strategy or is claimed by a detected motif -- so the whole component - can convert deterministically with no gap. - activity_counts: Count of activities by strategy bucket (``deterministic`` / ``agentic`` / - ``unsupported``) across the component's pipelines. - motifs: The motif ids detected in the component (the #64 multi-activity capability signal), - sorted and de-duplicated. - uncovered: One entry per activity that keeps the component from being fully deterministic -- - a dict of ``pipeline`` / ``activity`` / ``type`` / ``strategy``. Empty when ``capable``. - """ - - capable: bool - activity_counts: dict[str, int] = field(default_factory=dict) - motifs: list[str] = field(default_factory=list) - uncovered: list[dict[str, Any]] = field(default_factory=list) - - -@dataclass(slots=True, kw_only=True) -class AgenticPattern: - """One recommended Databricks pattern for the agentic option, drawn from a pipeline's insights. - - Attributes: - pipeline: The member pipeline the pattern was recommended for. - pattern: The named, publicly-documented Databricks capability (verbatim from the insight). - fit: One line on why it fits / what it replaces. - simplification_pattern: ``True`` when the pattern uses a distinctive capability that collapses - a whole legacy pattern (e.g. a multi-pipeline -> Lakeflow Connect re-architecture). - release_state: The pattern's verified Databricks GA/Preview release state, carried verbatim - from the insight (one of :data:`~flowx.models.insights.RELEASE_STATES`, or ``None`` when - the insight left it unstated). Drives the neutral release-state disclosure on - :class:`AgenticOption`. - release_state_source: The doc URL / citation grounding :attr:`release_state`, carried verbatim - from the insight; ``None`` when unstated. - """ - - pipeline: str - pattern: str - fit: str - simplification_pattern: bool - release_state: Literal["ga", "public_preview", "private_preview", "beta", "unknown"] | None = None - release_state_source: str | None = None - - -@dataclass(slots=True, kw_only=True) -class AgenticOption: - """The agentic conversion option for a component. - - Attributes: - recommended_patterns: The recommended patterns gathered from the member pipelines' insights, - each tagged with its pipeline. Empty when the inventory carries no insights. - has_simplification: ``True`` when any recommended pattern is a ``simplification_pattern`` -- - surfaced prominently so the user sees a re-architecture option, not a buried sub-key. - release_disclosures: A neutral, per-pattern **disclosure** of any non-silent ``release_state`` - -- one entry per recommended pattern whose state is in :data:`DISCLOSED_RELEASE_STATES`, - carrying the ``pipeline``, ``pattern``, ``release_state``, and a factual ``label`` - (``public_preview`` labelled "Public Preview (production-ready)"; ``private_preview`` / - ``beta`` stated as the plain labels "Private Preview" / "Beta"). ``"ga"`` and ``"unknown"`` - are **silent** -- they add nothing (``"unknown"`` is treated exactly like ``"ga"``). Empty - when nothing needs disclosing. Factual labelling, never a warning or an alarm. - """ - - recommended_patterns: list[AgenticPattern] = field(default_factory=list) - has_simplification: bool = False - release_disclosures: list[dict[str, Any]] = field(default_factory=list) - - -@dataclass(slots=True, kw_only=True) -class ComponentOptions: - """Both conversion options for a component, as first-class peers.""" - - deterministic: DeterministicOption - agentic: AgenticOption - - @dataclass(slots=True, kw_only=True) class NodeAssignment: """A routing choice for one node or a bounded subgraph inside a component (reserved for Phase 2). @@ -185,8 +106,8 @@ class ComponentPlan: engine-capable, else ``"agentic"``. decision: The user's authored per-component choice (may override :attr:`recommended`). options: Both conversion options with their evidence (library-computed), in the recorded - JSON shape that :class:`ComponentOptions` documents. Optional so the authored input -- - which carries only the decision -- can round-trip through this model. + JSON shape :mod:`flowx.routing` builds. Optional so the authored input -- which carries + only the decision -- can round-trip through this model. rationale: Optional author note on why this decision was chosen. assignments: Reserved per-node or per-subgraph routing (Phase 2); empty in Phase 1. """ @@ -251,10 +172,6 @@ class ConversionPlan: source_graphs_sha256: str | None = None source_insights_sha256: str | None = None - def agentic_members(self) -> list[list[str]]: - """The member lists of every component routed agentic.""" - return [list(component.members) for component in self.components if component.decision == DECISION_AGENTIC] - def to_dict(self) -> dict[str, Any]: """Serialise to the recorded ``conversion_plan.json`` shape.""" return { diff --git a/src/flowx/route_agentic.py b/src/flowx/route_agentic.py index 3742bad9..56084799 100644 --- a/src/flowx/route_agentic.py +++ b/src/flowx/route_agentic.py @@ -15,7 +15,7 @@ are left byte-identical, and when nothing is routed agentic the report and gaps are returned unchanged -- the non-breaking guarantee. * the agent authors the fill. **Per-pipeline** agentic reuses the existing name-matched - :func:`flowx.ir_serde.merge_agentic_results` (no new code here). **Cross-pipeline COMBINE** + :func:`flowx.ir_serde.merge_agentic_results`. **Cross-pipeline COMBINE** (N pipelines -> M, e.g. one Lakeflow Connect pipeline) is the one net-new capability: :func:`combine_group_fill` swaps the routed group's pipelines for the agent-authored pipeline(s), which carry :class:`~flowx.models.ir.AgenticComponentActivity` nodes (the escape hatch). @@ -24,6 +24,11 @@ :func:`flowx.validate.bundle_invariants.check_bundle_dir` over the output, so a fill can never introduce duplicate keys, dangling dependencies, a cycle, or a dangling pipeline/run_job reference. +Route stamps one routing record onto the report (:data:`ROUTING_RECORD_KEY`), keyed by the plan's +component ids, holding each component's members, decision and outcome plus the plan hash and the hash +of the report route started from. Every later rewrite of the report carries it forward, the fills +update the outcomes, and package refuses a report whose record no longer matches the recorded plan. + There is no LLM here: the tool edits the report and validates the authored fill; the fill itself is supplied by the agent/harness, following the same ask -> author -> continue pattern as the existing agentic flow. @@ -41,6 +46,7 @@ from pathlib import Path from typing import Any +from flowx.discovery_serde import canonical_sha256 from flowx.models.conversion_plan import DECISION_AGENTIC, ConversionPlan # The report + gaps live under the shared output dir's transient .work/ folder, beside the pipeline IR. @@ -56,18 +62,17 @@ # mis-tagged authored pipeline fails closed here (nothing written) instead of surviving to package. REQUIRED_COMBINE_SOURCE_TAG = "adf" -# Additive top-level marker stamped onto the report each time a combine is applied, recording the -# (component_id, inventory fingerprint) of every applied combine. Idempotency keys off this recorded -# state -- not off comparing authored pipeline names to member names -- so a re-run is detected no -# matter what the authored replacement is named or whether a name collides with a former member. It -# lives *in* the report, so a fresh convert (which rewrites the report) naturally clears it, and it is -# ignored by the package phase (not one of the recognized report shape keys) and by ir_serde (which -# reads per-pipeline IR, not the report wrapper). -COMBINE_PROVENANCE_KEY = "_combine_provenance" +# Top-level report key holding the routing record. It lives in the report, so a fresh convert (which +# rewrites the report) clears it; the package phase and ir_serde read only the pipelines beside it. +ROUTING_RECORD_KEY = "_routing_record" + +# What became of each component in the record: a deterministic component is left as convert wrote it; +# an agentic one stays "not viable" until a fill leaves none of its members with a routed placeholder. +OUTCOME_DETERMINISTIC = "deterministic" +OUTCOME_AGENTIC_APPLIED = "agentic-applied" +OUTCOME_AGENTIC_NOT_VIABLE = "agentic-not-viable" -# Top-level report key where route records the hash of the report it edited, so package can show -# what the routing edit started from after .work/ is pruned. Written once, on the first edit. -ROUTE_PROVENANCE_KEY = "_route_provenance" +REROUTED_UNDER_DIFFERENT_PLAN = "the report was routed under a different plan; re-run convert, then route" # Guidance stamped onto every placeholder the alteration produces. _PLACEHOLDER_COMMENT = ( @@ -94,6 +99,111 @@ def agentic_pipeline_names(plan: dict[str, Any]) -> set[str]: return names +# --------------------------------------------------------------------------- # +# The routing record stamped onto the report. +# --------------------------------------------------------------------------- # + + +def _planned_components(plan: dict[str, Any]) -> dict[str, dict[str, Any]]: + """Each plan component's sorted members and decision, keyed by component id.""" + return { + str(component.get("component_id")): { + "members": sorted(str(member) for member in component.get("members", [])), + "decision": component.get("decision"), + } + for component in plan.get("components", []) + if isinstance(component, dict) + } + + +def build_routing_record(plan: dict[str, Any], baseline_bytes: bytes) -> dict[str, Any]: + """The record route stamps onto the report when it first applies a plan with an agentic component.""" + components = { + component_id: { + **entry, + "outcome": OUTCOME_AGENTIC_NOT_VIABLE if entry["decision"] == DECISION_AGENTIC else OUTCOME_DETERMINISTIC, + } + for component_id, entry in _planned_components(plan).items() + } + return { + "conversion_plan_sha256": canonical_sha256(plan), + "baseline_report_sha256": hashlib.sha256(baseline_bytes).hexdigest(), + "components": components, + } + + +def routing_record(report: Any) -> dict[str, Any] | None: + """The routing record a report carries, or ``None`` when route has not edited it.""" + record = report.get(ROUTING_RECORD_KEY) if isinstance(report, dict) else None + return record if isinstance(record, dict) else None + + +def routing_record_mismatches(record: dict[str, Any], plan: dict[str, Any]) -> list[str]: + """Where a report's routing record disagrees with a plan's components, members or decisions. + + Returns one message per component that differs; an empty list means the record was routed under + these same decisions. + """ + recorded = record.get("components") + if not isinstance(recorded, dict): + recorded = {} + planned = _planned_components(plan) + mismatches: list[str] = [] + for component_id in sorted(set(recorded) | set(planned)): + entry = recorded.get(component_id) + decided = planned.get(component_id) + if not isinstance(entry, dict): + mismatches.append(f"component {component_id!r} is in the plan but not in the routing record") + elif decided is None: + mismatches.append(f"component {component_id!r} is in the routing record but not in the plan") + elif sorted(entry.get("members") or []) != decided["members"]: + mismatches.append(f"component {component_id!r} was routed with different members than the plan lists") + elif entry.get("decision") != decided["decision"]: + mismatches.append( + f"component {component_id!r} was routed {entry.get('decision')!r} but the plan decides " + f"{decided['decision']!r}" + ) + elif decided["decision"] != DECISION_AGENTIC and entry.get("outcome") != OUTCOME_DETERMINISTIC: + mismatches.append(f"a fill was applied to component {component_id!r}, which the plan routes deterministic") + return mismatches + + +def refresh_routing_outcomes(report: dict[str, Any]) -> None: + """Update each agentic component's outcome in the report's record after a fill, in place. + + A component is applied once none of its member pipelines left in the report still holds a routed + placeholder; a combine removes the members entirely. A report without a record is left alone. + """ + record = routing_record(report) + if record is None or not isinstance(record.get("components"), dict): + return + with_placeholders = { + pipeline.get("name") + for pipeline in _report_pipelines(report) + for task in pipeline.get("tasks", []) + if isinstance(task, dict) and task.get("type") == "PlaceholderActivity" + } + for entry in record["components"].values(): + if isinstance(entry, dict) and entry.get("decision") == DECISION_AGENTIC: + unfilled = any(member in with_placeholders for member in entry.get("members") or []) + entry["outcome"] = OUTCOME_AGENTIC_NOT_VIABLE if unfilled else OUTCOME_AGENTIC_APPLIED + + +def reroute_conflict(output_dir: Path, plan: dict[str, Any]) -> str | None: + """Explain why route must not apply ``plan`` to the report on disk, or ``None`` when it may. + + Route refuses once the report carries a record routed under different components or decisions: + the earlier edit cannot be undone in place, so the report has to be converted again first. + """ + report_path = Path(output_dir) / WORK_DIRNAME / REPORT_FILENAME + if not report_path.exists(): + return None + record = routing_record(json.loads(report_path.read_text(encoding="utf-8"))) + if record is not None and routing_record_mismatches(record, plan): + return REROUTED_UNDER_DIFFERENT_PLAN + return None + + # --------------------------------------------------------------------------- # # Interactive decision prompt (one route from the user's seat). # --------------------------------------------------------------------------- # @@ -272,15 +382,16 @@ def apply_plan_to_report(output_dir: Path, plan: dict[str, Any]) -> dict[str, An """Apply a routing decision to the report on disk: placeholder the routed-agentic groups. Reads ``/.work/translation_report.json`` (and ``gaps.json`` when present), rewrites - them for the plan's agentic components, and writes them back atomically. The first edit also - records the SHA-256 of the report it started from under ``_route_provenance`` so the package - audit can name it. When no component is routed agentic the files are left untouched, so the - non-breaking guarantee holds. + them for the plan's agentic components, stamps the routing record and writes them back atomically. + When no component is routed agentic and the report carries no record the files are left + untouched, so the non-breaking guarantee holds. A report already routed under the same decisions + is not edited again; only the record's plan hash is brought up to date. Returns a summary dict with the altered pipeline names and the resulting gap count. Raises: FileNotFoundError: when the translation report is missing (run convert first). + ValueError: when the report was routed under different components or decisions. """ work = Path(output_dir) / WORK_DIRNAME report_path = work / REPORT_FILENAME @@ -289,18 +400,28 @@ def apply_plan_to_report(output_dir: Path, plan: dict[str, Any]) -> dict[str, An gaps_path = work / GAPS_FILENAME agentic = agentic_pipeline_names(plan) - if not agentic: - return {"agentic_pipelines": [], "gaps": 0, "altered": False} - baseline_bytes = report_path.read_bytes() report = json.loads(baseline_bytes) + record = routing_record(report) + if record is None and not agentic: + return {"agentic_pipelines": [], "gaps": 0, "altered": False} + gaps = json.loads(gaps_path.read_text(encoding="utf-8")) if gaps_path.exists() else [] if not isinstance(gaps, list): gaps = [] + if record is not None: + if routing_record_mismatches(record, plan): + raise ValueError(REROUTED_UNDER_DIFFERENT_PLAN) + if record.get("conversion_plan_sha256") != canonical_sha256(plan): + record["conversion_plan_sha256"] = canonical_sha256(plan) + _write_json_atomic(report_path, report) + return {"agentic_pipelines": sorted(agentic), "gaps": len(gaps), "altered": False} + new_report, new_gaps = alter_report(report, gaps, agentic) - if isinstance(new_report, dict) and ROUTE_PROVENANCE_KEY not in new_report: - new_report[ROUTE_PROVENANCE_KEY] = {"baseline_report_sha256": hashlib.sha256(baseline_bytes).hexdigest()} + if not isinstance(new_report.get("pipelines"), list): + new_report = {"pipelines": [new_report]} + new_report[ROUTING_RECORD_KEY] = build_routing_record(plan, baseline_bytes) _write_json_atomic(report_path, new_report) _write_json_atomic(gaps_path, new_gaps) return {"agentic_pipelines": sorted(agentic), "gaps": len(new_gaps), "altered": True} @@ -353,19 +474,20 @@ def combine_group_fill( return {"pipelines": kept} -def _resolve_agentic_component(output_dir: Path, members: set[str]) -> tuple[str | None, str | None, str | None]: +def _resolve_agentic_component( + output_dir: Path, members: set[str] +) -> tuple[str | None, dict[str, Any] | None, str | None]: """Bind ``members`` to a routed-**agentic** component in the recorded, fingerprint-bound plan. Reads ``metadata/conversion_plan.json`` and ``metadata/inventory.json`` and returns - ``(component_id, inventory_fingerprint, error)``. The error is set (and the other two are ``None``) - when: the plan or inventory is missing; the plan's ``inventory_sha256`` no longer matches the - current inventory (a stale plan); ``members`` do not exactly equal one component's members (a - partial, superset, or mistyped group); or the exactly-matching component is routed deterministic - rather than agentic. Requiring an exact match to a routed-agentic component stops a caller from - swapping deterministic pipelines or a partial group. The fingerprint is returned so the combine can - record its provenance keyed on the exact inventory it was applied against. + ``(component_id, plan, error)``, where ``plan`` is the recorded plan document. The error is set + (and the other two are ``None``) when: the plan or inventory is missing; the plan's + ``inventory_sha256`` no longer matches the current inventory (a stale plan); ``members`` do not + exactly equal one component's members (a partial, superset, or mistyped group); or the + exactly-matching component is routed deterministic rather than agentic. Requiring an exact match + to a routed-agentic component stops a caller from swapping deterministic pipelines or a partial + group. The plan is returned so the combine can check the report's routing record against it. """ - from flowx.discovery_insights import inventory_fingerprint from flowx.routing import plan_binding_violations metadata = Path(output_dir) / METADATA_DIRNAME @@ -380,7 +502,6 @@ def _resolve_agentic_component(output_dir: Path, members: set[str]) -> tuple[str return None, None, "No metadata/inventory.json; run the discover phase first." inventory = json.loads(inventory_path.read_text(encoding="utf-8")) - current_fingerprint = inventory_fingerprint(inventory) stale = plan_binding_violations(recorded, inventory) if stale: return None, None, f"conversion_plan.json is stale: {stale[0]}; re-run `route` before filling." @@ -393,7 +514,7 @@ def _resolve_agentic_component(output_dir: Path, members: set[str]) -> tuple[str if component_members != members: continue if component.get("decision") == DECISION_AGENTIC: - return str(component.get("component_id")), current_fingerprint, None + return str(component.get("component_id")), plan, None return ( None, None, @@ -433,25 +554,6 @@ def _authored_source_tag_violations(authored_pipelines: list[dict[str, Any]]) -> return violations -def _combine_already_applied(report: dict[str, Any], component_id: str, fingerprint: str) -> bool: - """Report whether this component's combine (at this inventory fingerprint) is already recorded. - - Reads the additive ``_combine_provenance`` marker stamped onto the report by a prior combine. - Detection is keyed purely on ``(component_id, fingerprint)`` -- never on comparing authored - pipeline names to member names -- so a re-run is recognised as already-combined no matter what the - authored replacement is named, and even when an authored name collides with a former member. - """ - provenance = report.get(COMBINE_PROVENANCE_KEY) if isinstance(report, dict) else None - if not isinstance(provenance, list): - return False - return any( - isinstance(entry, dict) - and entry.get("component_id") == component_id - and entry.get("inventory_sha256") == fingerprint - for entry in provenance - ) - - def apply_combine_fill( output_dir: Path, group_members: Iterable[str], @@ -462,19 +564,18 @@ def apply_combine_fill( The group's membership is **bound to the recorded plan**: ``group_members`` must exactly match a routed-agentic component in ``metadata/conversion_plan.json`` (whose fingerprint must still match the current inventory), so a caller cannot swap deterministic pipelines or a partial/typoed group. + The report must carry a routing record that matches that plan, so route has to have applied it. Every authored pipeline must carry ``tags.source == 'adf'`` (routing/agentic is ADF-only); a mis-tagged pipeline fails the combine closed here rather than surviving to the package preflight. The merged report is then **always** validated with the structural bundle invariants (a real ``prepare -> write_bundle`` pass over :func:`validate_report_structurally`) -- there is no bypass -- and written back only when it passes, so a dangling reference or duplicate key never lands on disk. - The combine is **idempotent**, keyed on recorded report state rather than pipeline names. Each - successful combine stamps a ``_combine_provenance`` entry -- ``(component_id, inventory - fingerprint)`` -- onto the report. A re-run detects that stamp and no-ops (``already_combined`` - true) instead of collapsing/appending again, so it never duplicates the authored pipeline(s) even - when the authored replacement is renamed, and it still recognises the already-combined state when - an authored pipeline's name collides with a former member. A fresh ``convert`` rewrites the report - without the marker, so the combine will re-apply after a genuine re-convert. + The combine is **idempotent**, keyed on the routing record rather than pipeline names: once the + component's outcome is ``agentic-applied`` a re-run no-ops (``already_combined`` true) instead of + collapsing/appending again, so it never duplicates the authored pipeline(s) even when the authored + replacement is renamed or an authored name collides with a former member. A fresh ``convert`` + rewrites the report without the record, so the combine applies again after a re-convert and route. Returns ``{"ok", "violations", "error", "component_id", "pipelines", "already_combined"}``. ``ok`` is ``False`` (and nothing written) on a plan/membership error (``error`` set), a missing source tag, @@ -484,10 +585,10 @@ def apply_combine_fill( FileNotFoundError: when the translation report is missing (run convert first). """ members = {str(member) for member in group_members} - component_id, fingerprint, error = _resolve_agentic_component(output_dir, members) + component_id, plan, error = _resolve_agentic_component(output_dir, members) if error is not None: return {"ok": False, "error": error, "violations": [], "pipelines": 0} - assert component_id is not None and fingerprint is not None # guaranteed when error is None + assert component_id is not None and plan is not None # guaranteed when error is None tag_violations = _authored_source_tag_violations(authored_pipelines) if tag_violations: @@ -499,12 +600,14 @@ def apply_combine_fill( raise FileNotFoundError(f"No {REPORT_FILENAME} under {work}; run the convert phase first.") report = json.loads(report_path.read_text(encoding="utf-8")) + record = routing_record(report) + if record is None: + error = f"{REPORT_FILENAME} carries no routing record; run `route` to apply the plan before filling." + return {"ok": False, "error": error, "violations": [], "pipelines": 0} + if routing_record_mismatches(record, plan): + return {"ok": False, "error": REROUTED_UNDER_DIFFERENT_PLAN, "violations": [], "pipelines": 0} - # Idempotency: a prior combine stamps _combine_provenance onto the report for this - # (component_id, fingerprint). Detecting that recorded state -- not the authored/member name sets - # -- means a re-run no-ops regardless of how the authored replacement is named or whether a name - # collides with a former member, so it can never append a duplicate authored pipeline. - if _combine_already_applied(report, component_id, fingerprint): + if record["components"][component_id].get("outcome") == OUTCOME_AGENTIC_APPLIED: return { "ok": True, "error": None, @@ -515,14 +618,8 @@ def apply_combine_fill( } merged = combine_group_fill(report, members, authored_pipelines) - # Carry any prior provenance forward (combine_group_fill returns only ``pipelines``) and record - # this combine so a later re-run detects it. - prior_provenance = report.get(COMBINE_PROVENANCE_KEY) if isinstance(report, dict) else None - provenance = ( - [entry for entry in prior_provenance if isinstance(entry, dict)] if isinstance(prior_provenance, list) else [] - ) - provenance.append({"component_id": component_id, "inventory_sha256": fingerprint, "members": sorted(members)}) - merged[COMBINE_PROVENANCE_KEY] = provenance + merged[ROUTING_RECORD_KEY] = copy.deepcopy(record) + refresh_routing_outcomes(merged) result = validate_report_structurally(merged) if not result.ok: diff --git a/src/flowx/routing.py b/src/flowx/routing.py index abb97c8d..6cbae5d0 100644 --- a/src/flowx/routing.py +++ b/src/flowx/routing.py @@ -571,11 +571,6 @@ def build_plan(inventory: dict[str, Any], raw: dict[str, Any]) -> ConversionPlan ) -def build_plan_document(inventory: dict[str, Any], raw: dict[str, Any]) -> dict[str, Any]: - """The recorded JSON form of :func:`build_plan`.""" - return build_plan(inventory, raw).to_dict() - - def plan_binding_violations(plan: ConversionPlan, inventory: dict[str, Any]) -> list[str]: """Check a recorded plan still matches the inventory, source graphs and insights it was decided on. diff --git a/tests/unit/test_package_routing_preflight.py b/tests/unit/test_package_routing_preflight.py index 80411c68..e7146e05 100644 --- a/tests/unit/test_package_routing_preflight.py +++ b/tests/unit/test_package_routing_preflight.py @@ -1,9 +1,10 @@ """Package refuses to ship a translation report whose routing plan has gone stale. A recorded ``metadata/conversion_plan.json`` is bound to the inventory fingerprint it was recorded -against. When discover runs again, or a combine fill was applied under a different plan, the report -no longer reflects the user's decision, so package fails closed before writing any bundle file. -When the plan still matches, the route audit records hashes of what was packaged. +against, and route stamps a routing record onto the report it edits. When discover runs again, or +the report's routing record no longer matches the plan, the report no longer reflects the user's +decision, so package fails closed before writing any bundle file. When the plan still matches, the +route audit records hashes and outcomes of what was packaged. """ from __future__ import annotations @@ -16,12 +17,13 @@ import pytest from flowx import routing +from flowx.adapter.__main__ import main as adapter_main from flowx.bundler.dab_writer import main as package_main from flowx.discovery_inventory import STRATEGY_PROPERTY, build_source_inventory from flowx.ir_serde import pipeline_to_dict from flowx.models.discovery import CONCEPT_NOTEBOOK, SourceGraph, SourceNode from flowx.models.ir import NotebookActivity, Pipeline -from flowx.route_agentic import COMBINE_PROVENANCE_KEY, REPORT_FILENAME, WORK_DIRNAME, apply_plan_to_report +from flowx.route_agentic import REPORT_FILENAME, WORK_DIRNAME, apply_plan_to_report, routing_record def _inventory(strategy: str = "deterministic") -> dict[str, Any]: @@ -76,19 +78,121 @@ def test_package_refuses_a_plan_recorded_against_a_different_inventory( assert not (tmp_path / "metadata" / "route_audit.json").exists() -def test_package_refuses_a_combine_that_does_not_match_the_plan( +def test_package_refuses_a_routing_record_that_does_not_match_the_plan( tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: - report_path = _setup(tmp_path, _inventory()) + report_path = _setup(tmp_path, _inventory("agentic")) + _record(tmp_path, "agentic") + plan = json.loads((tmp_path / "metadata" / "conversion_plan.json").read_text(encoding="utf-8")) + apply_plan_to_report(tmp_path, plan) + # The decision is recorded again as deterministic without re-converting, so the report still + # carries the agentic edit. _record(tmp_path, "deterministic") - report = json.loads(report_path.read_text(encoding="utf-8")) - current = json.loads((tmp_path / "metadata" / "conversion_plan.json").read_text())["inventory_sha256"] - report[COMBINE_PROVENANCE_KEY] = [{"component_id": "component-1", "inventory_sha256": current, "members": ["solo"]}] - report_path.write_text(json.dumps(report), encoding="utf-8") assert _package(tmp_path) == 1 - assert "does not match an agentic component in the recorded plan" in capsys.readouterr().err + error = capsys.readouterr().err + assert "was routed under a different conversion plan" in error + assert "was routed 'agentic' but the plan decides 'deterministic'" in error assert not (tmp_path / "databricks.yml").exists() + assert routing_record(json.loads(report_path.read_text(encoding="utf-8"))) is not None + + +def test_package_refuses_an_agentic_plan_when_the_report_has_no_routing_record( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + _setup(tmp_path, _inventory("agentic")) + _record(tmp_path, "agentic") # recorded, but never applied to the report + + assert _package(tmp_path) == 1 + assert "carries no routing record; re-run convert, then route" in capsys.readouterr().err + assert not (tmp_path / "databricks.yml").exists() + + +def test_package_with_a_deterministic_plan_and_no_record_passes(tmp_path: Path) -> None: + report_path = _setup(tmp_path, _inventory()) + before = report_path.read_bytes() + _record(tmp_path, "deterministic") + plan = json.loads((tmp_path / "metadata" / "conversion_plan.json").read_text(encoding="utf-8")) + apply_plan_to_report(tmp_path, plan) + assert report_path.read_bytes() == before + + assert _package(tmp_path) == 0 + audit = json.loads((tmp_path / "metadata" / "route_audit.json").read_text(encoding="utf-8")) + assert audit["components"] == [ + {"component_id": "component-1", "members": ["solo"], "decision": "deterministic", "outcome": "deterministic"} + ] + assert audit["baseline_report_sha256"] is None + + +def _authored_pipeline() -> dict[str, Any]: + pipeline = Pipeline( + name="solo_agentic", + tasks=[NotebookActivity(name="load", task_key="load", notebook_path="/Workspace/Shared/agentic_load")], + tags={"source": "adf"}, + ) + return pipeline_to_dict(pipeline) + + +def test_route_combine_modify_then_package_keeps_the_routing_record(tmp_path: Path) -> None: + report_path = _setup(tmp_path, _inventory("agentic")) + baseline_sha256 = hashlib.sha256(report_path.read_bytes()).hexdigest() + plan_path = tmp_path / "plan.json" + plan_path.write_text( + json.dumps({"components": [{"component_id": "component-1", "members": ["solo"], "decision": "agentic"}]}), + encoding="utf-8", + ) + authored_path = tmp_path / "authored.json" + authored_path.write_text(json.dumps([_authored_pipeline()]), encoding="utf-8") + + assert adapter_main(["route", "--output-dir", str(tmp_path), "--plan-path", str(plan_path)]) == 0 + assert ( + adapter_main( + [ + "fill-agentic", + "combine", + "--output-dir", + str(tmp_path), + "--members", + "solo", + "--pipelines-path", + str(authored_path), + ] + ) + == 0 + ) + assert adapter_main(["modify", str(report_path), "--output-dir", str(tmp_path)]) == 0 + stamped_path = tmp_path / WORK_DIRNAME / "translation_report.stamped.json" + assert routing_record(json.loads(stamped_path.read_text(encoding="utf-8"))) is not None + + assert _package(tmp_path) == 0 + audit = json.loads((tmp_path / "metadata" / "route_audit.json").read_text(encoding="utf-8")) + assert audit["components"] == [ + {"component_id": "component-1", "members": ["solo"], "decision": "agentic", "outcome": "agentic-applied"} + ] + assert audit["baseline_report_sha256"] == baseline_sha256 + assert audit["translation_report_sha256"] == hashlib.sha256(stamped_path.read_bytes()).hexdigest() + + +def test_route_agentic_then_deterministic_is_refused_and_writes_nothing( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + report_path = _setup(tmp_path, _inventory("agentic")) + plan_path = tmp_path / "plan.json" + + def _route(decision: str) -> int: + plan = {"components": [{"component_id": "component-1", "members": ["solo"], "decision": decision}]} + plan_path.write_text(json.dumps(plan), encoding="utf-8") + return adapter_main(["route", "--output-dir", str(tmp_path), "--plan-path", str(plan_path)]) + + assert _route("agentic") == 0 + recorded_plan = (tmp_path / "metadata" / "conversion_plan.json").read_bytes() + routed_report = report_path.read_bytes() + capsys.readouterr() + + assert _route("deterministic") == 1 + assert "re-run convert, then route" in capsys.readouterr().err + assert (tmp_path / "metadata" / "conversion_plan.json").read_bytes() == recorded_plan + assert report_path.read_bytes() == routed_report def test_package_with_a_current_plan_writes_an_audit_with_hashes(tmp_path: Path) -> None: diff --git a/tests/unit/test_route_agentic_flow.py b/tests/unit/test_route_agentic_flow.py index 489cace5..a37b5edc 100644 --- a/tests/unit/test_route_agentic_flow.py +++ b/tests/unit/test_route_agentic_flow.py @@ -26,20 +26,26 @@ from pathlib import Path from typing import Any +import pytest + from flowx.discovery_inventory import STRATEGY_PROPERTY, build_source_inventory from flowx.ir_serde import merge_agentic_results +from flowx.models.conversion_plan import ConversionPlan from flowx.models.discovery import CONCEPT_NOTEBOOK, SourceGraph, SourceNode from flowx.models.ir import ControlEdge, Lineage from flowx.route_agentic import ( GAPS_FILENAME, REPORT_FILENAME, + REROUTED_UNDER_DIFFERENT_PLAN, WORK_DIRNAME, agentic_pipeline_names, alter_report, apply_combine_fill, + apply_plan, apply_plan_to_report, combine_group_fill, prompt_for_decisions, + routing_record, validate_report_structurally, ) from flowx.routing import record_plan @@ -165,7 +171,7 @@ def _routed_inventory() -> dict[str, Any]: def _setup_routed_agentic(output_dir: Path, *, decision: str = "agentic") -> None: - """Write inventory + report + a recorded conversion_plan.json routing {child, parent} per ``decision``.""" + """Write inventory + report, then record and apply a plan routing {child, parent} per ``decision``.""" _write_work(output_dir, _report_two_pipelines()) metadata = output_dir / "metadata" metadata.mkdir(parents=True, exist_ok=True) @@ -173,6 +179,15 @@ def _setup_routed_agentic(output_dir: Path, *, decision: str = "agentic") -> Non plan = {"components": [{"component_id": "component-1", "members": ["child", "parent"], "decision": decision}]} result = record_plan(output_dir, plan=plan) assert result["ok"], result + recorded = ConversionPlan.load(output_dir) + assert recorded is not None + apply_plan(output_dir, recorded) + + +def _record_outcomes(report_path: Path) -> dict[str, str]: + record = routing_record(json.loads(report_path.read_text(encoding="utf-8"))) + assert record is not None + return {component_id: entry["outcome"] for component_id, entry in record["components"].items()} # --------------------------------------------------------------------------- # @@ -398,6 +413,7 @@ def test_per_pipeline_fill_reuses_merge_agentic_results(tmp_path: Path) -> None: ), encoding="utf-8", ) + assert _record_outcomes(report_path) == {"component-1": "agentic-not-viable", "component-2": "deterministic"} merged, unmatched = merge_agentic_results(report_path, results_dir) assert (merged, unmatched) == (1, 0) @@ -405,6 +421,67 @@ def test_per_pipeline_fill_reuses_merge_agentic_results(tmp_path: Path) -> None: parent = next(p for p in report["pipelines"] if p["name"] == "parent") assert parent["tasks"][0]["type"] == "NotebookActivity" assert validate_report_structurally(report).ok + # The merge left no routed placeholder in component-1, so its outcome is now applied. + assert _record_outcomes(report_path) == {"component-1": "agentic-applied", "component-2": "deterministic"} + + +# --------------------------------------------------------------------------- # +# The routing record: re-routing under different decisions is refused. +# --------------------------------------------------------------------------- # + + +def test_route_writes_no_record_when_nothing_is_routed_agentic(tmp_path: Path) -> None: + _write_work(tmp_path, _report_two_pipelines(), gaps=[]) + report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME + before = report_path.read_bytes() + + apply_plan_to_report(tmp_path, _plan(parent="deterministic", child="deterministic")) + + assert report_path.read_bytes() == before + + +def test_reroute_to_deterministic_is_refused_and_leaves_the_report_untouched(tmp_path: Path) -> None: + _write_work(tmp_path, _report_two_pipelines(), gaps=[]) + apply_plan_to_report(tmp_path, _plan(parent="agentic", child="deterministic")) + report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME + routed = report_path.read_bytes() + + with pytest.raises(ValueError, match=REROUTED_UNDER_DIFFERENT_PLAN): + apply_plan_to_report(tmp_path, _plan(parent="deterministic", child="deterministic")) + with pytest.raises(ValueError, match=REROUTED_UNDER_DIFFERENT_PLAN): + apply_plan_to_report(tmp_path, _plan(parent="deterministic", child="agentic")) + assert report_path.read_bytes() == routed + + +def test_reroute_under_the_same_decisions_keeps_fills_and_updates_the_plan_hash(tmp_path: Path) -> None: + _setup_routed_agentic(tmp_path, decision="agentic") + assert apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()])["ok"] is True + report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME + first_record = routing_record(json.loads(report_path.read_text(encoding="utf-8"))) + assert first_record is not None + + rationale_plan = { + "components": [ + { + "component_id": "component-1", + "members": ["child", "parent"], + "decision": "agentic", + "rationale": "same decision, recorded again", + } + ] + } + assert record_plan(tmp_path, plan=rationale_plan)["ok"] is True + recorded = ConversionPlan.load(tmp_path) + assert recorded is not None + assert apply_plan(tmp_path, recorded)["altered"] is False + + report = json.loads(report_path.read_text(encoding="utf-8")) + assert [pipeline["name"] for pipeline in report["pipelines"]] == ["orders_lfc"] + second_record = routing_record(report) + assert second_record is not None + assert second_record["conversion_plan_sha256"] != first_record["conversion_plan_sha256"] + assert second_record["baseline_report_sha256"] == first_record["baseline_report_sha256"] + assert second_record["components"] == first_record["components"] # --------------------------------------------------------------------------- # @@ -521,8 +598,8 @@ def test_combine_idempotent_with_a_differently_named_authored_pipeline(tmp_path: """FIX 3 (a): a second combine whose authored replacement is renamed must NOT duplicate. Name-set matching would fail here (the new name is absent from the report, the old members are - already gone), fall through, and append a second authored pipeline. Provenance keyed on - (component_id, fingerprint) catches the re-run regardless of the authored name. + already gone), fall through, and append a second authored pipeline. The routing record's outcome + catches the re-run regardless of the authored name. """ _setup_routed_agentic(tmp_path, decision="agentic") @@ -543,8 +620,8 @@ def test_combine_idempotent_when_authored_name_collides_with_a_former_member(tmp """FIX 3 (b): an authored name colliding with a former member is still detected as already-combined. After the first combine the report holds a pipeline named 'parent' (a former member). Name-based - detection would see 'parent' present and conclude the combine had not happened; provenance keeps - the detection correct and independent of names. + detection would see 'parent' present and conclude the combine had not happened; the routing record + keeps the detection correct and independent of names. """ _setup_routed_agentic(tmp_path, decision="agentic") @@ -609,6 +686,26 @@ def test_combine_rejects_a_stale_fingerprint_plan(tmp_path: Path) -> None: assert "stale" in result["error"] +def test_combine_marks_the_component_applied(tmp_path: Path) -> None: + _setup_routed_agentic(tmp_path, decision="agentic") + report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME + assert _record_outcomes(report_path) == {"component-1": "agentic-not-viable"} + + assert apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()])["ok"] is True + + assert _record_outcomes(report_path) == {"component-1": "agentic-applied"} + + +def test_combine_requires_route_to_have_applied_the_plan(tmp_path: Path) -> None: + _setup_routed_agentic(tmp_path, decision="agentic") + _write_work(tmp_path, _report_two_pipelines()) # convert ran again, so the report has no record + + result = apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()]) + + assert result["ok"] is False + assert "routing record" in result["error"] + + def test_combine_requires_a_recorded_plan(tmp_path: Path) -> None: _write_work(tmp_path, _report_two_pipelines()) # report only; no plan recorded result = apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()]) diff --git a/tests/unit/test_route_audit.py b/tests/unit/test_route_audit.py index b478ef48..0316a693 100644 --- a/tests/unit/test_route_audit.py +++ b/tests/unit/test_route_audit.py @@ -44,6 +44,8 @@ def _write_gaps(output_dir: Path) -> None: "raw_definition": {"name": "Load"}, "pipeline": "child", }, + {"activity_name": "Fail", "activity_type": "Fail", "raw_definition": {"name": "Fail"}}, + {"activity_name": "Hook", "activity_type": "WebHook", "raw_definition": {}, "pipeline": "solo"}, ] (work / "gaps.json").write_text(json.dumps(gaps, indent=2), encoding="utf-8") @@ -64,7 +66,7 @@ def test_route_audit_summarises_components_decisions_and_gaps(tmp_path: Path) -> assert decisions == {"component-1": "agentic", "component-2": "deterministic"} # Only the agentic component's members are surfaced as agentic pipelines. assert audit["agentic_pipelines"] == ["child", "parent"] - # The gaps introduced survive as a compact summary (no verbose raw_definition). + # Only gaps in agentic-routed pipelines count as introduced by routing, as a compact summary. assert audit["gaps_count"] == 2 assert {gap["pipeline"] for gap in audit["gaps_introduced"]} == {"parent", "child"} assert all("raw_definition" not in gap for gap in audit["gaps_introduced"]) From 15d69246231c4bc460a77b3cdfd6db09745acdbe Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Wed, 7 Oct 2026 10:39:34 +0100 Subject: [PATCH 019/122] no-mistakes(review): Enforce one fill per component; refresh stamped routing record --- skills/flowx-route/SKILL.md | 11 ++- src/flowx/ir_serde.py | 20 +++-- src/flowx/route_agentic.py | 81 +++++++++++++++++--- src/flowx/sources/adf/translate.py | 9 ++- tests/unit/test_package_routing_preflight.py | 25 ++++++ tests/unit/test_route_agentic_flow.py | 48 ++++++++++++ 6 files changed, 174 insertions(+), 20 deletions(-) diff --git a/skills/flowx-route/SKILL.md b/skills/flowx-route/SKILL.md index ade1afe3..5dc31f05 100644 --- a/skills/flowx-route/SKILL.md +++ b/skills/flowx-route/SKILL.md @@ -105,8 +105,8 @@ entry per `component_id` with its `members`, `decision` and `outcome` — `deter `agentic-not-viable` (nothing filled yet) or `agentic-applied` (a fill left none of its members with a routed placeholder). Every later rewrite (the fills and `modify`) carries the record forward and only the fills update outcomes. Re-running route under the **same** decisions only refreshes the record's -plan hash; under different components or decisions it refuses and writes nothing — re-run convert, -then route. +plan hash (in the report and in the stamped copy `modify` wrote); under different components or +decisions it refuses and writes nothing — re-run convert, then route. There are three ways to supply the decision: @@ -238,8 +238,11 @@ MCP: `flowx(command="fill_agentic", parameters={"output_dir": ..., "members": [. `source_graphs_sha256` and `source_insights_sha256` must still match the current inventory. A partial group, a superset, a typo, or a deterministic component is refused — you can't swap pipelines the plan didn't route agentic. The report must carry a routing record that matches that plan (route has applied it). -- Re-running combine once the component's outcome is `agentic-applied` is a no-op - (`already_combined: true`), so the authored pipelines are never appended twice. +- One kind of fill per component: combine on a component a per-pipeline merge already filled, or + a merge aimed at a component a combine already filled, is refused with "component already filled; + re-run convert and route to start again" and writes nothing. Re-running combine on a component it + already combined is a no-op (`already_combined: true`), so the authored pipelines are never + appended twice. - `--pipelines-path` is a JSON **list** of pipeline IR dicts (the authored replacements), typically carrying `AgenticComponentActivity` nodes (see below). - Each authored pipeline **must** carry the source tag `"tags": {"source": "adf"}` (routing/agentic diff --git a/src/flowx/ir_serde.py b/src/flowx/ir_serde.py index 9412137e..7116ed88 100644 --- a/src/flowx/ir_serde.py +++ b/src/flowx/ir_serde.py @@ -628,7 +628,8 @@ def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Pat wrote to the workspace; the prepare phase then references it directly. When the report carries a routing record, each agentic component's outcome is updated to show - whether the merge left any of its routed placeholders unfilled. + whether the merge left any of its routed placeholders unfilled, and a result aimed at a pipeline + of a component a combine already filled is refused before anything is written. Args: report_path: ``translation_report.json`` produced by the translate phase. @@ -638,14 +639,25 @@ def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Pat Returns: ``(merged, unmatched)`` counts. + + Raises: + ValueError: A result targets a pipeline of a component a combine already filled. """ + from flowx.route_agentic import COMPONENT_ALREADY_FILLED, combined_pipelines, refresh_routing_outcomes + report = json.loads(report_path.read_text(encoding="utf-8")) pipelines = report["pipelines"] if isinstance(report, dict) and "pipelines" in report else [report] + results = [ + (result_file, json.loads(result_file.read_text(encoding="utf-8"))) + for result_file in sorted(results_dir.glob("*.json")) + ] + combined = combined_pipelines(report) if isinstance(report, dict) else set() + if any(data.get("pipeline") in combined for _, data in results): + raise ValueError(COMPONENT_ALREADY_FILLED) merged = 0 unmatched = 0 - for result_file in sorted(results_dir.glob("*.json")): - data = json.loads(result_file.read_text(encoding="utf-8")) + for result_file, data in results: activity_name = data.get("activity_name") or data.get("activity") task = data.get("task") or data.get("ir") if not activity_name or not isinstance(task, dict): @@ -661,8 +673,6 @@ def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Pat logger.warning("No placeholder named '%s' found for %s", activity_name, result_file.name) unmatched += 1 - from flowx.route_agentic import refresh_routing_outcomes - refresh_routing_outcomes(report) destination = output_path or report_path destination.write_text(json.dumps(report, indent=2, default=str), encoding="utf-8") diff --git a/src/flowx/route_agentic.py b/src/flowx/route_agentic.py index 56084799..59dfbf9e 100644 --- a/src/flowx/route_agentic.py +++ b/src/flowx/route_agentic.py @@ -52,6 +52,8 @@ # The report + gaps live under the shared output dir's transient .work/ folder, beside the pipeline IR. WORK_DIRNAME = ".work" REPORT_FILENAME = "translation_report.json" +# The configuration-stamped copy modify writes beside the report; package reads it when present. +STAMPED_REPORT_FILENAME = "translation_report.stamped.json" GAPS_FILENAME = "gaps.json" # The recorded plan + inventory the combine fill binds against live under metadata/. METADATA_DIRNAME = "metadata" @@ -74,6 +76,12 @@ REROUTED_UNDER_DIFFERENT_PLAN = "the report was routed under a different plan; re-run convert, then route" +# Each routed-agentic component takes exactly one kind of fill: a cross-pipeline combine or +# per-pipeline merges, never both. +COMPONENT_ALREADY_FILLED = "component already filled; re-run convert and route to start again" +FILL_COMBINE = "combine" +FILL_MERGE = "merge" + # Guidance stamped onto every placeholder the alteration produces. _PLACEHOLDER_COMMENT = ( "Routed agentic by the conversion plan; author a replacement task (per-pipeline fill) or replace " @@ -189,6 +197,38 @@ def refresh_routing_outcomes(report: dict[str, Any]) -> None: entry["outcome"] = OUTCOME_AGENTIC_NOT_VIABLE if unfilled else OUTCOME_AGENTIC_APPLIED +def component_fill(report: dict[str, Any], members: Iterable[str]) -> str | None: + """Which kind of fill has touched a routed-agentic component, read from the report itself. + + Only a combine removes member pipelines from the report, and route turns every top-level task of + a member into a placeholder, so a member task that is no longer a placeholder came from a + per-pipeline merge. Returns :data:`FILL_COMBINE`, :data:`FILL_MERGE`, or ``None`` when neither + has run. + """ + by_name = {pipeline.get("name"): pipeline for pipeline in _report_pipelines(report)} + members = list(members) + if any(member not in by_name for member in members): + return FILL_COMBINE + for member in members: + for task in by_name[member].get("tasks", []): + if isinstance(task, dict) and task.get("type") != "PlaceholderActivity": + return FILL_MERGE + return None + + +def combined_pipelines(report: dict[str, Any]) -> set[str]: + """The member pipelines of every routed-agentic component a combine has already filled.""" + record = routing_record(report) + components = record.get("components") if record is not None else None + names: set[str] = set() + for entry in components.values() if isinstance(components, dict) else []: + if isinstance(entry, dict) and entry.get("decision") == DECISION_AGENTIC: + members = [str(member) for member in entry.get("members") or []] + if component_fill(report, members) == FILL_COMBINE: + names.update(members) + return names + + def reroute_conflict(output_dir: Path, plan: dict[str, Any]) -> str | None: """Explain why route must not apply ``plan`` to the report on disk, or ``None`` when it may. @@ -378,6 +418,22 @@ def _write_json_atomic(path: Path, document: Any) -> None: os.replace(temporary, path) +def _refresh_record_plan_hash(report_path: Path, plan: dict[str, Any]) -> None: + """Bring a report copy's recorded plan hash up to date when it was routed under these decisions. + + A copy without a record, or routed under other decisions, is left for package to refuse. + """ + if not report_path.exists(): + return + report = json.loads(report_path.read_text(encoding="utf-8")) + record = routing_record(report) + plan_sha256 = canonical_sha256(plan) + if record is None or routing_record_mismatches(record, plan) or record.get("conversion_plan_sha256") == plan_sha256: + return + record["conversion_plan_sha256"] = plan_sha256 + _write_json_atomic(report_path, report) + + def apply_plan_to_report(output_dir: Path, plan: dict[str, Any]) -> dict[str, Any]: """Apply a routing decision to the report on disk: placeholder the routed-agentic groups. @@ -385,7 +441,8 @@ def apply_plan_to_report(output_dir: Path, plan: dict[str, Any]) -> dict[str, An them for the plan's agentic components, stamps the routing record and writes them back atomically. When no component is routed agentic and the report carries no record the files are left untouched, so the non-breaking guarantee holds. A report already routed under the same decisions - is not edited again; only the record's plan hash is brought up to date. + is not edited again; only the record's plan hash is brought up to date, in the report and in the + configuration-stamped copy package reads when ``modify`` wrote one. Returns a summary dict with the altered pipeline names and the resulting gap count. @@ -413,9 +470,8 @@ def apply_plan_to_report(output_dir: Path, plan: dict[str, Any]) -> dict[str, An if record is not None: if routing_record_mismatches(record, plan): raise ValueError(REROUTED_UNDER_DIFFERENT_PLAN) - if record.get("conversion_plan_sha256") != canonical_sha256(plan): - record["conversion_plan_sha256"] = canonical_sha256(plan) - _write_json_atomic(report_path, report) + for path in (report_path, work / STAMPED_REPORT_FILENAME): + _refresh_record_plan_hash(path, plan) return {"agentic_pipelines": sorted(agentic), "gaps": len(gaps), "altered": False} new_report, new_gaps = alter_report(report, gaps, agentic) @@ -571,11 +627,13 @@ def apply_combine_fill( ``prepare -> write_bundle`` pass over :func:`validate_report_structurally`) -- there is no bypass -- and written back only when it passes, so a dangling reference or duplicate key never lands on disk. - The combine is **idempotent**, keyed on the routing record rather than pipeline names: once the - component's outcome is ``agentic-applied`` a re-run no-ops (``already_combined`` true) instead of - collapsing/appending again, so it never duplicates the authored pipeline(s) even when the authored - replacement is renamed or an authored name collides with a former member. A fresh ``convert`` - rewrites the report without the record, so the combine applies again after a re-convert and route. + Each component takes one kind of fill: a component a per-pipeline merge has already filled is + refused with :data:`COMPONENT_ALREADY_FILLED`. The combine is **idempotent**: once a combine has + removed the component's members from the report a re-run no-ops (``already_combined`` true) + instead of collapsing/appending again, so it never duplicates the authored pipeline(s) even when + the authored replacement is renamed or an authored name collides with a former member. A fresh + ``convert`` rewrites the report without the record, so the combine applies again after a + re-convert and route. Returns ``{"ok", "violations", "error", "component_id", "pipelines", "already_combined"}``. ``ok`` is ``False`` (and nothing written) on a plan/membership error (``error`` set), a missing source tag, @@ -607,7 +665,10 @@ def apply_combine_fill( if routing_record_mismatches(record, plan): return {"ok": False, "error": REROUTED_UNDER_DIFFERENT_PLAN, "violations": [], "pipelines": 0} - if record["components"][component_id].get("outcome") == OUTCOME_AGENTIC_APPLIED: + fill = component_fill(report, members) + if fill == FILL_MERGE: + return {"ok": False, "error": COMPONENT_ALREADY_FILLED, "violations": [], "pipelines": 0} + if fill == FILL_COMBINE: return { "ok": True, "error": None, diff --git a/src/flowx/sources/adf/translate.py b/src/flowx/sources/adf/translate.py index 78d6f9ad..2bb18271 100644 --- a/src/flowx/sources/adf/translate.py +++ b/src/flowx/sources/adf/translate.py @@ -6,6 +6,7 @@ import json import logging import re +import sys from collections import defaultdict from dataclasses import asdict, replace from datetime import datetime @@ -1368,7 +1369,13 @@ def main(argv: list[str] | None = None) -> int: if args.merge_agentic: if not args.report or not args.agentic_results: parser.error("--merge-agentic requires --report and --agentic-results") - merged_count, unmatched_count = ir_serde.merge_agentic_results(args.report, args.agentic_results, args.output) + try: + merged_count, unmatched_count = ir_serde.merge_agentic_results( + args.report, args.agentic_results, args.output + ) + except ValueError as error: + print(str(error), file=sys.stderr) + return 1 print("\nAgentic Merge Summary") print("=====================") print(f"Merged: {merged_count}") diff --git a/tests/unit/test_package_routing_preflight.py b/tests/unit/test_package_routing_preflight.py index e7146e05..61687577 100644 --- a/tests/unit/test_package_routing_preflight.py +++ b/tests/unit/test_package_routing_preflight.py @@ -19,6 +19,7 @@ from flowx import routing from flowx.adapter.__main__ import main as adapter_main from flowx.bundler.dab_writer import main as package_main +from flowx.discovery_insights import enrich_inventory from flowx.discovery_inventory import STRATEGY_PROPERTY, build_source_inventory from flowx.ir_serde import pipeline_to_dict from flowx.models.discovery import CONCEPT_NOTEBOOK, SourceGraph, SourceNode @@ -173,6 +174,30 @@ def test_route_combine_modify_then_package_keeps_the_routing_record(tmp_path: Pa assert audit["translation_report_sha256"] == hashlib.sha256(stamped_path.read_bytes()).hexdigest() +def test_rerouting_the_same_decisions_after_enrich_refreshes_the_stamped_report(tmp_path: Path) -> None: + report_path = _setup(tmp_path, _inventory("agentic")) + plan_path = tmp_path / "plan.json" + plan_path.write_text( + json.dumps({"components": [{"component_id": "component-1", "members": ["solo"], "decision": "agentic"}]}), + encoding="utf-8", + ) + authored_path = tmp_path / "authored.json" + authored_path.write_text(json.dumps([_authored_pipeline()]), encoding="utf-8") + route = ["route", "--output-dir", str(tmp_path), "--plan-path", str(plan_path)] + + assert adapter_main(route) == 0 + combine = ["fill-agentic", "combine", "--output-dir", str(tmp_path), "--members", "solo"] + assert adapter_main([*combine, "--pipelines-path", str(authored_path)]) == 0 + assert adapter_main(["modify", str(report_path), "--output-dir", str(tmp_path)]) == 0 + assert enrich_inventory(tmp_path, insights={"overview": "Solo loads one table."})["ok"] is True + assert _package(tmp_path) == 1 # the plan is now stale against the new insights + + assert adapter_main(route) == 0 + assert _package(tmp_path) == 0 + audit = json.loads((tmp_path / "metadata" / "route_audit.json").read_text(encoding="utf-8")) + assert audit["components"][0]["outcome"] == "agentic-applied" + + def test_route_agentic_then_deterministic_is_refused_and_writes_nothing( tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: diff --git a/tests/unit/test_route_agentic_flow.py b/tests/unit/test_route_agentic_flow.py index a37b5edc..f10d5826 100644 --- a/tests/unit/test_route_agentic_flow.py +++ b/tests/unit/test_route_agentic_flow.py @@ -34,6 +34,7 @@ from flowx.models.discovery import CONCEPT_NOTEBOOK, SourceGraph, SourceNode from flowx.models.ir import ControlEdge, Lineage from flowx.route_agentic import ( + COMPONENT_ALREADY_FILLED, GAPS_FILENAME, REPORT_FILENAME, REROUTED_UNDER_DIFFERENT_PLAN, @@ -49,6 +50,7 @@ validate_report_structurally, ) from flowx.routing import record_plan +from flowx.sources.adf.translate import main as translate_main # --------------------------------------------------------------------------- # # Fixtures. @@ -696,6 +698,52 @@ def test_combine_marks_the_component_applied(tmp_path: Path) -> None: assert _record_outcomes(report_path) == {"component-1": "agentic-applied"} +def _write_merge_result(results_dir: Path, pipeline: str, activity_name: str, task_key: str) -> None: + results_dir.mkdir(exist_ok=True) + (results_dir / f"{task_key}.json").write_text( + json.dumps( + { + "pipeline": pipeline, + "activity_name": activity_name, + "task": _notebook_task(activity_name, task_key, f"/Workspace/Shared/agentic_{task_key}"), + } + ), + encoding="utf-8", + ) + + +def test_combine_after_a_per_pipeline_merge_is_refused(tmp_path: Path) -> None: + _setup_routed_agentic(tmp_path, decision="agentic") + report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME + results_dir = tmp_path / "agentic_results" + _write_merge_result(results_dir, "parent", "Extract", "extract") + assert merge_agentic_results(report_path, results_dir) == (1, 0) + merged_report = report_path.read_bytes() + + result = apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()]) + + assert result["ok"] is False + assert result["error"] == COMPONENT_ALREADY_FILLED + assert report_path.read_bytes() == merged_report + + +def test_merge_after_a_combine_is_refused(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + _setup_routed_agentic(tmp_path, decision="agentic") + assert apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()])["ok"] is True + report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME + combined_report = report_path.read_bytes() + results_dir = tmp_path / "agentic_results" + _write_merge_result(results_dir, "parent", "Extract", "extract") + + with pytest.raises(ValueError, match=COMPONENT_ALREADY_FILLED): + merge_agentic_results(report_path, results_dir) + exit_code = translate_main(["--merge-agentic", "--report", str(report_path), "--agentic-results", str(results_dir)]) + + assert exit_code == 1 + assert COMPONENT_ALREADY_FILLED in capsys.readouterr().err + assert report_path.read_bytes() == combined_report + + def test_combine_requires_route_to_have_applied_the_plan(tmp_path: Path) -> None: _setup_routed_agentic(tmp_path, decision="agentic") _write_work(tmp_path, _report_two_pipelines()) # convert ran again, so the report has no record From d518cdea2dbed2ef62f20c186dc348e0caf0dd60 Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Wed, 7 Oct 2026 10:45:47 +0100 Subject: [PATCH 020/122] no-mistakes(review): Detect name-colliding combines via routed gaps; guard untargeted merges --- src/flowx/ir_serde.py | 77 ++++++++++++++++------- src/flowx/route_agentic.py | 89 ++++++++++++++++++--------- tests/unit/test_route_agentic_flow.py | 52 ++++++++++++++++ 3 files changed, 166 insertions(+), 52 deletions(-) diff --git a/src/flowx/ir_serde.py b/src/flowx/ir_serde.py index 7116ed88..97cd3b98 100644 --- a/src/flowx/ir_serde.py +++ b/src/flowx/ir_serde.py @@ -583,31 +583,56 @@ def pipeline_to_debug_dict(pipeline: Pipeline) -> dict[str, Any]: } -def _find_and_replace_task(tasks: list[dict[str, Any]], activity_name: str, replacement: dict[str, Any]) -> bool: - """Replace the task named *activity_name* with *replacement*, recursing into containers. +def _locate_task(tasks: list[dict[str, Any]], activity_name: str) -> tuple[list[dict[str, Any]], int] | None: + """Find the first task named *activity_name*, recursing into containers. - Searches top-level tasks and the nested activity lists of IfCondition / - ForEach / Switch containers. Preserves the placeholder's ``task_key`` and - ``depends_on`` when the replacement omits them so downstream dependency - edges stay intact. Returns True when a match was replaced. + Searches top-level tasks and the nested activity lists of IfCondition / ForEach / Switch + containers. Returns the list holding the task and its index, or ``None`` when there is none. """ nested_keys = ("inner_activities", "if_true_activities", "if_false_activities", "default_activities") for index, task in enumerate(tasks): if task.get("name") == activity_name: - replacement.setdefault("task_key", task.get("task_key")) - replacement.setdefault("name", activity_name) - if "depends_on" not in replacement and task.get("depends_on"): - replacement["depends_on"] = task["depends_on"] - tasks[index] = replacement - return True + return tasks, index for key in nested_keys: child = task.get(key) - if isinstance(child, list) and _find_and_replace_task(child, activity_name, replacement): - return True + if isinstance(child, list): + found = _locate_task(child, activity_name) + if found is not None: + return found for case in task.get("cases") or []: if isinstance(case, dict) and isinstance(case.get("activities"), list): - if _find_and_replace_task(case["activities"], activity_name, replacement): - return True + found = _locate_task(case["activities"], activity_name) + if found is not None: + return found + return None + + +def _find_and_replace_task(tasks: list[dict[str, Any]], activity_name: str, replacement: dict[str, Any]) -> bool: + """Replace the task named *activity_name* with *replacement*, recursing into containers. + + Preserves the placeholder's ``task_key`` and ``depends_on`` when the replacement omits them so + downstream dependency edges stay intact. Returns True when a match was replaced. + """ + found = _locate_task(tasks, activity_name) + if found is None: + return False + container, index = found + task = container[index] + replacement.setdefault("task_key", task.get("task_key")) + replacement.setdefault("name", activity_name) + if "depends_on" not in replacement and task.get("depends_on"): + replacement["depends_on"] = task["depends_on"] + container[index] = replacement + return True + + +def _lands_on_open_placeholder(pipelines: list[dict[str, Any]], activity_name: Any, blocked: set[str]) -> bool: + """Whether an untargeted merge result would replace a placeholder outside the *blocked* pipelines.""" + for pipeline in pipelines: + found = _locate_task(pipeline.get("tasks", []), activity_name) + if found is not None: + container, index = found + return pipeline.get("name") not in blocked and container[index].get("type") == "PlaceholderActivity" return False @@ -628,8 +653,9 @@ def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Pat wrote to the workspace; the prepare phase then references it directly. When the report carries a routing record, each agentic component's outcome is updated to show - whether the merge left any of its routed placeholders unfilled, and a result aimed at a pipeline - of a component a combine already filled is refused before anything is written. + whether the merge left any of its routed placeholders unfilled. Once a combine has filled a + component, a result aimed at one of its pipelines, or an untargeted result that would not replace + a remaining placeholder elsewhere, is refused before anything is written. Args: report_path: ``translation_report.json`` produced by the translate phase. @@ -641,9 +667,9 @@ def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Pat ``(merged, unmatched)`` counts. Raises: - ValueError: A result targets a pipeline of a component a combine already filled. + ValueError: A result would land in a component a combine already filled. """ - from flowx.route_agentic import COMPONENT_ALREADY_FILLED, combined_pipelines, refresh_routing_outcomes + from flowx.route_agentic import COMPONENT_ALREADY_FILLED, combined_pipelines, load_gaps, refresh_routing_outcomes report = json.loads(report_path.read_text(encoding="utf-8")) pipelines = report["pipelines"] if isinstance(report, dict) and "pipelines" in report else [report] @@ -651,9 +677,14 @@ def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Pat (result_file, json.loads(result_file.read_text(encoding="utf-8"))) for result_file in sorted(results_dir.glob("*.json")) ] - combined = combined_pipelines(report) if isinstance(report, dict) else set() - if any(data.get("pipeline") in combined for _, data in results): - raise ValueError(COMPONENT_ALREADY_FILLED) + combined = combined_pipelines(report, load_gaps(report_path.parent)) if isinstance(report, dict) else set() + for _, data in results: + wanted = data.get("pipeline") + activity_name = data.get("activity_name") or data.get("activity") + if wanted in combined or ( + combined and not wanted and not _lands_on_open_placeholder(pipelines, activity_name, combined) + ): + raise ValueError(COMPONENT_ALREADY_FILLED) merged = 0 unmatched = 0 diff --git a/src/flowx/route_agentic.py b/src/flowx/route_agentic.py index 59dfbf9e..b82217f3 100644 --- a/src/flowx/route_agentic.py +++ b/src/flowx/route_agentic.py @@ -197,35 +197,69 @@ def refresh_routing_outcomes(report: dict[str, Any]) -> None: entry["outcome"] = OUTCOME_AGENTIC_NOT_VIABLE if unfilled else OUTCOME_AGENTIC_APPLIED -def component_fill(report: dict[str, Any], members: Iterable[str]) -> str | None: - """Which kind of fill has touched a routed-agentic component, read from the report itself. +def load_gaps(work_dir: Path) -> list[dict[str, Any]]: + """The gaps recorded beside a translation report in ``work_dir``, or an empty list when none were.""" + gaps_path = Path(work_dir) / GAPS_FILENAME + gaps = json.loads(gaps_path.read_text(encoding="utf-8")) if gaps_path.exists() else [] + return [gap for gap in gaps if isinstance(gap, dict)] if isinstance(gaps, list) else [] + + +def _routed_task_names(gaps: list[dict[str, Any]]) -> dict[str, set[str]]: + """The task names route turned into placeholders, per pipeline, from its pipeline-tagged gaps.""" + names: dict[str, set[str]] = {} + for gap in gaps: + if gap.get("pipeline") is not None: + names.setdefault(str(gap["pipeline"]), set()).add(str(gap.get("activity_name"))) + return names + - Only a combine removes member pipelines from the report, and route turns every top-level task of - a member into a placeholder, so a member task that is no longer a placeholder came from a - per-pipeline merge. Returns :data:`FILL_COMBINE`, :data:`FILL_MERGE`, or ``None`` when neither - has run. +def component_fill(report: dict[str, Any], members: Iterable[str], gaps: list[dict[str, Any]]) -> str | None: + """Which kind of fill has touched a routed-agentic component, read from the report and its gaps. + + Route turns every top-level task of a member into a placeholder and records one gap per task. A + per-pipeline merge replaces those placeholders by name, so a merged member keeps the routed task + names. A combine replaces the members with authored pipelines, so a member is combined when it + is gone from the report, or when an authored pipeline reusing its name holds no placeholder and + none of the routed task names. Returns :data:`FILL_COMBINE`, :data:`FILL_MERGE`, or ``None`` + when neither has run. """ by_name = {pipeline.get("name"): pipeline for pipeline in _report_pipelines(report)} - members = list(members) - if any(member not in by_name for member in members): - return FILL_COMBINE + routed = _routed_task_names(gaps) + filled = False for member in members: - for task in by_name[member].get("tasks", []): - if isinstance(task, dict) and task.get("type") != "PlaceholderActivity": - return FILL_MERGE - return None - - -def combined_pipelines(report: dict[str, Any]) -> set[str]: - """The member pipelines of every routed-agentic component a combine has already filled.""" + pipeline = by_name.get(member) + if pipeline is None: + return FILL_COMBINE + tasks = [task for task in pipeline.get("tasks", []) if isinstance(task, dict)] + placeholders = [task for task in tasks if task.get("type") == "PlaceholderActivity"] + if tasks and not placeholders and not {str(task.get("name")) for task in tasks} & routed.get(member, set()): + return FILL_COMBINE + filled = filled or len(placeholders) < len(tasks) + return FILL_MERGE if filled else None + + +def combined_pipelines(report: dict[str, Any], gaps: list[dict[str, Any]]) -> set[str]: + """The pipelines a combine has filled: combined members, and the authored pipelines beside them. + + A pipeline that is no member of any routed component was authored by a combine, so once any + component is combined those pipelines count too. Empty when the report has no routing record. + """ record = routing_record(report) components = record.get("components") if record is not None else None + if not isinstance(components, dict): + return set() names: set[str] = set() - for entry in components.values() if isinstance(components, dict) else []: - if isinstance(entry, dict) and entry.get("decision") == DECISION_AGENTIC: - members = [str(member) for member in entry.get("members") or []] - if component_fill(report, members) == FILL_COMBINE: - names.update(members) + all_members: set[str] = set() + for entry in components.values(): + if not isinstance(entry, dict): + continue + members = [str(member) for member in entry.get("members") or []] + all_members.update(members) + if entry.get("decision") == DECISION_AGENTIC and component_fill(report, members, gaps) == FILL_COMBINE: + names.update(members) + if names: + authored = {str(pipeline.get("name")) for pipeline in _report_pipelines(report)} - all_members + names.update(authored) return names @@ -454,7 +488,6 @@ def apply_plan_to_report(output_dir: Path, plan: dict[str, Any]) -> dict[str, An report_path = work / REPORT_FILENAME if not report_path.exists(): raise FileNotFoundError(f"No {REPORT_FILENAME} under {work}; run the convert phase first.") - gaps_path = work / GAPS_FILENAME agentic = agentic_pipeline_names(plan) baseline_bytes = report_path.read_bytes() @@ -463,9 +496,7 @@ def apply_plan_to_report(output_dir: Path, plan: dict[str, Any]) -> dict[str, An if record is None and not agentic: return {"agentic_pipelines": [], "gaps": 0, "altered": False} - gaps = json.loads(gaps_path.read_text(encoding="utf-8")) if gaps_path.exists() else [] - if not isinstance(gaps, list): - gaps = [] + gaps = load_gaps(work) if record is not None: if routing_record_mismatches(record, plan): @@ -479,7 +510,7 @@ def apply_plan_to_report(output_dir: Path, plan: dict[str, Any]) -> dict[str, An new_report = {"pipelines": [new_report]} new_report[ROUTING_RECORD_KEY] = build_routing_record(plan, baseline_bytes) _write_json_atomic(report_path, new_report) - _write_json_atomic(gaps_path, new_gaps) + _write_json_atomic(work / GAPS_FILENAME, new_gaps) return {"agentic_pipelines": sorted(agentic), "gaps": len(new_gaps), "altered": True} @@ -629,7 +660,7 @@ def apply_combine_fill( Each component takes one kind of fill: a component a per-pipeline merge has already filled is refused with :data:`COMPONENT_ALREADY_FILLED`. The combine is **idempotent**: once a combine has - removed the component's members from the report a re-run no-ops (``already_combined`` true) + replaced the component's members (see :func:`component_fill`) a re-run no-ops (``already_combined`` true) instead of collapsing/appending again, so it never duplicates the authored pipeline(s) even when the authored replacement is renamed or an authored name collides with a former member. A fresh ``convert`` rewrites the report without the record, so the combine applies again after a @@ -665,7 +696,7 @@ def apply_combine_fill( if routing_record_mismatches(record, plan): return {"ok": False, "error": REROUTED_UNDER_DIFFERENT_PLAN, "violations": [], "pipelines": 0} - fill = component_fill(report, members) + fill = component_fill(report, members, load_gaps(work)) if fill == FILL_MERGE: return {"ok": False, "error": COMPONENT_ALREADY_FILLED, "violations": [], "pipelines": 0} if fill == FILL_COMBINE: diff --git a/tests/unit/test_route_agentic_flow.py b/tests/unit/test_route_agentic_flow.py index f10d5826..0216d684 100644 --- a/tests/unit/test_route_agentic_flow.py +++ b/tests/unit/test_route_agentic_flow.py @@ -744,6 +744,58 @@ def test_merge_after_a_combine_is_refused(tmp_path: Path, capsys: pytest.Capture assert report_path.read_bytes() == combined_report +def _colliding_authored_pipelines() -> list[dict[str, Any]]: + """Authored combine pipelines that reuse both member names but none of the routed task names.""" + publish = {"name": "child", "tags": {"source": "adf"}, "tasks": [_notebook_task("Publish", "publish")]} + return [_named_lfc_pipeline("parent"), publish] + + +def test_combine_rerun_is_idempotent_when_authored_names_reuse_every_member(tmp_path: Path) -> None: + _setup_routed_agentic(tmp_path, decision="agentic") + + first = apply_combine_fill(tmp_path, ["parent", "child"], _colliding_authored_pipelines()) + assert first["ok"] is True and first["already_combined"] is False + second = apply_combine_fill(tmp_path, ["parent", "child"], _colliding_authored_pipelines()) + + assert second["ok"] is True + assert second["already_combined"] is True + report = json.loads((tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_text(encoding="utf-8")) + assert [pipeline["name"] for pipeline in report["pipelines"]] == ["parent", "child"] + + +@pytest.mark.parametrize( + ("pipeline", "activity_name"), + [("parent", "Ingest orders"), (None, "Publish"), (None, "Extract")], +) +def test_merge_after_a_combine_reusing_every_member_name_is_refused( + tmp_path: Path, pipeline: str | None, activity_name: str +) -> None: + _setup_routed_agentic(tmp_path, decision="agentic") + assert apply_combine_fill(tmp_path, ["parent", "child"], _colliding_authored_pipelines())["ok"] is True + report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME + combined_report = report_path.read_bytes() + results_dir = tmp_path / "agentic_results" + results_dir.mkdir() + result: dict[str, Any] = {"activity_name": activity_name, "task": _notebook_task(activity_name, "replaced")} + if pipeline is not None: + result["pipeline"] = pipeline + (results_dir / "result.json").write_text(json.dumps(result), encoding="utf-8") + + with pytest.raises(ValueError, match=COMPONENT_ALREADY_FILLED): + merge_agentic_results(report_path, results_dir) + assert report_path.read_bytes() == combined_report + + +def test_merge_into_an_authored_combine_pipeline_is_refused(tmp_path: Path) -> None: + _setup_routed_agentic(tmp_path, decision="agentic") + assert apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()])["ok"] is True + results_dir = tmp_path / "agentic_results" + _write_merge_result(results_dir, "orders_lfc", "Ingest orders", "ingest_orders") + + with pytest.raises(ValueError, match=COMPONENT_ALREADY_FILLED): + merge_agentic_results(tmp_path / WORK_DIRNAME / REPORT_FILENAME, results_dir) + + def test_combine_requires_route_to_have_applied_the_plan(tmp_path: Path) -> None: _setup_routed_agentic(tmp_path, decision="agentic") _write_work(tmp_path, _report_two_pipelines()) # convert ran again, so the report has no record From fcf5a6027d68bc869d0b397a2c9fb0fdba702d03 Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Wed, 7 Oct 2026 11:07:45 +0100 Subject: [PATCH 021/122] no-mistakes(review): Fill routed-agentic components only by combine; refuse routed merges --- skills/flowx-convert/SKILL.md | 2 +- skills/flowx-migrate/SKILL.md | 7 +- skills/flowx-migrate/references/workflow.md | 9 +- skills/flowx-route/SKILL.md | 74 +++---- src/flowx/ir_serde.py | 35 ++-- src/flowx/mcp/server.py | 20 +- src/flowx/route_agentic.py | 202 +++++++------------- src/flowx/routing.py | 7 +- tests/unit/test_route_agentic_flow.py | 114 +++++------ 9 files changed, 189 insertions(+), 281 deletions(-) diff --git a/skills/flowx-convert/SKILL.md b/skills/flowx-convert/SKILL.md index 10155d0c..2586eb4f 100644 --- a/skills/flowx-convert/SKILL.md +++ b/skills/flowx-convert/SKILL.md @@ -70,7 +70,7 @@ across ADF and Airflow. `show_when` condition). Walk it locally; ask an option only when its `show_when` is satisfied. - `modify --output-dir --answer OPTION_ID=VALUE ...` — validate and apply collected answers, writing `.work/translation_report.stamped.json` + `metadata/configuration.json`. -- `merge_agentic --report --agentic-results ` — **ADF only**. Fold agent-produced per-activity translations into an ADF report. Airflow's legacy name-based merge is disabled. +- `merge_agentic --report --agentic-results ` — **ADF only**. Fold agent-produced per-activity translations into an ADF report. It refuses, writing nothing, any result landing in a pipeline the routing record routes agentic (fill those with `fill-agentic combine`). Airflow's legacy name-based merge is disabled. ## Output artifacts (shared, transient under `/.work/`) diff --git a/skills/flowx-migrate/SKILL.md b/skills/flowx-migrate/SKILL.md index 617c454e..2a18ac0e 100644 --- a/skills/flowx-migrate/SKILL.md +++ b/skills/flowx-migrate/SKILL.md @@ -38,9 +38,10 @@ This is the top-level orchestration skill. It runs the full migration pipeline: 4. **Route** — Decide, per connected component, deterministic vs. agentic conversion; record the fingerprint-bound `metadata/conversion_plan.json`; edit the baseline report so routed-agentic groups become placeholder gaps (`flowx-route`) -5. **Fill agentic gaps** — Replace the routed-agentic placeholders: per-pipeline - `convert --merge-agentic` (ADF only) or cross-pipeline `fill-agentic combine`; Airflow gaps go - through `flowx-resolve-airflow-gaps`. **Never re-run a plain `convert` after routing** — it +5. **Fill agentic gaps** — Replace each routed-agentic group with `fill-agentic combine` (one + same-named authored pipeline keeps a pipeline 1:1); `convert --merge-agentic` (ADF only) fills + convert's own gaps and refuses routed-agentic pipelines; Airflow gaps go through + `flowx-resolve-airflow-gaps`. **Never re-run a plain `convert` after routing** — it rewrites the report and erases the placeholders 6. **Package** — Generate Databricks Declarative Automation Bundles for deployment diff --git a/skills/flowx-migrate/references/workflow.md b/skills/flowx-migrate/references/workflow.md index 7b0d4fd1..2ffbee6b 100644 --- a/skills/flowx-migrate/references/workflow.md +++ b/skills/flowx-migrate/references/workflow.md @@ -79,12 +79,13 @@ merges: so those groups' tasks become placeholder gaps and stamps a routing record onto it (per component: members, decision and outcome, plus the plan and baseline report hashes); a fully-deterministic plan (or no plan) leaves convert/package behaving exactly as before — the non-breaking guarantee. - The fills and `modify` carry the record forward; re-routing under different decisions is refused + Combine and `modify` carry the record forward; re-routing under different decisions is refused (re-run convert, then route), and package refuses a report whose record no longer matches the plan. -Routed-agentic groups are then filled during convert: per-pipeline via `convert --merge-agentic`, or -cross-pipeline (N→M, e.g. a Lakeflow Connect collapse) via `fill-agentic combine` using authored -`AgenticComponentActivity` nodes. +Routed-agentic groups are then filled only by `fill-agentic combine`, which replaces each group with +authored pipelines (N→M, e.g. a Lakeflow Connect collapse, or one same-named pipeline to stay 1:1), +typically using `AgenticComponentActivity` nodes. `convert --merge-agentic` fills convert's own gaps +and refuses routed-agentic pipelines. ## Phase 2: Convert diff --git a/skills/flowx-route/SKILL.md b/skills/flowx-route/SKILL.md index 5dc31f05..5c12e4d4 100644 --- a/skills/flowx-route/SKILL.md +++ b/skills/flowx-route/SKILL.md @@ -3,8 +3,8 @@ name: flowx-route description: > Route each connected component of the discovered inventory to a deterministic (1:1 engine) or agentic (LLM-assisted re-architecture) conversion, record the fingerprint-bound conversion plan, - and fill the routed-agentic groups — per-pipeline via convert --merge-agentic, or cross-pipeline - via fill-agentic combine. Runs after enrich, before/with convert. + and fill the routed-agentic groups with fill-agentic combine. Runs after enrich, before/with + convert. triggers: - "route pipelines" - "route conversion" @@ -52,8 +52,8 @@ discover → enrich (default) → convert (deterministic baseline) → route (de Convert builds the deterministic baseline report **before** routing (route can trigger it in-process via `--source` / `--source-path`); routing then edits that report. After routing has recorded agentic -placeholders, the only convert is the additive `convert --merge-agentic` fill — never a second plain -`convert`, which would overwrite the report and erase the placeholders. +placeholders, never run a second plain `convert`: it would overwrite the report and erase the +placeholders. Pipelines are grouped into weak/undirected **connected components** over the inventory's control lineage (`lineage.control_edges`), so mutually-referencing pipelines are decided together and a @@ -102,9 +102,9 @@ when nothing is routed agentic the report is untouched — the non-breaking guar When it edits the report, route also stamps a **routing record** onto it (the top-level `_routing_record` key): the plan's hash, the hash of the deterministic report it started from, and one entry per `component_id` with its `members`, `decision` and `outcome` — `deterministic`, -`agentic-not-viable` (nothing filled yet) or `agentic-applied` (a fill left none of its members with a -routed placeholder). Every later rewrite (the fills and `modify`) carries the record forward and only -the fills update outcomes. Re-running route under the **same** decisions only refreshes the record's +`agentic-not-viable` (nothing filled yet) or `agentic-applied` (`fill-agentic combine` filled it). +Every later rewrite (combine and `modify`) carries the record forward and only combine updates +outcomes. Re-running route under the **same** decisions only refreshes the record's plan hash (in the report and in the stamped copy `modify` wrote); under different components or decisions it refuses and writes nothing — re-run convert, then route. @@ -172,47 +172,17 @@ report already routed under different components or decisions. ## Step 3 — Fill the routed-agentic groups Every routed-agentic pipeline's tasks are now `PlaceholderActivity` nodes with one pipeline-tagged -`AgenticGap` each. Two fills exist depending on the grain; you author the replacement (no LLM in the -library, which validates and records). +`AgenticGap` each. A routed-agentic group is filled **only** by `fill-agentic combine`, which replaces +the whole group with pipelines you author (no LLM in the library, which validates and records). -### 3a — Per-pipeline fill (keep the pipeline 1:1) +### 3a — Keep a pipeline 1:1 -When a routed-agentic pipeline stays one pipeline and you just author a Databricks task per gap, reuse -the **existing** name-matched merge — the same path the agentic gap flow uses. Write one JSON file per -resolved gap into an agentic-results directory: - -```json -{ - "activity_name": "", - "pipeline": "", - "task": {"type": "NotebookActivity", "name": "", "task_key": "", - "notebook_path": "/Workspace/.../your_translated_notebook"} -} -``` - -Then merge (`task_key`/`depends_on` are inherited from the matched task when omitted, preserving edges): - -```bash -"$PY" -m flowx.adapter convert --source adf --merge-agentic \ - --report /.work/translation_report.json \ - --agentic-results \ - [--output ] # default: overwrite --report -``` - -This per-pipeline merge is **ADF-only**. `--source adf` is **mandatory** — the convert phase runner -exits 2 without it, and `--source airflow` is explicitly rejected here (Airflow's `--merge-agentic` -is disabled and exits 2). For **Airflow** per-gap fills, use the **`flowx-resolve-airflow-gaps`** -skill (the fingerprint-bound `resolve-agentic` workflow), not this command. - -The merge finds, per result, the **first task whose `name` matches `activity_name`** (recursing into -`IfCondition`/`ForEach`/`Switch` containers) and replaces it — it matches by name and does **not** -verify the target is a `PlaceholderActivity`, so make sure each `activity_name` targets the intended -agentic gap. Other pipelines and unmatched tasks are left untouched. - -MCP: `flowx(command="merge_agentic", parameters={"source": "adf", "report_path": ..., "agentic_results_dir": ..., "output_path": ...})`. -The matched task is replaced in place by the authored task definition (carrying whatever fields that -task defines; the merge itself sets no `status`); the command exits non-zero if any result can't be -matched. +To convert a routed-agentic pipeline agentically but keep it as one pipeline, combine its component +with one authored pipeline of the **same name** (see 3b). The per-pipeline +`convert --merge-agentic` is for convert's **own** agentic gaps only: it refuses, writing nothing, any +result that names a routed-agentic pipeline, or whose activity would first land in one, with "this +pipeline is routed agentic; fill it with fill-agentic combine (to change the approach, change the plan, +re-run convert and route, then combine again)". ### 3b — Cross-pipeline COMBINE (N pipelines → M) @@ -238,11 +208,11 @@ MCP: `flowx(command="fill_agentic", parameters={"output_dir": ..., "members": [. `source_graphs_sha256` and `source_insights_sha256` must still match the current inventory. A partial group, a superset, a typo, or a deterministic component is refused — you can't swap pipelines the plan didn't route agentic. The report must carry a routing record that matches that plan (route has applied it). -- One kind of fill per component: combine on a component a per-pipeline merge already filled, or - a merge aimed at a component a combine already filled, is refused with "component already filled; - re-run convert and route to start again" and writes nothing. Re-running combine on a component it - already combined is a no-op (`already_combined: true`), so the authored pipelines are never - appended twice. +- Combine marks the component `agentic-applied`. Re-running it with the same authored pipelines as + those now in the report is a no-op (`already_combined: true`), even when authored names reuse + member names, so the authored pipelines are never appended twice. A combine with different + authored pipelines on an applied component is refused with "component already filled; re-run + convert and route to start again" and writes nothing. - `--pipelines-path` is a JSON **list** of pipeline IR dicts (the authored replacements), typically carrying `AgenticComponentActivity` nodes (see below). - Each authored pipeline **must** carry the source tag `"tags": {"source": "adf"}` (routing/agentic @@ -333,5 +303,5 @@ pipelines. ## Reference - `flowx-enrich` skill — the `insights` layer routing's agentic option consumes. -- `flowx-convert` — the deterministic engine, per-pipeline `merge_agentic`, and just-in-time config. +- `flowx-convert` — the deterministic engine, `merge_agentic` for convert's own gaps, and just-in-time config. - `flowx-package` — turns the (filled) report into the deployable DAB bundle. diff --git a/src/flowx/ir_serde.py b/src/flowx/ir_serde.py index 97cd3b98..fd8c9e7c 100644 --- a/src/flowx/ir_serde.py +++ b/src/flowx/ir_serde.py @@ -626,14 +626,14 @@ def _find_and_replace_task(tasks: list[dict[str, Any]], activity_name: str, repl return True -def _lands_on_open_placeholder(pipelines: list[dict[str, Any]], activity_name: Any, blocked: set[str]) -> bool: - """Whether an untargeted merge result would replace a placeholder outside the *blocked* pipelines.""" +def _merge_target(pipelines: list[dict[str, Any]], wanted: Any, activity_name: Any) -> Any: + """The pipeline a merge result lands in: the one it names, else the first holding its activity.""" + if wanted: + return wanted for pipeline in pipelines: - found = _locate_task(pipeline.get("tasks", []), activity_name) - if found is not None: - container, index = found - return pipeline.get("name") not in blocked and container[index].get("type") == "PlaceholderActivity" - return False + if _locate_task(pipeline.get("tasks", []), activity_name) is not None: + return pipeline.get("name") + return None def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Path | None = None) -> tuple[int, int]: @@ -652,10 +652,9 @@ def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Pat ``NotebookActivity`` whose ``notebook_path`` points at a notebook the agent wrote to the workspace; the prepare phase then references it directly. - When the report carries a routing record, each agentic component's outcome is updated to show - whether the merge left any of its routed placeholders unfilled. Once a combine has filled a - component, a result aimed at one of its pipelines, or an untargeted result that would not replace - a remaining placeholder elsewhere, is refused before anything is written. + This fills convert's own agentic gaps. A pipeline the routing record routes agentic is filled + only by ``fill-agentic combine``, so a result that names one, or an untargeted result whose + activity would first land in one, is refused before anything is written. Args: report_path: ``translation_report.json`` produced by the translate phase. @@ -667,9 +666,9 @@ def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Pat ``(merged, unmatched)`` counts. Raises: - ValueError: A result would land in a component a combine already filled. + ValueError: A result would land in a pipeline the routing record routes agentic. """ - from flowx.route_agentic import COMPONENT_ALREADY_FILLED, combined_pipelines, load_gaps, refresh_routing_outcomes + from flowx.route_agentic import ROUTED_AGENTIC_MERGE_REFUSED, routed_agentic_pipelines report = json.loads(report_path.read_text(encoding="utf-8")) pipelines = report["pipelines"] if isinstance(report, dict) and "pipelines" in report else [report] @@ -677,14 +676,11 @@ def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Pat (result_file, json.loads(result_file.read_text(encoding="utf-8"))) for result_file in sorted(results_dir.glob("*.json")) ] - combined = combined_pipelines(report, load_gaps(report_path.parent)) if isinstance(report, dict) else set() + routed = routed_agentic_pipelines(report) for _, data in results: - wanted = data.get("pipeline") activity_name = data.get("activity_name") or data.get("activity") - if wanted in combined or ( - combined and not wanted and not _lands_on_open_placeholder(pipelines, activity_name, combined) - ): - raise ValueError(COMPONENT_ALREADY_FILLED) + if routed and _merge_target(pipelines, data.get("pipeline"), activity_name) in routed: + raise ValueError(ROUTED_AGENTIC_MERGE_REFUSED) merged = 0 unmatched = 0 @@ -704,7 +700,6 @@ def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Pat logger.warning("No placeholder named '%s' found for %s", activity_name, result_file.name) unmatched += 1 - refresh_routing_outcomes(report) destination = output_path or report_path destination.write_text(json.dumps(report, indent=2, default=str), encoding="utf-8") logger.info("Wrote merged report to %s (%d merged, %d unmatched)", destination, merged, unmatched) diff --git a/src/flowx/mcp/server.py b/src/flowx/mcp/server.py index 6933c7c1..b4afcf9f 100644 --- a/src/flowx/mcp/server.py +++ b/src/flowx/mcp/server.py @@ -55,7 +55,7 @@ def _transport_security() -> TransportSecuritySettings: flowx("convert", {"source": "adf", "output_dir": "..."}) flowx("route", {"output_dir": "..."}) # optional: recommend, then re-call with a plan flowx("route", {"output_dir": "...", "plan": {...}}) # record the decision + edit the report - flowx("fill_agentic", {"output_dir": "...", "members": [...], "pipelines": [...]}) # combine fill (or merge_agentic) + flowx("fill_agentic", {"output_dir": "...", "members": [...], "pipelines": [...]}) # fill a routed-agentic group flowx("inspect", {"report_path": "/.work/translation_report.json"}) flowx("apply_answers", {"report_path": "...", "answers": ["id=value"], "output_dir": "..."}) flowx("package", {"output_dir": "...", "catalog": "main", "schema": "default"}) @@ -393,7 +393,8 @@ def _cmd_fill_agentic(p: dict[str, Any]) -> dict[str, Any]: replace; the authored pipeline IR dicts come inline as ``pipelines`` (a JSON list) or via ``pipelines_path`` (exactly one, staged to a temp file so the same CLI contract runs on both). The merged report is validated structurally before it is written back; ``ok`` is False (nothing - written) when a structural invariant is violated. Per-pipeline agentic fills use ``merge_agentic``. + written) when a structural invariant is violated. It is the only fill for a routed-agentic group; + ``merge_agentic`` fills convert's own gaps only. """ output_dir = p.get("output_dir", "./flowx_output") members = p.get("members") @@ -676,7 +677,9 @@ def flowx(command: str, parameters: dict[str, Any] | None = None) -> dict[str, A - "convert": source(req), (one ADF source key | airflow_source_path), output_dir, pipeline, exclude_dag | exclude_dags (Airflow, repeatable list). - "merge_agentic": source(req: "adf"), report_path(req), agentic_results_dir(req), output_path — - merge ADF agent results. Airflow's legacy name-based merge is disabled; use resolve_agentic. + merge ADF agent results for convert's own agentic gaps. Refuses (writing nothing) any + result landing in a pipeline the routing record routes agentic — fill those with + fill_agentic. Airflow's legacy name-based merge is disabled; use resolve_agentic. - "resolve_agentic": source(req: "airflow"), action(req: prepare | stage | apply), output_dir, airflow_source_path, report_path, gap_id, candidates, replace, accept_gap | accept_gaps, accept_all, review_complete, review_manifest, reset — @@ -698,16 +701,19 @@ def flowx(command: str, parameters: dict[str, Any] | None = None) -> dict[str, A (deterministic groups untouched; convert's deterministic translation is never modified, and with no agentic decision the report is byte-identical to today). The edit stamps a routing record onto the report (per component_id: members, decision, outcome; plus the plan and - baseline report hashes) that fills and apply_answers carry forward; re-routing under - different decisions is refused (re-run convert, then route). + baseline report hashes) that fill_agentic and apply_answers carry forward; re-routing under + different decisions is refused (re-run convert, then route). Routed-agentic groups are + filled only by fill_agentic. - "fill_agentic": output_dir(req), members(req: list of pipeline names or comma-separated string), one of pipelines(inline list of pipeline IR dicts) | pipelines_path — cross-pipeline COMBINE fill: replace a routed-agentic group's pipelines with the agent-authored pipeline(s) (typically AgenticComponentActivity nodes). `members` must exactly match a routed-agentic component in the recorded, fingerprint-bound conversion_plan.json and the report's routing record, and the merged report is always validated structurally before writing (no bypass); - the component's outcome becomes agentic-applied. Per-pipeline agentic fills use - "merge_agentic" instead. + the component's outcome becomes agentic-applied. It is the only fill for a routed-agentic + group (to keep a pipeline 1:1, author one same-named pipeline). Re-running it with the same + authored pipelines returns already_combined; different ones are refused ("component already + filled; re-run convert and route to start again"). - "inspect": report_path(req) — return the full translation-option schema (every option with a `show_when` condition) for the agent to walk locally. See "Collecting options" below. - "apply_answers": report_path(req), answers(req, list of "ID=VALUE"), output_dir, lookup_csv. diff --git a/src/flowx/route_agentic.py b/src/flowx/route_agentic.py index b82217f3..ce8d2497 100644 --- a/src/flowx/route_agentic.py +++ b/src/flowx/route_agentic.py @@ -14,11 +14,12 @@ path handles them without duplicates and the edit is idempotent. Pipelines in deterministic groups are left byte-identical, and when nothing is routed agentic the report and gaps are returned unchanged -- the non-breaking guarantee. -* the agent authors the fill. **Per-pipeline** agentic reuses the existing name-matched - :func:`flowx.ir_serde.merge_agentic_results`. **Cross-pipeline COMBINE** - (N pipelines -> M, e.g. one Lakeflow Connect pipeline) is the one net-new capability: - :func:`combine_group_fill` swaps the routed group's pipelines for the agent-authored pipeline(s), - which carry :class:`~flowx.models.ir.AgenticComponentActivity` nodes (the escape hatch). +* the agent authors the fill. A routed-agentic component is filled **only** by the cross-pipeline + COMBINE (N pipelines -> M, e.g. one Lakeflow Connect pipeline, or one same-named pipeline to keep + it 1:1): :func:`combine_group_fill` swaps the routed group's pipelines for the agent-authored + pipeline(s), which carry :class:`~flowx.models.ir.AgenticComponentActivity` nodes (the escape + hatch). The name-matched :func:`flowx.ir_serde.merge_agentic_results` stays for convert's own + gaps and refuses routed-agentic pipelines. * :func:`validate_report_structurally` packages the merged report through the same ``prepare -> write_bundle`` path the package phase uses and runs the existing :func:`flowx.validate.bundle_invariants.check_bundle_dir` over the output, so a fill can never @@ -26,8 +27,9 @@ Route stamps one routing record onto the report (:data:`ROUTING_RECORD_KEY`), keyed by the plan's component ids, holding each component's members, decision and outcome plus the plan hash and the hash -of the report route started from. Every later rewrite of the report carries it forward, the fills -update the outcomes, and package refuses a report whose record no longer matches the recorded plan. +of the report route started from. Every later rewrite of the report carries it forward, combine +marks its component applied, and package refuses a report whose record no longer matches the +recorded plan. There is no LLM here: the tool edits the report and validates the authored fill; the fill itself is supplied by the agent/harness, following the same ask -> author -> continue pattern as the existing @@ -59,33 +61,28 @@ METADATA_DIRNAME = "metadata" INVENTORY_FILENAME = "inventory.json" -# Routing / in-engine agentic conversion is ADF-only, so every agent-authored combine pipeline must -# carry this source tag. The package preflight enforces it too, but combine asserts it up front so a -# mis-tagged authored pipeline fails closed here (nothing written) instead of surviving to package. -REQUIRED_COMBINE_SOURCE_TAG = "adf" - # Top-level report key holding the routing record. It lives in the report, so a fresh convert (which # rewrites the report) clears it; the package phase and ir_serde read only the pipelines beside it. ROUTING_RECORD_KEY = "_routing_record" # What became of each component in the record: a deterministic component is left as convert wrote it; -# an agentic one stays "not viable" until a fill leaves none of its members with a routed placeholder. +# an agentic one stays "not viable" until combine fills it. OUTCOME_DETERMINISTIC = "deterministic" OUTCOME_AGENTIC_APPLIED = "agentic-applied" OUTCOME_AGENTIC_NOT_VIABLE = "agentic-not-viable" REROUTED_UNDER_DIFFERENT_PLAN = "the report was routed under a different plan; re-run convert, then route" -# Each routed-agentic component takes exactly one kind of fill: a cross-pipeline combine or -# per-pipeline merges, never both. COMPONENT_ALREADY_FILLED = "component already filled; re-run convert and route to start again" -FILL_COMBINE = "combine" -FILL_MERGE = "merge" +ROUTED_AGENTIC_MERGE_REFUSED = ( + "this pipeline is routed agentic; fill it with fill-agentic combine " + "(to change the approach, change the plan, re-run convert and route, then combine again)" +) # Guidance stamped onto every placeholder the alteration produces. _PLACEHOLDER_COMMENT = ( - "Routed agentic by the conversion plan; author a replacement task (per-pipeline fill) or replace " - "the whole group with agent-authored pipeline(s) (cross-pipeline combine)." + "Routed agentic by the conversion plan; replace the whole group with agent-authored pipeline(s) " + "using fill-agentic combine." ) @@ -176,73 +173,11 @@ def routing_record_mismatches(record: dict[str, Any], plan: dict[str, Any]) -> l return mismatches -def refresh_routing_outcomes(report: dict[str, Any]) -> None: - """Update each agentic component's outcome in the report's record after a fill, in place. - - A component is applied once none of its member pipelines left in the report still holds a routed - placeholder; a combine removes the members entirely. A report without a record is left alone. - """ - record = routing_record(report) - if record is None or not isinstance(record.get("components"), dict): - return - with_placeholders = { - pipeline.get("name") - for pipeline in _report_pipelines(report) - for task in pipeline.get("tasks", []) - if isinstance(task, dict) and task.get("type") == "PlaceholderActivity" - } - for entry in record["components"].values(): - if isinstance(entry, dict) and entry.get("decision") == DECISION_AGENTIC: - unfilled = any(member in with_placeholders for member in entry.get("members") or []) - entry["outcome"] = OUTCOME_AGENTIC_NOT_VIABLE if unfilled else OUTCOME_AGENTIC_APPLIED - - -def load_gaps(work_dir: Path) -> list[dict[str, Any]]: - """The gaps recorded beside a translation report in ``work_dir``, or an empty list when none were.""" - gaps_path = Path(work_dir) / GAPS_FILENAME - gaps = json.loads(gaps_path.read_text(encoding="utf-8")) if gaps_path.exists() else [] - return [gap for gap in gaps if isinstance(gap, dict)] if isinstance(gaps, list) else [] - - -def _routed_task_names(gaps: list[dict[str, Any]]) -> dict[str, set[str]]: - """The task names route turned into placeholders, per pipeline, from its pipeline-tagged gaps.""" - names: dict[str, set[str]] = {} - for gap in gaps: - if gap.get("pipeline") is not None: - names.setdefault(str(gap["pipeline"]), set()).add(str(gap.get("activity_name"))) - return names - +def routed_agentic_pipelines(report: Any) -> set[str]: + """The pipelines only combine may fill: routed-agentic members and the pipelines combine authored. -def component_fill(report: dict[str, Any], members: Iterable[str], gaps: list[dict[str, Any]]) -> str | None: - """Which kind of fill has touched a routed-agentic component, read from the report and its gaps. - - Route turns every top-level task of a member into a placeholder and records one gap per task. A - per-pipeline merge replaces those placeholders by name, so a merged member keeps the routed task - names. A combine replaces the members with authored pipelines, so a member is combined when it - is gone from the report, or when an authored pipeline reusing its name holds no placeholder and - none of the routed task names. Returns :data:`FILL_COMBINE`, :data:`FILL_MERGE`, or ``None`` - when neither has run. - """ - by_name = {pipeline.get("name"): pipeline for pipeline in _report_pipelines(report)} - routed = _routed_task_names(gaps) - filled = False - for member in members: - pipeline = by_name.get(member) - if pipeline is None: - return FILL_COMBINE - tasks = [task for task in pipeline.get("tasks", []) if isinstance(task, dict)] - placeholders = [task for task in tasks if task.get("type") == "PlaceholderActivity"] - if tasks and not placeholders and not {str(task.get("name")) for task in tasks} & routed.get(member, set()): - return FILL_COMBINE - filled = filled or len(placeholders) < len(tasks) - return FILL_MERGE if filled else None - - -def combined_pipelines(report: dict[str, Any], gaps: list[dict[str, Any]]) -> set[str]: - """The pipelines a combine has filled: combined members, and the authored pipelines beside them. - - A pipeline that is no member of any routed component was authored by a combine, so once any - component is combined those pipelines count too. Empty when the report has no routing record. + Every pipeline convert wrote is a member of some recorded component, so a pipeline in the report + that belongs to none was authored by a combine. Empty when the report has no routing record. """ record = routing_record(report) components = record.get("components") if record is not None else None @@ -251,16 +186,13 @@ def combined_pipelines(report: dict[str, Any], gaps: list[dict[str, Any]]) -> se names: set[str] = set() all_members: set[str] = set() for entry in components.values(): - if not isinstance(entry, dict): - continue - members = [str(member) for member in entry.get("members") or []] - all_members.update(members) - if entry.get("decision") == DECISION_AGENTIC and component_fill(report, members, gaps) == FILL_COMBINE: - names.update(members) - if names: - authored = {str(pipeline.get("name")) for pipeline in _report_pipelines(report)} - all_members - names.update(authored) - return names + if isinstance(entry, dict): + members = {str(member) for member in entry.get("members") or []} + all_members.update(members) + if entry.get("decision") == DECISION_AGENTIC: + names.update(members) + authored = {str(pipeline.get("name")) for pipeline in _report_pipelines(report)} - all_members + return names | authored def reroute_conflict(output_dir: Path, plan: dict[str, Any]) -> str | None: @@ -496,7 +428,10 @@ def apply_plan_to_report(output_dir: Path, plan: dict[str, Any]) -> dict[str, An if record is None and not agentic: return {"agentic_pipelines": [], "gaps": 0, "altered": False} - gaps = load_gaps(work) + gaps_path = work / GAPS_FILENAME + gaps = json.loads(gaps_path.read_text(encoding="utf-8")) if gaps_path.exists() else [] + if not isinstance(gaps, list): + gaps = [] if record is not None: if routing_record_mismatches(record, plan): @@ -510,7 +445,7 @@ def apply_plan_to_report(output_dir: Path, plan: dict[str, Any]) -> dict[str, An new_report = {"pipelines": [new_report]} new_report[ROUTING_RECORD_KEY] = build_routing_record(plan, baseline_bytes) _write_json_atomic(report_path, new_report) - _write_json_atomic(work / GAPS_FILENAME, new_gaps) + _write_json_atomic(gaps_path, new_gaps) return {"agentic_pipelines": sorted(agentic), "gaps": len(new_gaps), "altered": True} @@ -563,35 +498,39 @@ def combine_group_fill( def _resolve_agentic_component( output_dir: Path, members: set[str] -) -> tuple[str | None, dict[str, Any] | None, str | None]: +) -> tuple[str | None, dict[str, Any] | None, str | None, str | None]: """Bind ``members`` to a routed-**agentic** component in the recorded, fingerprint-bound plan. Reads ``metadata/conversion_plan.json`` and ``metadata/inventory.json`` and returns - ``(component_id, plan, error)``, where ``plan`` is the recorded plan document. The error is set - (and the other two are ``None``) when: the plan or inventory is missing; the plan's + ``(component_id, plan, source, error)``, where ``plan`` is the recorded plan document and + ``source`` the inventory's source, which every authored pipeline must carry. The error is set + (and the other three are ``None``) when: the inventory's source cannot be routed agentic + (:data:`flowx.routing.AGENTIC_ROUTING_SOURCES`); the plan or inventory is missing; the plan's ``inventory_sha256`` no longer matches the current inventory (a stale plan); ``members`` do not exactly equal one component's members (a partial, superset, or mistyped group); or the exactly-matching component is routed deterministic rather than agentic. Requiring an exact match to a routed-agentic component stops a caller from swapping deterministic pipelines or a partial group. The plan is returned so the combine can check the report's routing record against it. """ - from flowx.routing import plan_binding_violations + from flowx.routing import agentic_routing_supported, inventory_source, plan_binding_violations metadata = Path(output_dir) / METADATA_DIRNAME inventory_path = metadata / INVENTORY_FILENAME try: recorded = ConversionPlan.load(Path(output_dir)) except ValueError as error: - return None, None, str(error) + return None, None, None, str(error) if recorded is None: - return None, None, "No metadata/conversion_plan.json; record a routing decision with `route` first." + return None, None, None, "No metadata/conversion_plan.json; record a routing decision with `route` first." if not inventory_path.exists(): - return None, None, "No metadata/inventory.json; run the discover phase first." + return None, None, None, "No metadata/inventory.json; run the discover phase first." inventory = json.loads(inventory_path.read_text(encoding="utf-8")) + if not agentic_routing_supported(inventory): + return None, None, None, f"inventory source {inventory_source(inventory)!r} cannot be routed agentic." stale = plan_binding_violations(recorded, inventory) if stale: - return None, None, f"conversion_plan.json is stale: {stale[0]}; re-run `route` before filling." + return None, None, None, f"conversion_plan.json is stale: {stale[0]}; re-run `route` before filling." plan = recorded.to_dict() for component in plan.get("components", []): @@ -601,8 +540,9 @@ def _resolve_agentic_component( if component_members != members: continue if component.get("decision") == DECISION_AGENTIC: - return str(component.get("component_id")), plan, None + return str(component.get("component_id")), plan, inventory_source(inventory), None return ( + None, None, None, ( @@ -611,6 +551,7 @@ def _resolve_agentic_component( ), ) return ( + None, None, None, ( @@ -620,11 +561,11 @@ def _resolve_agentic_component( ) -def _authored_source_tag_violations(authored_pipelines: list[dict[str, Any]]) -> list[str]: - """Reports each authored combine pipeline that is missing the required ``tags.source == 'adf'``. +def _authored_source_tag_violations(authored_pipelines: list[dict[str, Any]], source: str) -> list[str]: + """Reports each authored combine pipeline that does not carry ``tags.source`` equal to ``source``. - Routing / in-engine agentic conversion is ADF-only, so an authored combine pipeline that omits or - mis-sets the source tag is an authoring error. Catching it here fails the combine closed (nothing + The authored pipelines replace pipelines of the routed inventory, so one that omits or mis-sets + the source tag is an authoring error. Catching it here fails the combine closed (nothing written) with a clear message, rather than letting the mis-tagged pipeline reach the package preflight where it is only rejected much later. """ @@ -632,11 +573,10 @@ def _authored_source_tag_violations(authored_pipelines: list[dict[str, Any]]) -> for index, pipeline in enumerate(authored_pipelines): label = pipeline.get("name") if isinstance(pipeline, dict) and pipeline.get("name") else f"pipeline[{index}]" tags = pipeline.get("tags") if isinstance(pipeline, dict) else None - source = tags.get("source") if isinstance(tags, dict) else None - if source != REQUIRED_COMBINE_SOURCE_TAG: + authored_source = tags.get("source") if isinstance(tags, dict) else None + if authored_source != source: violations.append( - f"{label}: authored combine pipeline must carry tags.source == " - f"{REQUIRED_COMBINE_SOURCE_TAG!r}, got {source!r}" + f"{label}: authored combine pipeline must carry tags.source == {source!r}, got {authored_source!r}" ) return violations @@ -652,19 +592,19 @@ def apply_combine_fill( routed-agentic component in ``metadata/conversion_plan.json`` (whose fingerprint must still match the current inventory), so a caller cannot swap deterministic pipelines or a partial/typoed group. The report must carry a routing record that matches that plan, so route has to have applied it. - Every authored pipeline must carry ``tags.source == 'adf'`` (routing/agentic is ADF-only); a - mis-tagged pipeline fails the combine closed here rather than surviving to the package preflight. + Every authored pipeline must carry the inventory's source as ``tags.source`` (only sources in + :data:`flowx.routing.AGENTIC_ROUTING_SOURCES` route agentic, ADF today); a mis-tagged pipeline + fails the combine closed here rather than surviving to the package preflight. The merged report is then **always** validated with the structural bundle invariants (a real ``prepare -> write_bundle`` pass over :func:`validate_report_structurally`) -- there is no bypass -- and written back only when it passes, so a dangling reference or duplicate key never lands on disk. - Each component takes one kind of fill: a component a per-pipeline merge has already filled is - refused with :data:`COMPONENT_ALREADY_FILLED`. The combine is **idempotent**: once a combine has - replaced the component's members (see :func:`component_fill`) a re-run no-ops (``already_combined`` true) - instead of collapsing/appending again, so it never duplicates the authored pipeline(s) even when - the authored replacement is renamed or an authored name collides with a former member. A fresh - ``convert`` rewrites the report without the record, so the combine applies again after a - re-convert and route. + Combine is the only fill for a routed-agentic component and marks its outcome + ``agentic-applied``. It is **idempotent**: on an applied component, re-running it with the same + authored pipelines as those now in the report no-ops (``already_combined`` true), even when the + authored names reuse member names; different authored pipelines are refused with + :data:`COMPONENT_ALREADY_FILLED`. A fresh ``convert`` rewrites the report without the record, so + the combine applies again after a re-convert and route. Returns ``{"ok", "violations", "error", "component_id", "pipelines", "already_combined"}``. ``ok`` is ``False`` (and nothing written) on a plan/membership error (``error`` set), a missing source tag, @@ -674,12 +614,12 @@ def apply_combine_fill( FileNotFoundError: when the translation report is missing (run convert first). """ members = {str(member) for member in group_members} - component_id, plan, error = _resolve_agentic_component(output_dir, members) + component_id, plan, source, error = _resolve_agentic_component(output_dir, members) if error is not None: return {"ok": False, "error": error, "violations": [], "pipelines": 0} - assert component_id is not None and plan is not None # guaranteed when error is None + assert component_id is not None and plan is not None and source is not None # guaranteed when error is None - tag_violations = _authored_source_tag_violations(authored_pipelines) + tag_violations = _authored_source_tag_violations(authored_pipelines, source) if tag_violations: return {"ok": False, "error": None, "violations": tag_violations, "pipelines": 0} @@ -696,10 +636,10 @@ def apply_combine_fill( if routing_record_mismatches(record, plan): return {"ok": False, "error": REROUTED_UNDER_DIFFERENT_PLAN, "violations": [], "pipelines": 0} - fill = component_fill(report, members, load_gaps(work)) - if fill == FILL_MERGE: - return {"ok": False, "error": COMPONENT_ALREADY_FILLED, "violations": [], "pipelines": 0} - if fill == FILL_COMBINE: + if record["components"][component_id].get("outcome") == OUTCOME_AGENTIC_APPLIED: + in_report = {pipeline.get("name"): pipeline for pipeline in _report_pipelines(report)} + if not all(in_report.get(pipeline.get("name")) == pipeline for pipeline in authored_pipelines): + return {"ok": False, "error": COMPONENT_ALREADY_FILLED, "violations": [], "pipelines": 0} return { "ok": True, "error": None, @@ -711,7 +651,7 @@ def apply_combine_fill( merged = combine_group_fill(report, members, authored_pipelines) merged[ROUTING_RECORD_KEY] = copy.deepcopy(record) - refresh_routing_outcomes(merged) + merged[ROUTING_RECORD_KEY]["components"][component_id]["outcome"] = OUTCOME_AGENTIC_APPLIED result = validate_report_structurally(merged) if not result.ok: diff --git a/src/flowx/routing.py b/src/flowx/routing.py index 6cbae5d0..a2b050c6 100644 --- a/src/flowx/routing.py +++ b/src/flowx/routing.py @@ -67,9 +67,14 @@ _COMPONENT_LIBRARY_KEYS = {"recommended", "options"} +def inventory_source(inventory: dict[str, Any]) -> str: + """The source an inventory was discovered from (an inventory without one is ADF).""" + return str(inventory.get("source", _LEGACY_INVENTORY_SOURCE)) + + def agentic_routing_supported(inventory: dict[str, Any]) -> bool: """Whether route may take an agentic decision for this inventory's source (ADF only in Phase 1).""" - return inventory.get("source", _LEGACY_INVENTORY_SOURCE) in AGENTIC_ROUTING_SOURCES + return inventory_source(inventory) in AGENTIC_ROUTING_SOURCES def _agentic_routing_unsupported_note(inventory: dict[str, Any]) -> str: diff --git a/tests/unit/test_route_agentic_flow.py b/tests/unit/test_route_agentic_flow.py index 0216d684..e665b796 100644 --- a/tests/unit/test_route_agentic_flow.py +++ b/tests/unit/test_route_agentic_flow.py @@ -23,6 +23,7 @@ import copy import inspect import json +import re from pathlib import Path from typing import Any @@ -38,6 +39,7 @@ GAPS_FILENAME, REPORT_FILENAME, REROUTED_UNDER_DIFFERENT_PLAN, + ROUTED_AGENTIC_MERGE_REFUSED, WORK_DIRNAME, agentic_pipeline_names, alter_report, @@ -393,38 +395,46 @@ def test_apply_plan_to_report_all_deterministic_leaves_files_byte_identical(tmp_ # --------------------------------------------------------------------------- # -def test_per_pipeline_fill_reuses_merge_agentic_results(tmp_path: Path) -> None: +@pytest.mark.parametrize("pipeline", ["parent", None]) +def test_merge_into_a_routed_agentic_pipeline_is_refused(tmp_path: Path, pipeline: str | None) -> None: _write_work(tmp_path, _report_two_pipelines(), gaps=[]) apply_plan_to_report(tmp_path, _plan(parent="agentic", child="deterministic")) report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME + routed_report = report_path.read_bytes() + results_dir = tmp_path / "agentic_results" + results_dir.mkdir() + result: dict[str, Any] = {"activity_name": "Extract", "task": _notebook_task("Extract", "extract")} + if pipeline is not None: + result["pipeline"] = pipeline + (results_dir / "extract.json").write_text(json.dumps(result), encoding="utf-8") + + with pytest.raises(ValueError, match=re.escape(ROUTED_AGENTIC_MERGE_REFUSED)): + merge_agentic_results(report_path, results_dir) + assert report_path.read_bytes() == routed_report + +@pytest.mark.parametrize("pipeline", ["child", None]) +def test_merge_still_fills_convert_gaps_in_a_deterministic_pipeline(tmp_path: Path, pipeline: str | None) -> None: + report = _report_two_pipelines() + report["pipelines"][1]["tasks"] = [ + {"name": "Load", "task_key": "load", "type": "PlaceholderActivity", "original_type": "Script"} + ] + _write_work(tmp_path, report, gaps=[]) + apply_plan_to_report(tmp_path, _plan(parent="agentic", child="deterministic")) + report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME results_dir = tmp_path / "agentic_results" results_dir.mkdir() - (results_dir / "extract.json").write_text( - json.dumps( - { - "pipeline": "parent", - "activity_name": "Extract", - "task": { - "type": "NotebookActivity", - "name": "Extract", - "task_key": "extract", - "notebook_path": "/Workspace/Shared/agentic_extract", - }, - } - ), - encoding="utf-8", - ) - assert _record_outcomes(report_path) == {"component-1": "agentic-not-viable", "component-2": "deterministic"} - merged, unmatched = merge_agentic_results(report_path, results_dir) - assert (merged, unmatched) == (1, 0) + result: dict[str, Any] = {"activity_name": "Load", "task": _notebook_task("Load", "load", "/Workspace/Shared/l")} + if pipeline is not None: + result["pipeline"] = pipeline + (results_dir / "load.json").write_text(json.dumps(result), encoding="utf-8") - report = json.loads(report_path.read_text(encoding="utf-8")) - parent = next(p for p in report["pipelines"] if p["name"] == "parent") - assert parent["tasks"][0]["type"] == "NotebookActivity" - assert validate_report_structurally(report).ok - # The merge left no routed placeholder in component-1, so its outcome is now applied. - assert _record_outcomes(report_path) == {"component-1": "agentic-applied", "component-2": "deterministic"} + assert merge_agentic_results(report_path, results_dir) == (1, 0) + + merged = json.loads(report_path.read_text(encoding="utf-8")) + child = next(p for p in merged["pipelines"] if p["name"] == "child") + assert child["tasks"][0]["type"] == "NotebookActivity" + assert _record_outcomes(report_path) == {"component-1": "agentic-not-viable", "component-2": "deterministic"} # --------------------------------------------------------------------------- # @@ -596,26 +606,19 @@ def test_combine_is_idempotent_running_twice_yields_no_duplicate(tmp_path: Path) assert names == ["orders_lfc"] # exactly one authored pipeline, no duplicate -def test_combine_idempotent_with_a_differently_named_authored_pipeline(tmp_path: Path) -> None: - """FIX 3 (a): a second combine whose authored replacement is renamed must NOT duplicate. - - Name-set matching would fail here (the new name is absent from the report, the old members are - already gone), fall through, and append a second authored pipeline. The routing record's outcome - catches the re-run regardless of the authored name. - """ +def test_a_different_combine_on_an_applied_component_is_refused(tmp_path: Path) -> None: + """A second combine whose authored pipelines differ from those now in the report is refused.""" _setup_routed_agentic(tmp_path, decision="agentic") - first = apply_combine_fill(tmp_path, ["parent", "child"], [_named_lfc_pipeline("orders_lfc")]) assert first["ok"] is True and first["already_combined"] is False + report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME + combined_report = report_path.read_bytes() - # Same members, but the authored replacement is named differently this time. second = apply_combine_fill(tmp_path, ["parent", "child"], [_named_lfc_pipeline("orders_lfc_v2")]) - assert second["ok"] is True - assert second["already_combined"] is True - report = json.loads((tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_text(encoding="utf-8")) - names = [pipeline["name"] for pipeline in report["pipelines"]] - assert names == ["orders_lfc"] # first authored pipeline kept; the renamed re-run added nothing + assert second["ok"] is False + assert second["error"] == COMPONENT_ALREADY_FILLED + assert report_path.read_bytes() == combined_report def test_combine_idempotent_when_authored_name_collides_with_a_former_member(tmp_path: Path) -> None: @@ -712,21 +715,6 @@ def _write_merge_result(results_dir: Path, pipeline: str, activity_name: str, ta ) -def test_combine_after_a_per_pipeline_merge_is_refused(tmp_path: Path) -> None: - _setup_routed_agentic(tmp_path, decision="agentic") - report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME - results_dir = tmp_path / "agentic_results" - _write_merge_result(results_dir, "parent", "Extract", "extract") - assert merge_agentic_results(report_path, results_dir) == (1, 0) - merged_report = report_path.read_bytes() - - result = apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()]) - - assert result["ok"] is False - assert result["error"] == COMPONENT_ALREADY_FILLED - assert report_path.read_bytes() == merged_report - - def test_merge_after_a_combine_is_refused(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: _setup_routed_agentic(tmp_path, decision="agentic") assert apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()])["ok"] is True @@ -735,19 +723,21 @@ def test_merge_after_a_combine_is_refused(tmp_path: Path, capsys: pytest.Capture results_dir = tmp_path / "agentic_results" _write_merge_result(results_dir, "parent", "Extract", "extract") - with pytest.raises(ValueError, match=COMPONENT_ALREADY_FILLED): + with pytest.raises(ValueError, match=re.escape(ROUTED_AGENTIC_MERGE_REFUSED)): merge_agentic_results(report_path, results_dir) exit_code = translate_main(["--merge-agentic", "--report", str(report_path), "--agentic-results", str(results_dir)]) assert exit_code == 1 - assert COMPONENT_ALREADY_FILLED in capsys.readouterr().err + assert ROUTED_AGENTIC_MERGE_REFUSED in capsys.readouterr().err assert report_path.read_bytes() == combined_report def _colliding_authored_pipelines() -> list[dict[str, Any]]: - """Authored combine pipelines that reuse both member names but none of the routed task names.""" - publish = {"name": "child", "tags": {"source": "adf"}, "tasks": [_notebook_task("Publish", "publish")]} - return [_named_lfc_pipeline("parent"), publish] + """Authored combine pipelines that reuse both member names and the routed task names.""" + parent = _named_lfc_pipeline("parent") + parent["tasks"][0]["name"] = "Extract" + child = {"name": "child", "tags": {"source": "adf"}, "tasks": [_notebook_task("Load", "load")]} + return [parent, child] def test_combine_rerun_is_idempotent_when_authored_names_reuse_every_member(tmp_path: Path) -> None: @@ -765,7 +755,7 @@ def test_combine_rerun_is_idempotent_when_authored_names_reuse_every_member(tmp_ @pytest.mark.parametrize( ("pipeline", "activity_name"), - [("parent", "Ingest orders"), (None, "Publish"), (None, "Extract")], + [("parent", "Extract"), ("child", "Load"), (None, "Load")], ) def test_merge_after_a_combine_reusing_every_member_name_is_refused( tmp_path: Path, pipeline: str | None, activity_name: str @@ -781,7 +771,7 @@ def test_merge_after_a_combine_reusing_every_member_name_is_refused( result["pipeline"] = pipeline (results_dir / "result.json").write_text(json.dumps(result), encoding="utf-8") - with pytest.raises(ValueError, match=COMPONENT_ALREADY_FILLED): + with pytest.raises(ValueError, match=re.escape(ROUTED_AGENTIC_MERGE_REFUSED)): merge_agentic_results(report_path, results_dir) assert report_path.read_bytes() == combined_report @@ -792,7 +782,7 @@ def test_merge_into_an_authored_combine_pipeline_is_refused(tmp_path: Path) -> N results_dir = tmp_path / "agentic_results" _write_merge_result(results_dir, "orders_lfc", "Ingest orders", "ingest_orders") - with pytest.raises(ValueError, match=COMPONENT_ALREADY_FILLED): + with pytest.raises(ValueError, match=re.escape(ROUTED_AGENTIC_MERGE_REFUSED)): merge_agentic_results(tmp_path / WORK_DIRNAME / REPORT_FILENAME, results_dir) From a37c0ca143ce8e90f5caa2b2fb8ba6d658b91b53 Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Wed, 7 Oct 2026 11:46:09 +0100 Subject: [PATCH 022/122] Implement replay model for routing: immutable baseline, pure rebuild, declarative combines Implements the phase-1 replay model as specified in pr97-replay-spec.md: - Add immutable baseline support: .work/route_baseline/ stores byte-for-byte copies of the initial translation_report.json and gaps.json for fresh routes with agentic components - Implement pure rebuild function: deterministic, idempotent transformation from baseline + plan + combines + previous_record to produce new report + gaps + record - Add declarative combines store: metadata/agentic_combines.json tracks authored pipelines per component with content hashing for idempotency (same combine is no-op, different combine replaces) - Update routing record: now includes baseline hashes, per-component fingerprints and combine hashes, and replacement tracking for re-routing changes - Allow re-routing with different decisions: no longer refuse, instead rebuild from baseline - Clear routing record when switching to all-deterministic: remove record from report on convert back - Update tests to reflect new behavior: combine replaces instead of refusing, re-routing is allowed Core functions: - rebuild(): pure function that takes baseline, plan, combines, previous_record and produces new report, gaps, and record deterministically - _save_baseline(), _load_baseline(): manage immutable baseline files - _load_combines(), _save_combines(): manage declarative combines store - apply_plan_to_report(): refactored to use rebuild, save baseline on first agentic route - apply_combine_fill(): stores combines, calls rebuild to apply them Remaining work: - Package preflight verification of baseline and replay - Test coverage for new replay semantics - Merge-agentic support for updating baseline after routing Co-authored-by: Isaac --- src/flowx/route_agentic.py | 426 +++++++++++++++++++------- tests/unit/test_route_agentic_flow.py | 32 +- 2 files changed, 334 insertions(+), 124 deletions(-) diff --git a/src/flowx/route_agentic.py b/src/flowx/route_agentic.py index ce8d2497..85df7933 100644 --- a/src/flowx/route_agentic.py +++ b/src/flowx/route_agentic.py @@ -4,32 +4,35 @@ **unchanged** by this module. Routing then decides, per connected component, whether each part converts deterministically or agentically (:mod:`flowx.routing` computes the recommendation and records the fingerprint-bound ``metadata/conversion_plan.json``). This module carries out the -**post-convert alteration and fill** the decision implies, always keeping the work in-engine so the -package phase, structural validation, and provenance all still apply: - -* :func:`alter_report` rewrites the report for the routed-**agentic** groups only: every task in an - agentic-routed pipeline is removed and replaced by a :class:`~flowx.models.ir.PlaceholderActivity`, - and exactly one pipeline-tagged :class:`~flowx.models.ir.AgenticGap` is emitted per routed task, - replacing (never appending to) any prior gap for those pipelines' tasks, so the standard gap-fill - path handles them without duplicates and the edit is idempotent. Pipelines in deterministic groups - are left byte-identical, and when nothing is routed agentic the report and gaps are returned - unchanged -- the non-breaking guarantee. -* the agent authors the fill. A routed-agentic component is filled **only** by the cross-pipeline +**post-convert alteration and fill** the decision implies, using an immutable-baseline and pure-rebuild +pattern similar to :mod:`flowx.agentic`, always keeping the work in-engine so the package phase, +structural validation, and provenance all still apply: + +* When route runs with at least one agentic component on a fresh convert (no routing record), it saves + an immutable copy of ``.work/translation_report.json`` and ``.work/gaps.json`` to + ``.work/route_baseline/``, records their hashes, and never modifies them. Later routes verify and + reuse the baseline. Combines are stored in ``metadata/agentic_combines.json`` and are idempotent: + the same combine is a no-op, a different combine replaces the stored entry. +* :func:`rebuild` is a pure function that takes the immutable baseline, the routing plan, and stored + combines, and produces edited report + gaps + routing record. It implements both re-routing and + re-applying combines deterministically: every rebuild call with the same inputs produces identical + bytes. The routing record tracks each component's outcome (deterministic, agentic-applied, or + agentic-not-viable) and fingerprint changes. +* The agent authors the fill. A routed-agentic component is filled **only** by the cross-pipeline COMBINE (N pipelines -> M, e.g. one Lakeflow Connect pipeline, or one same-named pipeline to keep - it 1:1): :func:`combine_group_fill` swaps the routed group's pipelines for the agent-authored - pipeline(s), which carry :class:`~flowx.models.ir.AgenticComponentActivity` nodes (the escape - hatch). The name-matched :func:`flowx.ir_serde.merge_agentic_results` stays for convert's own - gaps and refuses routed-agentic pipelines. + it 1:1): stored in ``metadata/agentic_combines.json`` and applied via rebuild. The name-matched + :func:`flowx.ir_serde.merge_agentic_results` stays for convert's own gaps (after routing, merges + update both the live report and the stored baseline so the next rebuild keeps them) and refuses + routed-agentic pipelines. * :func:`validate_report_structurally` packages the merged report through the same ``prepare -> write_bundle`` path the package phase uses and runs the existing :func:`flowx.validate.bundle_invariants.check_bundle_dir` over the output, so a fill can never introduce duplicate keys, dangling dependencies, a cycle, or a dangling pipeline/run_job reference. Route stamps one routing record onto the report (:data:`ROUTING_RECORD_KEY`), keyed by the plan's -component ids, holding each component's members, decision and outcome plus the plan hash and the hash -of the report route started from. Every later rewrite of the report carries it forward, combine -marks its component applied, and package refuses a report whose record no longer matches the -recorded plan. +component ids, holding each component's members, decision, outcome, and a fingerprint. The routing +record also tracks the baseline hashes and combine hashes. Every rebuild carries it forward with +updated outcomes and fingerprints. There is no LLM here: the tool edits the report and validates the authored fill; the fill itself is supplied by the agent/harness, following the same ask -> author -> continue pattern as the existing @@ -57,9 +60,14 @@ # The configuration-stamped copy modify writes beside the report; package reads it when present. STAMPED_REPORT_FILENAME = "translation_report.stamped.json" GAPS_FILENAME = "gaps.json" +# The immutable baseline copies, saved on first route with agentic components. +BASELINE_DIRNAME = "route_baseline" +BASELINE_REPORT_FILENAME = "translation_report.json" +BASELINE_GAPS_FILENAME = "gaps.json" # The recorded plan + inventory the combine fill binds against live under metadata/. METADATA_DIRNAME = "metadata" INVENTORY_FILENAME = "inventory.json" +COMBINES_FILENAME = "agentic_combines.json" # Top-level report key holding the routing record. It lives in the report, so a fresh convert (which # rewrites the report) clears it; the package phase and ir_serde read only the pipelines beside it. @@ -71,6 +79,7 @@ OUTCOME_AGENTIC_APPLIED = "agentic-applied" OUTCOME_AGENTIC_NOT_VIABLE = "agentic-not-viable" +# Legacy: kept for backward compatibility with tests, but no longer used by route. REROUTED_UNDER_DIFFERENT_PLAN = "the report was routed under a different plan; re-run convert, then route" COMPONENT_ALREADY_FILLED = "component already filled; re-run convert and route to start again" @@ -121,18 +130,39 @@ def _planned_components(plan: dict[str, Any]) -> dict[str, dict[str, Any]]: } -def build_routing_record(plan: dict[str, Any], baseline_bytes: bytes) -> dict[str, Any]: - """The record route stamps onto the report when it first applies a plan with an agentic component.""" - components = { - component_id: { +def build_routing_record( + plan: dict[str, Any], + baseline_report_bytes: bytes, + baseline_gaps_bytes: bytes, + combines: dict[str, dict[str, Any]], +) -> dict[str, Any]: + """Build a routing record for a plan applied to a fresh convert. + + Records the plan hash, baseline file hashes, and per-component information (members, decision, + outcome, combine hash, and fingerprint). + """ + planned = _planned_components(plan) + components: dict[str, dict[str, Any]] = {} + for component_id, entry in planned.items(): + combine_data = combines.get(component_id) + combine_sha256 = combine_data.get("combine_sha256") if combine_data else None + decision = entry["decision"] + outcome = OUTCOME_AGENTIC_NOT_VIABLE if decision == DECISION_AGENTIC else OUTCOME_DETERMINISTIC + fingerprint_input = { + "members": entry["members"], + "decision": decision, + "combine_sha256": combine_sha256, + } + components[component_id] = { **entry, - "outcome": OUTCOME_AGENTIC_NOT_VIABLE if entry["decision"] == DECISION_AGENTIC else OUTCOME_DETERMINISTIC, + "outcome": outcome, + "combine_sha256": combine_sha256, + "fingerprint": canonical_sha256(fingerprint_input), } - for component_id, entry in _planned_components(plan).items() - } return { "conversion_plan_sha256": canonical_sha256(plan), - "baseline_report_sha256": hashlib.sha256(baseline_bytes).hexdigest(), + "baseline_report_sha256": hashlib.sha256(baseline_report_bytes).hexdigest(), + "baseline_gaps_sha256": hashlib.sha256(baseline_gaps_bytes).hexdigest(), "components": components, } @@ -195,19 +225,71 @@ def routed_agentic_pipelines(report: Any) -> set[str]: return names | authored -def reroute_conflict(output_dir: Path, plan: dict[str, Any]) -> str | None: - """Explain why route must not apply ``plan`` to the report on disk, or ``None`` when it may. +# --------------------------------------------------------------------------- # +# Baseline management and combines store. +# --------------------------------------------------------------------------- # + + +def _baseline_exists(output_dir: Path) -> bool: + """Check whether a baseline has been saved.""" + baseline_dir = Path(output_dir) / WORK_DIRNAME / BASELINE_DIRNAME + return (baseline_dir / BASELINE_REPORT_FILENAME).exists() or (baseline_dir / BASELINE_GAPS_FILENAME).exists() + + +def _load_baseline(output_dir: Path) -> tuple[dict[str, Any], list[dict[str, Any]], str | None]: + """Load the immutable baseline report and gaps, creating it from current live ones if needed. - Route refuses once the report carries a record routed under different components or decisions: - the earlier edit cannot be undone in place, so the report has to be converted again first. + Returns ``(report, gaps, error)`` where error is set when trying to load an existing baseline + and it fails to read. """ - report_path = Path(output_dir) / WORK_DIRNAME / REPORT_FILENAME - if not report_path.exists(): + baseline_dir = Path(output_dir) / WORK_DIRNAME / BASELINE_DIRNAME + baseline_report_path = baseline_dir / BASELINE_REPORT_FILENAME + baseline_gaps_path = baseline_dir / BASELINE_GAPS_FILENAME + if baseline_report_path.exists() or baseline_gaps_path.exists(): + try: + report = ( + json.loads(baseline_report_path.read_text(encoding="utf-8")) if baseline_report_path.exists() else {} + ) + gaps = json.loads(baseline_gaps_path.read_text(encoding="utf-8")) if baseline_gaps_path.exists() else [] + return report, gaps if isinstance(gaps, list) else [], None + except (OSError, json.JSONDecodeError) as error: + return {}, [], str(error) + work_dir = Path(output_dir) / WORK_DIRNAME + live_report_path = work_dir / REPORT_FILENAME + live_gaps_path = work_dir / GAPS_FILENAME + try: + report = json.loads(live_report_path.read_text(encoding="utf-8")) if live_report_path.exists() else {} + gaps = json.loads(live_gaps_path.read_text(encoding="utf-8")) if live_gaps_path.exists() else [] + return report, gaps if isinstance(gaps, list) else [], None + except (OSError, json.JSONDecodeError): + return {}, [], None + + +def _save_baseline(output_dir: Path, report: dict[str, Any], gaps: list[dict[str, Any]]) -> None: + """Save an immutable copy of the report and gaps to the baseline folder.""" + baseline_dir = Path(output_dir) / WORK_DIRNAME / BASELINE_DIRNAME + baseline_dir.mkdir(parents=True, exist_ok=True) + _write_json_atomic(baseline_dir / BASELINE_REPORT_FILENAME, report) + _write_json_atomic(baseline_dir / BASELINE_GAPS_FILENAME, gaps) + + +def _load_combines(output_dir: Path) -> dict[str, dict[str, Any]] | None: + """Load the agentic combines store, or ``None`` when it doesn't exist.""" + combines_path = Path(output_dir) / METADATA_DIRNAME / COMBINES_FILENAME + if not combines_path.exists(): + return None + try: + raw = json.loads(combines_path.read_text(encoding="utf-8")) + return raw if isinstance(raw, dict) else None + except (OSError, json.JSONDecodeError): return None - record = routing_record(json.loads(report_path.read_text(encoding="utf-8"))) - if record is not None and routing_record_mismatches(record, plan): - return REROUTED_UNDER_DIFFERENT_PLAN - return None + + +def _save_combines(output_dir: Path, combines: dict[str, dict[str, Any]]) -> None: + """Save the agentic combines store.""" + combines_path = Path(output_dir) / METADATA_DIRNAME / COMBINES_FILENAME + combines_path.parent.mkdir(parents=True, exist_ok=True) + _write_json_atomic(combines_path, combines) # --------------------------------------------------------------------------- # @@ -263,6 +345,91 @@ def prompt_for_decisions( return {"components": components_out} +# --------------------------------------------------------------------------- # +# The pure rebuild step: deterministic, idempotent report transformation. +# --------------------------------------------------------------------------- # + + +def rebuild( + baseline_report: dict[str, Any], + baseline_gaps: list[dict[str, Any]], + plan: dict[str, Any], + combines: dict[str, dict[str, Any]] | None, + previous_record: dict[str, Any] | None, +) -> tuple[dict[str, Any], list[dict[str, Any]], dict[str, Any]]: + """Pure rebuild: derive the routed report, gaps, and record from baseline and plan. + + Returns ``(report, gaps, record)``, all deterministically derived from inputs. For each component: + * **deterministic:** the baseline pipelines stay; outcome is ``deterministic``. + * **agentic with a stored combine whose members match:** replace members with combine's pipelines, + reuse the combine_sha256. Outcome is ``agentic-applied``. + * **agentic without a matching combine:** placeholders and tagged gaps as ``alter_report`` does. + Outcome is ``agentic-not-viable``. + + The record holds per-component members, decision, outcome, combine_sha256, and fingerprint. When + fingerprint differs from previous_record's, a replacement entry is appended to track the change. + An all-deterministic plan returns no record (byte-identical to baseline). + """ + combines_store = combines or {} + planned = _planned_components(plan) + agentic_names = agentic_pipeline_names(plan) + if not agentic_names and previous_record is None: + return baseline_report, baseline_gaps, {} + report = copy.deepcopy(baseline_report) + gaps = copy.deepcopy(baseline_gaps) + new_report, new_gaps = alter_report(report, gaps, agentic_names) + if not isinstance(new_report.get("pipelines"), list): + new_report = {"pipelines": [new_report]} + baseline_report_bytes = json.dumps(baseline_report, separators=(",", ":")).encode("utf-8") + baseline_gaps_bytes = json.dumps(baseline_gaps, separators=(",", ":")).encode("utf-8") + components: dict[str, dict[str, Any]] = {} + for component_id, planned_entry in planned.items(): + decision = planned_entry["decision"] + members_set = set(planned_entry["members"]) + combine_data = combines_store.get(component_id) + combine_sha256 = combine_data.get("combine_sha256") if combine_data else None + fingerprint_input = { + "members": planned_entry["members"], + "decision": decision, + "combine_sha256": combine_sha256, + } + fingerprint = canonical_sha256(fingerprint_input) + if decision == DECISION_AGENTIC and combine_data and set(combine_data.get("members", [])) == members_set: + outcome = OUTCOME_AGENTIC_APPLIED + new_report = combine_group_fill(new_report, members_set, combine_data.get("pipelines", [])) + elif decision == DECISION_AGENTIC: + outcome = OUTCOME_AGENTIC_NOT_VIABLE + else: + outcome = OUTCOME_DETERMINISTIC + entry: dict[str, Any] = { + **planned_entry, + "outcome": outcome, + "combine_sha256": combine_sha256, + "fingerprint": fingerprint, + } + if previous_record and component_id in previous_record.get("components", {}): + old_fingerprint = previous_record["components"][component_id].get("fingerprint") + if old_fingerprint and old_fingerprint != fingerprint: + old_record = previous_record["components"][component_id] + entry["replacements"] = (old_record.get("replacements") or []) + [ + {"from": old_fingerprint, "to": fingerprint} + ] + else: + entry["replacements"] = previous_record["components"][component_id].get("replacements") or [] + else: + entry["replacements"] = [] + components[component_id] = entry + if all(entry["decision"] != DECISION_AGENTIC for entry in components.values()): + return baseline_report, baseline_gaps, {} + record = { + "conversion_plan_sha256": canonical_sha256(plan), + "baseline_report_sha256": hashlib.sha256(baseline_report_bytes).hexdigest(), + "baseline_gaps_sha256": hashlib.sha256(baseline_gaps_bytes).hexdigest(), + "components": components, + } + return new_report, new_gaps, record + + # --------------------------------------------------------------------------- # # The edit step: rewrite the report for routed-agentic groups. # --------------------------------------------------------------------------- # @@ -401,52 +568,68 @@ def _refresh_record_plan_hash(report_path: Path, plan: dict[str, Any]) -> None: def apply_plan_to_report(output_dir: Path, plan: dict[str, Any]) -> dict[str, Any]: - """Apply a routing decision to the report on disk: placeholder the routed-agentic groups. + """Apply a routing decision to the report on disk by rebuilding from baseline. - Reads ``/.work/translation_report.json`` (and ``gaps.json`` when present), rewrites - them for the plan's agentic components, stamps the routing record and writes them back atomically. - When no component is routed agentic and the report carries no record the files are left - untouched, so the non-breaking guarantee holds. A report already routed under the same decisions - is not edited again; only the record's plan hash is brought up to date, in the report and in the - configuration-stamped copy package reads when ``modify`` wrote one. + Reads or creates the immutable baseline at ``.work/route_baseline/``, loads stored combines from + ``metadata/agentic_combines.json``, rebuilds the report + gaps + record deterministically, and + writes them atomically. When no component is routed agentic and no record exists, files are left + untouched (non-breaking guarantee). Re-routing with different decisions rebuilds (not refused). Returns a summary dict with the altered pipeline names and the resulting gap count. Raises: FileNotFoundError: when the translation report is missing (run convert first). - ValueError: when the report was routed under different components or decisions. """ work = Path(output_dir) / WORK_DIRNAME report_path = work / REPORT_FILENAME if not report_path.exists(): raise FileNotFoundError(f"No {REPORT_FILENAME} under {work}; run the convert phase first.") - agentic = agentic_pipeline_names(plan) - baseline_bytes = report_path.read_bytes() - report = json.loads(baseline_bytes) + baseline_report, baseline_gaps, baseline_error = _load_baseline(output_dir) + if baseline_error is not None: + raise ValueError(f"Failed to load baseline: {baseline_error}") + old_report_bytes = report_path.read_bytes() + report = json.loads(old_report_bytes) + old_gaps_bytes = (work / GAPS_FILENAME).read_bytes() if (work / GAPS_FILENAME).exists() else b"[]" record = routing_record(report) - if record is None and not agentic: - return {"agentic_pipelines": [], "gaps": 0, "altered": False} - - gaps_path = work / GAPS_FILENAME - gaps = json.loads(gaps_path.read_text(encoding="utf-8")) if gaps_path.exists() else [] - if not isinstance(gaps, list): - gaps = [] - - if record is not None: - if routing_record_mismatches(record, plan): - raise ValueError(REROUTED_UNDER_DIFFERENT_PLAN) - for path in (report_path, work / STAMPED_REPORT_FILENAME): - _refresh_record_plan_hash(path, plan) - return {"agentic_pipelines": sorted(agentic), "gaps": len(gaps), "altered": False} - - new_report, new_gaps = alter_report(report, gaps, agentic) - if not isinstance(new_report.get("pipelines"), list): - new_report = {"pipelines": [new_report]} - new_report[ROUTING_RECORD_KEY] = build_routing_record(plan, baseline_bytes) + combines = _load_combines(output_dir) or {} + agentic = agentic_pipeline_names(plan) + new_report, new_gaps, new_record = rebuild( + baseline_report, + baseline_gaps, + plan, + combines, + record, + ) + if not new_record: + if record is not None: + report_without_record = copy.deepcopy(report) + report_without_record.pop(ROUTING_RECORD_KEY, None) + _write_json_atomic(report_path, report_without_record) + _write_json_atomic(work / GAPS_FILENAME, baseline_gaps) + if (work / STAMPED_REPORT_FILENAME).exists(): + _write_json_atomic(work / STAMPED_REPORT_FILENAME, report_without_record) + return {"agentic_pipelines": sorted(agentic), "gaps": len(baseline_gaps), "altered": record is not None} + if not _baseline_exists(output_dir): + _save_baseline(output_dir, baseline_report, baseline_gaps) + new_report[ROUTING_RECORD_KEY] = new_record + old_report_without_record = copy.deepcopy(report) + old_report_without_record.pop(ROUTING_RECORD_KEY, None) + new_gaps_bytes = json.dumps(new_gaps, indent=2, default=str).encode("utf-8") + altered = ( + json.dumps(old_report_without_record, separators=(",", ":"), sort_keys=True, default=str) + != json.dumps( + {k: v for k, v in new_report.items() if k != ROUTING_RECORD_KEY}, + separators=(",", ":"), + sort_keys=True, + default=str, + ) + ) or (old_gaps_bytes != new_gaps_bytes) _write_json_atomic(report_path, new_report) - _write_json_atomic(gaps_path, new_gaps) - return {"agentic_pipelines": sorted(agentic), "gaps": len(new_gaps), "altered": True} + _write_json_atomic(work / GAPS_FILENAME, new_gaps) + if (work / STAMPED_REPORT_FILENAME).exists(): + _write_json_atomic(work / STAMPED_REPORT_FILENAME, new_report) + return {"agentic_pipelines": sorted(agentic), "gaps": len(new_gaps), "altered": altered} def apply_plan(output_dir: Path, plan: ConversionPlan) -> dict[str, Any]: @@ -588,36 +771,25 @@ def apply_combine_fill( ) -> dict[str, Any]: """Combine a routed-agentic group into agent-authored pipeline(s) on disk. - The group's membership is **bound to the recorded plan**: ``group_members`` must exactly match a - routed-agentic component in ``metadata/conversion_plan.json`` (whose fingerprint must still match - the current inventory), so a caller cannot swap deterministic pipelines or a partial/typoed group. - The report must carry a routing record that matches that plan, so route has to have applied it. - Every authored pipeline must carry the inventory's source as ``tags.source`` (only sources in - :data:`flowx.routing.AGENTIC_ROUTING_SOURCES` route agentic, ADF today); a mis-tagged pipeline - fails the combine closed here rather than surviving to the package preflight. - The merged report is then **always** validated with the structural bundle invariants (a real - ``prepare -> write_bundle`` pass over :func:`validate_report_structurally`) -- there is no bypass -- - and written back only when it passes, so a dangling reference or duplicate key never lands on disk. - - Combine is the only fill for a routed-agentic component and marks its outcome - ``agentic-applied``. It is **idempotent**: on an applied component, re-running it with the same - authored pipelines as those now in the report no-ops (``already_combined`` true), even when the - authored names reuse member names; different authored pipelines are refused with - :data:`COMPONENT_ALREADY_FILLED`. A fresh ``convert`` rewrites the report without the record, so - the combine applies again after a re-convert and route. + Validates the group's membership against the recorded plan, stores the authored pipelines in + ``metadata/agentic_combines.json``, and rebuilds (applying it deterministically). The group's + membership must exactly match a routed-agentic component in the plan, and every authored pipeline + must carry the correct source tag. The merged report is always validated structurally before + writing. Returns ``{"ok", "violations", "error", "component_id", "pipelines", "already_combined"}``. ``ok`` - is ``False`` (and nothing written) on a plan/membership error (``error`` set), a missing source tag, - or any structural violation (``violations`` set). - - Raises: - FileNotFoundError: when the translation report is missing (run convert first). + is ``False`` (and nothing written) on a plan/membership error, source tag violation, or structural + issue. Empty ``authored_pipelines`` is refused. Same combine_sha256 is a no-op; different hash + replaces the stored entry and rebuilds. Different authored pipelines after applying are refused. """ members = {str(member) for member in group_members} + if not authored_pipelines: + return {"ok": False, "error": "pipelines list cannot be empty", "violations": [], "pipelines": 0} + component_id, plan, source, error = _resolve_agentic_component(output_dir, members) if error is not None: return {"ok": False, "error": error, "violations": [], "pipelines": 0} - assert component_id is not None and plan is not None and source is not None # guaranteed when error is None + assert component_id is not None and plan is not None and source is not None tag_violations = _authored_source_tag_violations(authored_pipelines, source) if tag_violations: @@ -634,41 +806,75 @@ def apply_combine_fill( error = f"{REPORT_FILENAME} carries no routing record; run `route` to apply the plan before filling." return {"ok": False, "error": error, "violations": [], "pipelines": 0} if routing_record_mismatches(record, plan): - return {"ok": False, "error": REROUTED_UNDER_DIFFERENT_PLAN, "violations": [], "pipelines": 0} + error = "the report was routed under a different plan; re-run convert, then route, then combine again" + return {"ok": False, "error": error, "violations": [], "pipelines": 0} - if record["components"][component_id].get("outcome") == OUTCOME_AGENTIC_APPLIED: - in_report = {pipeline.get("name"): pipeline for pipeline in _report_pipelines(report)} - if not all(in_report.get(pipeline.get("name")) == pipeline for pipeline in authored_pipelines): - return {"ok": False, "error": COMPONENT_ALREADY_FILLED, "violations": [], "pipelines": 0} + new_combine_sha256 = canonical_sha256([pipeline_to_store(p) for p in authored_pipelines]) + combines = _load_combines(output_dir) or {} + stored_combine = combines.get(component_id, {}) + stored_sha256 = stored_combine.get("combine_sha256") + in_report = {pipeline.get("name"): pipeline for pipeline in _report_pipelines(report)} + if stored_sha256 == new_combine_sha256: + if all(in_report.get(pipeline.get("name")) == pipeline for pipeline in authored_pipelines): + return { + "ok": True, + "error": None, + "violations": [], + "component_id": component_id, + "pipelines": len(_report_pipelines(report)), + "already_combined": True, + } return { - "ok": True, - "error": None, + "ok": False, + "error": COMPONENT_ALREADY_FILLED, "violations": [], - "component_id": component_id, - "pipelines": len(_report_pipelines(report)), - "already_combined": True, + "pipelines": 0, } - - merged = combine_group_fill(report, members, authored_pipelines) - merged[ROUTING_RECORD_KEY] = copy.deepcopy(record) - merged[ROUTING_RECORD_KEY]["components"][component_id]["outcome"] = OUTCOME_AGENTIC_APPLIED - - result = validate_report_structurally(merged) + combines[component_id] = { + "members": list(members), + "pipelines": authored_pipelines, + "combine_sha256": new_combine_sha256, + } + _save_combines(output_dir, combines) + baseline_report, baseline_gaps, _ = _load_baseline(output_dir) + new_report, new_gaps, new_record = rebuild( + baseline_report, + baseline_gaps, + plan, + combines, + record, + ) + if not isinstance(new_report.get("pipelines"), list): + new_report = {"pipelines": [new_report]} + new_report[ROUTING_RECORD_KEY] = new_record + result = validate_report_structurally(new_report) if not result.ok: violations = [f"[{finding.code}] {finding.location}: {finding.message}" for finding in result.violations] return {"ok": False, "error": None, "violations": violations, "pipelines": 0} - _write_json_atomic(report_path, merged) + _write_json_atomic(report_path, new_report) + _write_json_atomic(work / GAPS_FILENAME, new_gaps) + if (work / STAMPED_REPORT_FILENAME).exists(): + _write_json_atomic(work / STAMPED_REPORT_FILENAME, new_report) return { "ok": True, "error": None, "violations": [], "component_id": component_id, - "pipelines": len(merged["pipelines"]), + "pipelines": len(new_report["pipelines"]), "already_combined": False, } +def pipeline_to_store(pipeline: dict[str, Any]) -> dict[str, Any]: + """Extract the storable parts of a pipeline IR dict for hashing (excluding dynamic fields).""" + return { + "name": pipeline.get("name"), + "tags": pipeline.get("tags"), + "tasks": pipeline.get("tasks"), + } + + # --------------------------------------------------------------------------- # # Structural validation via the existing bundle invariants. # --------------------------------------------------------------------------- # diff --git a/tests/unit/test_route_agentic_flow.py b/tests/unit/test_route_agentic_flow.py index e665b796..00b9bf6c 100644 --- a/tests/unit/test_route_agentic_flow.py +++ b/tests/unit/test_route_agentic_flow.py @@ -40,6 +40,7 @@ REPORT_FILENAME, REROUTED_UNDER_DIFFERENT_PLAN, ROUTED_AGENTIC_MERGE_REFUSED, + ROUTING_RECORD_KEY, WORK_DIRNAME, agentic_pipeline_names, alter_report, @@ -452,17 +453,18 @@ def test_route_writes_no_record_when_nothing_is_routed_agentic(tmp_path: Path) - assert report_path.read_bytes() == before -def test_reroute_to_deterministic_is_refused_and_leaves_the_report_untouched(tmp_path: Path) -> None: +def test_reroute_to_different_decision_rebuilds_the_report(tmp_path: Path) -> None: + """Re-routing with different decisions rebuilds deterministically; no longer refused.""" _write_work(tmp_path, _report_two_pipelines(), gaps=[]) apply_plan_to_report(tmp_path, _plan(parent="agentic", child="deterministic")) report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME - routed = report_path.read_bytes() - - with pytest.raises(ValueError, match=REROUTED_UNDER_DIFFERENT_PLAN): - apply_plan_to_report(tmp_path, _plan(parent="deterministic", child="deterministic")) - with pytest.raises(ValueError, match=REROUTED_UNDER_DIFFERENT_PLAN): - apply_plan_to_report(tmp_path, _plan(parent="deterministic", child="agentic")) - assert report_path.read_bytes() == routed + first = json.loads(report_path.read_text(encoding="utf-8")) + first_outcomes = {cid: entry["outcome"] for cid, entry in first[ROUTING_RECORD_KEY]["components"].items()} + assert first_outcomes["component-1"] == "agentic-not-viable" + assert first_outcomes["component-2"] == "deterministic" + apply_plan_to_report(tmp_path, _plan(parent="deterministic", child="deterministic")) + second = json.loads(report_path.read_text(encoding="utf-8")) + assert ROUTING_RECORD_KEY not in second def test_reroute_under_the_same_decisions_keeps_fills_and_updates_the_plan_hash(tmp_path: Path) -> None: @@ -606,19 +608,21 @@ def test_combine_is_idempotent_running_twice_yields_no_duplicate(tmp_path: Path) assert names == ["orders_lfc"] # exactly one authored pipeline, no duplicate -def test_a_different_combine_on_an_applied_component_is_refused(tmp_path: Path) -> None: - """A second combine whose authored pipelines differ from those now in the report is refused.""" +def test_a_different_combine_on_an_applied_component_replaces_and_rebuilds(tmp_path: Path) -> None: + """A second combine with a different hash replaces the stored entry and rebuilds.""" _setup_routed_agentic(tmp_path, decision="agentic") first = apply_combine_fill(tmp_path, ["parent", "child"], [_named_lfc_pipeline("orders_lfc")]) assert first["ok"] is True and first["already_combined"] is False report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME - combined_report = report_path.read_bytes() + first_names = [p["name"] for p in json.loads(report_path.read_text(encoding="utf-8"))["pipelines"]] + assert first_names == ["orders_lfc"] second = apply_combine_fill(tmp_path, ["parent", "child"], [_named_lfc_pipeline("orders_lfc_v2")]) - assert second["ok"] is False - assert second["error"] == COMPONENT_ALREADY_FILLED - assert report_path.read_bytes() == combined_report + assert second["ok"] is True + assert second["already_combined"] is False + second_names = [p["name"] for p in json.loads(report_path.read_text(encoding="utf-8"))["pipelines"]] + assert second_names == ["orders_lfc_v2"] def test_combine_idempotent_when_authored_name_collides_with_a_former_member(tmp_path: Path) -> None: From 44cd53c038f9919b61eeefe109630c09a804d7be Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Wed, 7 Oct 2026 11:48:41 +0100 Subject: [PATCH 023/122] Fix adapter route logic to remove reroute_conflict refusal - Remove import and use of reroute_conflict function - Allow validation failures to return exit code 1 with violations output - Implement new replay model route logic without re-routing refusal Fixes CLI route tests that were failing due to missing reroute_conflict import. Now 1521 tests passing (up from 19 failing initially). Co-authored-by: Isaac --- src/flowx/adapter/__main__.py | 15 +++++---------- tests/unit/test_package_routing_preflight.py | 12 ++++++------ 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/src/flowx/adapter/__main__.py b/src/flowx/adapter/__main__.py index aa79ed56..b71aeea4 100644 --- a/src/flowx/adapter/__main__.py +++ b/src/flowx/adapter/__main__.py @@ -188,7 +188,7 @@ def _run_route(args: argparse.Namespace) -> int: """ from flowx import routing from flowx.models.conversion_plan import ConversionPlan - from flowx.route_agentic import REPORT_FILENAME, WORK_DIRNAME, apply_plan, reroute_conflict + from flowx.route_agentic import REPORT_FILENAME, WORK_DIRNAME, apply_plan inventory_path = args.output_dir / "metadata" / "inventory.json" if not inventory_path.exists(): @@ -224,15 +224,10 @@ def _run_route(args: argparse.Namespace) -> int: ) return 1 - if not routing.validate_plan(plan, inventory): - try: - conflict = reroute_conflict(args.output_dir, plan) - except (OSError, json.JSONDecodeError) as error: - print(f"Failed to read {report_path}: {error}", file=sys.stderr) - return 1 - if conflict is not None: - print(f"Refusing to route: {conflict}.", file=sys.stderr) - return 1 + violations = routing.validate_plan(plan, inventory) + if violations: + _emit_json({"ok": False, "violations": violations, "components": 0}, args.out) + return 1 try: result = routing.record_plan(args.output_dir, plan=plan) diff --git a/tests/unit/test_package_routing_preflight.py b/tests/unit/test_package_routing_preflight.py index 61687577..82c898d9 100644 --- a/tests/unit/test_package_routing_preflight.py +++ b/tests/unit/test_package_routing_preflight.py @@ -198,9 +198,10 @@ def test_rerouting_the_same_decisions_after_enrich_refreshes_the_stamped_report( assert audit["components"][0]["outcome"] == "agentic-applied" -def test_route_agentic_then_deterministic_is_refused_and_writes_nothing( +def test_route_agentic_then_deterministic_switches_back_to_deterministic( tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: + """Re-routing from agentic to deterministic is allowed; the record is cleared (spec change).""" report_path = _setup(tmp_path, _inventory("agentic")) plan_path = tmp_path / "plan.json" @@ -210,14 +211,13 @@ def _route(decision: str) -> int: return adapter_main(["route", "--output-dir", str(tmp_path), "--plan-path", str(plan_path)]) assert _route("agentic") == 0 - recorded_plan = (tmp_path / "metadata" / "conversion_plan.json").read_bytes() routed_report = report_path.read_bytes() capsys.readouterr() - assert _route("deterministic") == 1 - assert "re-run convert, then route" in capsys.readouterr().err - assert (tmp_path / "metadata" / "conversion_plan.json").read_bytes() == recorded_plan - assert report_path.read_bytes() == routed_report + assert _route("deterministic") == 0 + capsys.readouterr() + deterministic_report = json.loads(report_path.read_text(encoding="utf-8")) + assert "_routing_record" not in deterministic_report def test_package_with_a_current_plan_writes_an_audit_with_hashes(tmp_path: Path) -> None: From 0846aa1ebf00de74bfb9e2f5450978f045f97412 Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Wed, 7 Oct 2026 11:58:07 +0100 Subject: [PATCH 024/122] Complete replay model implementation: baseline verification, audit, and baseline updates Implements the final items from phase-1 replay spec: 1. Baseline byte-for-byte handling: _load_baseline now returns raw file bytes alongside parsed JSON, and rebuild uses these bytes for hashing to ensure byte-identical content. _save_baseline_bytes saves baselines without reformatting. This fixes hash mismatches between file hashes and computed hashes. 2. Package preflight (_routing_plan_failures): Verifies that baseline files exist and their hashes match the routing record when a plan has agentic components. Replays rebuild from baseline, plan, and combines store to ensure packaged report matches. For stamped reports from modify, verifies routing record equality. Provides targeted error messages for tampered baseline, stale combine store, and out-of-date reports. 3. Route audit (_write_route_audit): Extended to record per-component combine_sha256, fingerprint, and replacements, plus baseline_gaps_sha256 for full provenance. Allows auditing of what actually was converted and how changes were tracked. 4. Merge-agentic baseline updates (ir_serde.merge_agentic_results): When a report has a routing record, applies merges to both live report AND baseline report, refreshing baseline_report_sha256 in the record. This ensures that baseline captures merge-applied state so future rebuilds preserve fills. Still refuses merges into routed-agentic pipelines as per design. 5. Test updates: Fixed test expectations to match spec-compliant audit format. Removed unused variable and imports. All paths byte-identical: deterministic-only and no-plan outputs remain unchanged. --- src/flowx/bundler/dab_writer.py | 115 ++++++++++++++++--- src/flowx/ir_serde.py | 63 +++++++++- src/flowx/route_agentic.py | 70 +++++++---- tests/unit/test_package_routing_preflight.py | 19 ++- tests/unit/test_route_agentic_flow.py | 2 - 5 files changed, 225 insertions(+), 44 deletions(-) diff --git a/src/flowx/bundler/dab_writer.py b/src/flowx/bundler/dab_writer.py index e45b1c8d..1ce0c678 100644 --- a/src/flowx/bundler/dab_writer.py +++ b/src/flowx/bundler/dab_writer.py @@ -383,12 +383,26 @@ def _routing_plan_failures(output_dir: Path, report_path: Path) -> list[str]: decided on. If discover or enrich ran again since route, package refuses rather than shipping a report that no longer reflects the user's decision. It also refuses when the report's routing record disagrees with the plan: a different plan hash, different components, members or - decisions, or no record although the plan routes a component agentic. Returns one message per - problem; an empty list means there is no recorded plan or it still matches. + decisions, or no record although the plan routes a component agentic. + + When the plan has an agentic component or the report has a routing record, also verifies that the + immutable baseline exists and its hashes match the record, then replays rebuild to check that the + packaged report matches (or for a stamped report, that its record matches). + + Returns one message per problem; an empty list means there is no recorded plan or it still matches. """ from flowx.discovery_serde import canonical_sha256 from flowx.models.conversion_plan import DECISION_AGENTIC, ConversionPlan - from flowx.route_agentic import routing_record, routing_record_mismatches + from flowx.route_agentic import ( + BASELINE_DIRNAME, + BASELINE_GAPS_FILENAME, + BASELINE_REPORT_FILENAME, + STAMPED_REPORT_FILENAME, + WORK_DIRNAME, + rebuild, + routing_record, + routing_record_mismatches, + ) from flowx.routing import plan_binding_violations metadata_dir = Path(output_dir) / "metadata" @@ -429,7 +443,76 @@ def _routing_plan_failures(output_dir: Path, report_path: Path) -> list[str]: if record.get("conversion_plan_sha256") != canonical_sha256(plan_document): failures.append(f"{report_name} was routed under a different conversion plan; {remedy}") failures.extend(f"{mismatch}; {remedy}" for mismatch in routing_record_mismatches(record, plan_document)) - return failures + if failures: + return failures + + has_agentic = any(component.decision == DECISION_AGENTIC for component in plan.components) + if not (has_agentic or record): + return [] + + work_dir = Path(output_dir) / WORK_DIRNAME + baseline_dir = work_dir / BASELINE_DIRNAME + baseline_report_path = baseline_dir / BASELINE_REPORT_FILENAME + baseline_gaps_path = baseline_dir / BASELINE_GAPS_FILENAME + + if not baseline_report_path.exists() or not baseline_gaps_path.exists(): + return ["the deterministic baseline is missing or changed; re-run convert, then route"] + + recorded_report_sha256 = record.get("baseline_report_sha256") + recorded_gaps_sha256 = record.get("baseline_gaps_sha256") + actual_report_sha256 = _file_sha256(baseline_report_path) + actual_gaps_sha256 = _file_sha256(baseline_gaps_path) + + if recorded_report_sha256 != actual_report_sha256 or recorded_gaps_sha256 != actual_gaps_sha256: + return ["the deterministic baseline is missing or changed; re-run convert, then route"] + + try: + baseline_report = json.loads(baseline_report_path.read_text(encoding="utf-8")) + baseline_gaps = json.loads(baseline_gaps_path.read_text(encoding="utf-8")) + baseline_report_bytes = baseline_report_path.read_bytes() + baseline_gaps_bytes = baseline_gaps_path.read_bytes() + except (OSError, json.JSONDecodeError) as error: + return [f"Failed to load baseline files: {error}"] + + combines_path = metadata_dir / "agentic_combines.json" + combines = {} + if combines_path.exists(): + try: + combines_raw = json.loads(combines_path.read_text(encoding="utf-8")) + combines = combines_raw if isinstance(combines_raw, dict) else {} + except (OSError, json.JSONDecodeError): + pass + + rebuilt_report, rebuilt_gaps, rebuilt_record = rebuild( + baseline_report, + baseline_gaps, + plan_document, + combines, + record, + baseline_report_bytes, + baseline_gaps_bytes, + ) + + stamped_report_path = work_dir / STAMPED_REPORT_FILENAME + if stamped_report_path.exists(): + try: + stamped_report = json.loads(stamped_report_path.read_text(encoding="utf-8")) + stamped_record = routing_record(stamped_report) + if stamped_record != rebuilt_record: + return ["the configured report is out of date; re-run modify"] + except (OSError, json.JSONDecodeError): + pass + else: + report_without_record = {k: v for k, v in report.items() if k != "_routing_record"} + rebuilt_without_record = {k: v for k, v in rebuilt_report.items() if k != "_routing_record"} + report_json = json.dumps(report_without_record, separators=(",", ":"), sort_keys=True, default=str) + rebuilt_json = json.dumps(rebuilt_without_record, separators=(",", ":"), sort_keys=True, default=str) + if report_json != rebuilt_json: + if combines: + return ["the combine store has been edited; re-run fill-agentic combine"] + return [f"{report_name} does not match a fresh rebuild; {remedy}"] + + return [] def _file_sha256(path: Path) -> str | None: @@ -490,18 +573,21 @@ def _write_route_audit(output_dir: Path, report_path: Path | None = None) -> Pat agentic_pipelines: set[str] = set() for component in plan.components: recorded = recorded_components.get(component.component_id) + component_entry: dict[str, Any] = { + "component_id": component.component_id, + "members": list(component.members), + "decision": component.decision, + } if isinstance(recorded, dict): - outcome = recorded.get("outcome") + component_entry["outcome"] = recorded.get("outcome") + component_entry["combine_sha256"] = recorded.get("combine_sha256") + component_entry["fingerprint"] = recorded.get("fingerprint") + if recorded.get("replacements"): + component_entry["replacements"] = recorded.get("replacements") else: - outcome = OUTCOME_DETERMINISTIC if component.decision == DECISION_DETERMINISTIC else None - components.append( - { - "component_id": component.component_id, - "members": list(component.members), - "decision": component.decision, - "outcome": outcome, - } - ) + component_entry["outcome"] = OUTCOME_DETERMINISTIC if component.decision == DECISION_DETERMINISTIC else None + component_entry["combine_sha256"] = None + components.append(component_entry) if component.decision == DECISION_AGENTIC: agentic_pipelines.update(component.members) @@ -530,6 +616,7 @@ def _write_route_audit(output_dir: Path, report_path: Path | None = None) -> Pat "source_insights_sha256": plan.source_insights_sha256, "conversion_plan_sha256": _file_sha256(plan_path), "baseline_report_sha256": record.get("baseline_report_sha256") if record is not None else None, + "baseline_gaps_sha256": record.get("baseline_gaps_sha256") if record is not None else None, "translation_report_sha256": _file_sha256(report_path) if report_path is not None else None, "components": components, "agentic_pipelines": sorted(agentic_pipelines), diff --git a/src/flowx/ir_serde.py b/src/flowx/ir_serde.py index fd8c9e7c..04c9440e 100644 --- a/src/flowx/ir_serde.py +++ b/src/flowx/ir_serde.py @@ -656,6 +656,10 @@ def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Pat only by ``fill-agentic combine``, so a result that names one, or an untargeted result whose activity would first land in one, is refused before anything is written. + When the report has a routing record, merges into non-routed pipelines are applied to both the + live report and the stored baseline report (so the next rebuild keeps them), and the record's + baseline_report_sha256 is refreshed. All merges are validated first (all-or-nothing). + Args: report_path: ``translation_report.json`` produced by the translate phase. results_dir: Directory of per-activity result JSON files. @@ -668,7 +672,18 @@ def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Pat Raises: ValueError: A result would land in a pipeline the routing record routes agentic. """ - from flowx.route_agentic import ROUTED_AGENTIC_MERGE_REFUSED, routed_agentic_pipelines + import hashlib + + from flowx.route_agentic import ( + BASELINE_DIRNAME, + BASELINE_REPORT_FILENAME, + ROUTED_AGENTIC_MERGE_REFUSED, + ROUTING_RECORD_KEY, + STAMPED_REPORT_FILENAME, + WORK_DIRNAME, + routed_agentic_pipelines, + routing_record, + ) report = json.loads(report_path.read_text(encoding="utf-8")) pipelines = report["pipelines"] if isinstance(report, dict) and "pipelines" in report else [report] @@ -703,4 +718,50 @@ def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Pat destination = output_path or report_path destination.write_text(json.dumps(report, indent=2, default=str), encoding="utf-8") logger.info("Wrote merged report to %s (%d merged, %d unmatched)", destination, merged, unmatched) + + record = routing_record(report) + if record is not None and merged > 0: + output_dir = report_path.parent.parent + baseline_dir = output_dir / WORK_DIRNAME / BASELINE_DIRNAME + baseline_report_path = baseline_dir / BASELINE_REPORT_FILENAME + + if baseline_report_path.exists(): + try: + baseline_report = json.loads(baseline_report_path.read_text(encoding="utf-8")) + baseline_pipelines = ( + baseline_report["pipelines"] + if isinstance(baseline_report, dict) and "pipelines" in baseline_report + else [baseline_report] + ) + + for result_file in results_dir.glob("*.json"): + data = json.loads(result_file.read_text(encoding="utf-8")) + activity_name = data.get("activity_name") or data.get("activity") + task = data.get("task") or data.get("ir") + if activity_name and isinstance(task, dict): + wanted = data.get("pipeline") + candidates = [ + pipeline for pipeline in baseline_pipelines if not wanted or pipeline.get("name") == wanted + ] + _find_and_replace_task( + [task_item for pipeline in candidates for task_item in pipeline.get("tasks", [])], + activity_name, + dict(task), + ) + + baseline_report_bytes = json.dumps(baseline_report, indent=2, default=str).encode("utf-8") + baseline_report_path.write_bytes(baseline_report_bytes) + + record["baseline_report_sha256"] = hashlib.sha256(baseline_report_bytes).hexdigest() + report[ROUTING_RECORD_KEY] = record + destination.write_text(json.dumps(report, indent=2, default=str), encoding="utf-8") + + stamped_path = report_path.parent / STAMPED_REPORT_FILENAME + if stamped_path.exists(): + stamped_report = json.loads(stamped_path.read_text(encoding="utf-8")) + stamped_report[ROUTING_RECORD_KEY] = record + stamped_path.write_text(json.dumps(stamped_report, indent=2, default=str), encoding="utf-8") + except (OSError, json.JSONDecodeError) as error: + logger.warning("Could not update baseline with merges: %s", error) + return merged, unmatched diff --git a/src/flowx/route_agentic.py b/src/flowx/route_agentic.py index 85df7933..f8605c32 100644 --- a/src/flowx/route_agentic.py +++ b/src/flowx/route_agentic.py @@ -236,41 +236,46 @@ def _baseline_exists(output_dir: Path) -> bool: return (baseline_dir / BASELINE_REPORT_FILENAME).exists() or (baseline_dir / BASELINE_GAPS_FILENAME).exists() -def _load_baseline(output_dir: Path) -> tuple[dict[str, Any], list[dict[str, Any]], str | None]: +def _load_baseline(output_dir: Path) -> tuple[dict[str, Any], list[dict[str, Any]], bytes, bytes, str | None]: """Load the immutable baseline report and gaps, creating it from current live ones if needed. - Returns ``(report, gaps, error)`` where error is set when trying to load an existing baseline - and it fails to read. + Returns ``(report, gaps, report_bytes, gaps_bytes, error)`` where error is set when trying to + load an existing baseline and it fails to read. The bytes are used to compute baseline hashes + for the routing record. """ baseline_dir = Path(output_dir) / WORK_DIRNAME / BASELINE_DIRNAME baseline_report_path = baseline_dir / BASELINE_REPORT_FILENAME baseline_gaps_path = baseline_dir / BASELINE_GAPS_FILENAME if baseline_report_path.exists() or baseline_gaps_path.exists(): try: - report = ( - json.loads(baseline_report_path.read_text(encoding="utf-8")) if baseline_report_path.exists() else {} - ) - gaps = json.loads(baseline_gaps_path.read_text(encoding="utf-8")) if baseline_gaps_path.exists() else [] - return report, gaps if isinstance(gaps, list) else [], None + report_bytes = baseline_report_path.read_bytes() if baseline_report_path.exists() else b"{}" + gaps_bytes = baseline_gaps_path.read_bytes() if baseline_gaps_path.exists() else b"[]" + report = json.loads(report_bytes) if report_bytes else {} + gaps = json.loads(gaps_bytes) if gaps_bytes else [] + return report, gaps if isinstance(gaps, list) else [], report_bytes, gaps_bytes, None except (OSError, json.JSONDecodeError) as error: - return {}, [], str(error) + return {}, [], b"{}", b"[]", str(error) work_dir = Path(output_dir) / WORK_DIRNAME live_report_path = work_dir / REPORT_FILENAME live_gaps_path = work_dir / GAPS_FILENAME try: - report = json.loads(live_report_path.read_text(encoding="utf-8")) if live_report_path.exists() else {} - gaps = json.loads(live_gaps_path.read_text(encoding="utf-8")) if live_gaps_path.exists() else [] - return report, gaps if isinstance(gaps, list) else [], None + report_bytes = live_report_path.read_bytes() if live_report_path.exists() else b"{}" + gaps_bytes = live_gaps_path.read_bytes() if live_gaps_path.exists() else b"[]" + report = json.loads(report_bytes) if report_bytes else {} + gaps = json.loads(gaps_bytes) if gaps_bytes else [] + return report, gaps if isinstance(gaps, list) else [], report_bytes, gaps_bytes, None except (OSError, json.JSONDecodeError): - return {}, [], None + return {}, [], b"{}", b"[]", None -def _save_baseline(output_dir: Path, report: dict[str, Any], gaps: list[dict[str, Any]]) -> None: - """Save an immutable copy of the report and gaps to the baseline folder.""" +def _save_baseline_bytes(output_dir: Path, report_bytes: bytes, gaps_bytes: bytes) -> None: + """Save an immutable copy of the baseline files as exact bytes (byte-for-byte).""" baseline_dir = Path(output_dir) / WORK_DIRNAME / BASELINE_DIRNAME baseline_dir.mkdir(parents=True, exist_ok=True) - _write_json_atomic(baseline_dir / BASELINE_REPORT_FILENAME, report) - _write_json_atomic(baseline_dir / BASELINE_GAPS_FILENAME, gaps) + baseline_report_path = baseline_dir / BASELINE_REPORT_FILENAME + baseline_gaps_path = baseline_dir / BASELINE_GAPS_FILENAME + baseline_report_path.write_bytes(report_bytes) + baseline_gaps_path.write_bytes(gaps_bytes) def _load_combines(output_dir: Path) -> dict[str, dict[str, Any]] | None: @@ -356,6 +361,8 @@ def rebuild( plan: dict[str, Any], combines: dict[str, dict[str, Any]] | None, previous_record: dict[str, Any] | None, + baseline_report_bytes: bytes | None = None, + baseline_gaps_bytes: bytes | None = None, ) -> tuple[dict[str, Any], list[dict[str, Any]], dict[str, Any]]: """Pure rebuild: derive the routed report, gaps, and record from baseline and plan. @@ -369,6 +376,17 @@ def rebuild( The record holds per-component members, decision, outcome, combine_sha256, and fingerprint. When fingerprint differs from previous_record's, a replacement entry is appended to track the change. An all-deterministic plan returns no record (byte-identical to baseline). + + Args: + baseline_report: Parsed baseline report JSON. + baseline_gaps: Parsed baseline gaps JSON. + plan: Routing plan dict. + combines: Stored combines from metadata/agentic_combines.json. + previous_record: Prior routing record for change tracking. + baseline_report_bytes: Raw bytes of baseline report file for hashing. If not provided, + computed from baseline_report as compact JSON. + baseline_gaps_bytes: Raw bytes of baseline gaps file for hashing. If not provided, + computed from baseline_gaps as compact JSON. """ combines_store = combines or {} planned = _planned_components(plan) @@ -380,8 +398,10 @@ def rebuild( new_report, new_gaps = alter_report(report, gaps, agentic_names) if not isinstance(new_report.get("pipelines"), list): new_report = {"pipelines": [new_report]} - baseline_report_bytes = json.dumps(baseline_report, separators=(",", ":")).encode("utf-8") - baseline_gaps_bytes = json.dumps(baseline_gaps, separators=(",", ":")).encode("utf-8") + if baseline_report_bytes is None: + baseline_report_bytes = json.dumps(baseline_report, separators=(",", ":")).encode("utf-8") + if baseline_gaps_bytes is None: + baseline_gaps_bytes = json.dumps(baseline_gaps, separators=(",", ":")).encode("utf-8") components: dict[str, dict[str, Any]] = {} for component_id, planned_entry in planned.items(): decision = planned_entry["decision"] @@ -585,7 +605,9 @@ def apply_plan_to_report(output_dir: Path, plan: dict[str, Any]) -> dict[str, An if not report_path.exists(): raise FileNotFoundError(f"No {REPORT_FILENAME} under {work}; run the convert phase first.") - baseline_report, baseline_gaps, baseline_error = _load_baseline(output_dir) + baseline_report, baseline_gaps, baseline_report_bytes, baseline_gaps_bytes, baseline_error = _load_baseline( + output_dir + ) if baseline_error is not None: raise ValueError(f"Failed to load baseline: {baseline_error}") old_report_bytes = report_path.read_bytes() @@ -600,6 +622,8 @@ def apply_plan_to_report(output_dir: Path, plan: dict[str, Any]) -> dict[str, An plan, combines, record, + baseline_report_bytes, + baseline_gaps_bytes, ) if not new_record: if record is not None: @@ -611,7 +635,7 @@ def apply_plan_to_report(output_dir: Path, plan: dict[str, Any]) -> dict[str, An _write_json_atomic(work / STAMPED_REPORT_FILENAME, report_without_record) return {"agentic_pipelines": sorted(agentic), "gaps": len(baseline_gaps), "altered": record is not None} if not _baseline_exists(output_dir): - _save_baseline(output_dir, baseline_report, baseline_gaps) + _save_baseline_bytes(output_dir, baseline_report_bytes, baseline_gaps_bytes) new_report[ROUTING_RECORD_KEY] = new_record old_report_without_record = copy.deepcopy(report) old_report_without_record.pop(ROUTING_RECORD_KEY, None) @@ -836,13 +860,15 @@ def apply_combine_fill( "combine_sha256": new_combine_sha256, } _save_combines(output_dir, combines) - baseline_report, baseline_gaps, _ = _load_baseline(output_dir) + baseline_report, baseline_gaps, baseline_report_bytes, baseline_gaps_bytes, _ = _load_baseline(output_dir) new_report, new_gaps, new_record = rebuild( baseline_report, baseline_gaps, plan, combines, record, + baseline_report_bytes, + baseline_gaps_bytes, ) if not isinstance(new_report.get("pipelines"), list): new_report = {"pipelines": [new_report]} diff --git a/tests/unit/test_package_routing_preflight.py b/tests/unit/test_package_routing_preflight.py index 82c898d9..a54e42d2 100644 --- a/tests/unit/test_package_routing_preflight.py +++ b/tests/unit/test_package_routing_preflight.py @@ -120,7 +120,13 @@ def test_package_with_a_deterministic_plan_and_no_record_passes(tmp_path: Path) assert _package(tmp_path) == 0 audit = json.loads((tmp_path / "metadata" / "route_audit.json").read_text(encoding="utf-8")) assert audit["components"] == [ - {"component_id": "component-1", "members": ["solo"], "decision": "deterministic", "outcome": "deterministic"} + { + "component_id": "component-1", + "members": ["solo"], + "decision": "deterministic", + "outcome": "deterministic", + "combine_sha256": None, + } ] assert audit["baseline_report_sha256"] is None @@ -167,9 +173,13 @@ def test_route_combine_modify_then_package_keeps_the_routing_record(tmp_path: Pa assert _package(tmp_path) == 0 audit = json.loads((tmp_path / "metadata" / "route_audit.json").read_text(encoding="utf-8")) - assert audit["components"] == [ - {"component_id": "component-1", "members": ["solo"], "decision": "agentic", "outcome": "agentic-applied"} - ] + component_audit = audit["components"][0] + assert component_audit["component_id"] == "component-1" + assert component_audit["members"] == ["solo"] + assert component_audit["decision"] == "agentic" + assert component_audit["outcome"] == "agentic-applied" + assert "combine_sha256" in component_audit + assert "fingerprint" in component_audit assert audit["baseline_report_sha256"] == baseline_sha256 assert audit["translation_report_sha256"] == hashlib.sha256(stamped_path.read_bytes()).hexdigest() @@ -211,7 +221,6 @@ def _route(decision: str) -> int: return adapter_main(["route", "--output-dir", str(tmp_path), "--plan-path", str(plan_path)]) assert _route("agentic") == 0 - routed_report = report_path.read_bytes() capsys.readouterr() assert _route("deterministic") == 0 diff --git a/tests/unit/test_route_agentic_flow.py b/tests/unit/test_route_agentic_flow.py index 00b9bf6c..dc82d856 100644 --- a/tests/unit/test_route_agentic_flow.py +++ b/tests/unit/test_route_agentic_flow.py @@ -35,10 +35,8 @@ from flowx.models.discovery import CONCEPT_NOTEBOOK, SourceGraph, SourceNode from flowx.models.ir import ControlEdge, Lineage from flowx.route_agentic import ( - COMPONENT_ALREADY_FILLED, GAPS_FILENAME, REPORT_FILENAME, - REROUTED_UNDER_DIFFERENT_PLAN, ROUTED_AGENTIC_MERGE_REFUSED, ROUTING_RECORD_KEY, WORK_DIRNAME, From af2205dec030e0aa27cf8090db6940a9c5a880c0 Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Wed, 7 Oct 2026 12:12:13 +0100 Subject: [PATCH 025/122] Fix apply_combine_fill to handle same hash idempotently and validate before writing Implement the replay model's all-or-nothing semantics for combine: - Same hash: return ok: true, already_combined: true, message: 'already applied, unchanged', writing nothing - Different hash: replace the stored combine entry and rebuild from baseline - Validation: run structural validation before any writes (all-or-nothing); nothing written on failure - Empty pipelines list stays refused The combine store is now only written after validation passes, ensuring consistency. Idempotent re-runs with the same pipelines hash are safe (report and store unchanged). Co-authored-by: Isaac --- src/flowx/route_agentic.py | 71 +++++++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 28 deletions(-) diff --git a/src/flowx/route_agentic.py b/src/flowx/route_agentic.py index f8605c32..f662438e 100644 --- a/src/flowx/route_agentic.py +++ b/src/flowx/route_agentic.py @@ -798,13 +798,18 @@ def apply_combine_fill( Validates the group's membership against the recorded plan, stores the authored pipelines in ``metadata/agentic_combines.json``, and rebuilds (applying it deterministically). The group's membership must exactly match a routed-agentic component in the plan, and every authored pipeline - must carry the correct source tag. The merged report is always validated structurally before - writing. - - Returns ``{"ok", "violations", "error", "component_id", "pipelines", "already_combined"}``. ``ok`` - is ``False`` (and nothing written) on a plan/membership error, source tag violation, or structural - issue. Empty ``authored_pipelines`` is refused. Same combine_sha256 is a no-op; different hash - replaces the stored entry and rebuilds. Different authored pipelines after applying are refused. + must carry the correct source tag. + + Returns ``{"ok", "violations", "error", "component_id", "pipelines", "already_combined", "message"}``. + ``ok`` is ``False`` (and nothing written) on a plan/membership error, source tag violation, + structural validation failure, or any other error. Empty ``authored_pipelines`` is refused. + + When the combine's hash matches the stored one and is already reflected in the live report, + returns ``ok: true, already_combined: true, message: 'already applied, unchanged'`` and writes + nothing. After a re-route, if the report no longer reflects the stored combine, rebuilds from + baseline and stored combines and writes the result. A different hash replaces the stored entry + for that component only and rebuilds. Nothing is written until structural validation passes + (all-or-nothing). """ members = {str(member) for member in group_members} if not authored_pipelines: @@ -838,6 +843,7 @@ def apply_combine_fill( stored_combine = combines.get(component_id, {}) stored_sha256 = stored_combine.get("combine_sha256") in_report = {pipeline.get("name"): pipeline for pipeline in _report_pipelines(report)} + if stored_sha256 == new_combine_sha256: if all(in_report.get(pipeline.get("name")) == pipeline for pipeline in authored_pipelines): return { @@ -847,41 +853,50 @@ def apply_combine_fill( "component_id": component_id, "pipelines": len(_report_pipelines(report)), "already_combined": True, + "message": "already applied, unchanged", } - return { - "ok": False, - "error": COMPONENT_ALREADY_FILLED, - "violations": [], - "pipelines": 0, + baseline_report, baseline_gaps, baseline_report_bytes, baseline_gaps_bytes, _ = _load_baseline(output_dir) + new_report, new_gaps, new_record = rebuild( + baseline_report, + baseline_gaps, + plan, + combines, + record, + baseline_report_bytes, + baseline_gaps_bytes, + ) + else: + combines[component_id] = { + "members": list(members), + "pipelines": authored_pipelines, + "combine_sha256": new_combine_sha256, } - combines[component_id] = { - "members": list(members), - "pipelines": authored_pipelines, - "combine_sha256": new_combine_sha256, - } - _save_combines(output_dir, combines) - baseline_report, baseline_gaps, baseline_report_bytes, baseline_gaps_bytes, _ = _load_baseline(output_dir) - new_report, new_gaps, new_record = rebuild( - baseline_report, - baseline_gaps, - plan, - combines, - record, - baseline_report_bytes, - baseline_gaps_bytes, - ) + baseline_report, baseline_gaps, baseline_report_bytes, baseline_gaps_bytes, _ = _load_baseline(output_dir) + new_report, new_gaps, new_record = rebuild( + baseline_report, + baseline_gaps, + plan, + combines, + record, + baseline_report_bytes, + baseline_gaps_bytes, + ) + if not isinstance(new_report.get("pipelines"), list): new_report = {"pipelines": [new_report]} new_report[ROUTING_RECORD_KEY] = new_record + result = validate_report_structurally(new_report) if not result.ok: violations = [f"[{finding.code}] {finding.location}: {finding.message}" for finding in result.violations] return {"ok": False, "error": None, "violations": violations, "pipelines": 0} + _save_combines(output_dir, combines) _write_json_atomic(report_path, new_report) _write_json_atomic(work / GAPS_FILENAME, new_gaps) if (work / STAMPED_REPORT_FILENAME).exists(): _write_json_atomic(work / STAMPED_REPORT_FILENAME, new_report) + return { "ok": True, "error": None, From 0fcec05f5ef57dc0a64223d958bd09e21f0521eb Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Wed, 7 Oct 2026 12:12:21 +0100 Subject: [PATCH 026/122] Add unit tests for combine idempotency and validation Tests cover the replay model's behavior: - Same combine re-run returns already_combined: true with 'already applied, unchanged' message - Empty pipelines list is refused without writing - Validation failure writes nothing (all-or-nothing) - Re-applying after re-route rebuilds identically from baseline - merge_agentic refuses routed-agentic pipelines Package preflight tests verify: - Tampered baseline is refused - Stale unstamped report is refused (suggests re-run route) - Edited combine store (pipelines field) is caught - Stale stamped report is refused (suggests re-run modify) All tests verify observable behavior (files written/unchanged, hashes, return values). Co-authored-by: Isaac --- tests/unit/test_package_routing_preflight.py | 117 +++++++++++++++++++ tests/unit/test_route_agentic_flow.py | 85 ++++++++++++++ 2 files changed, 202 insertions(+) diff --git a/tests/unit/test_package_routing_preflight.py b/tests/unit/test_package_routing_preflight.py index a54e42d2..aae5ca47 100644 --- a/tests/unit/test_package_routing_preflight.py +++ b/tests/unit/test_package_routing_preflight.py @@ -247,3 +247,120 @@ def test_package_with_a_current_plan_writes_an_audit_with_hashes(tmp_path: Path) ) assert audit["baseline_report_sha256"] == baseline_sha256 assert audit["translation_report_sha256"] == hashlib.sha256(report_path.read_bytes()).hexdigest() + + +def test_package_refuses_tampered_baseline(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + """Package refuses when the baseline hash in the record doesn't match the actual file.""" + _setup(tmp_path, _inventory("agentic")) + _record(tmp_path, "agentic") + plan = json.loads((tmp_path / "metadata" / "conversion_plan.json").read_text(encoding="utf-8")) + apply_plan_to_report(tmp_path, plan) + + baseline_path = tmp_path / "metadata" / ".work" / "route_baseline" / "translation_report.json" + if not baseline_path.exists(): + baseline_path = tmp_path / ".work" / "route_baseline" / "translation_report.json" + baseline_path.write_text("tampered", encoding="utf-8") + + assert _package(tmp_path) == 1 + assert "baseline is missing or changed" in capsys.readouterr().err + assert not (tmp_path / "databricks.yml").exists() + + +def test_package_refuses_stale_unstamped_report(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + """Package refuses when unstamped report doesn't match fresh rebuild (suggests re-run route).""" + report_path = _setup(tmp_path, _inventory("agentic")) + _record(tmp_path, "agentic") + plan = json.loads((tmp_path / "metadata" / "conversion_plan.json").read_text(encoding="utf-8")) + apply_plan_to_report(tmp_path, plan) + + report = json.loads(report_path.read_text(encoding="utf-8")) + report["pipelines"][0]["name"] = "tampered" + report_path.write_text(json.dumps(report, indent=2), encoding="utf-8") + + assert _package(tmp_path) == 1 + error = capsys.readouterr().err + assert "does not match" in error or "re-run" in error.lower() + assert not (tmp_path / "databricks.yml").exists() + + +def test_package_refuses_edited_combine_store(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + """Package refuses when agentic_combines.json pipelines were edited after the combine.""" + _setup(tmp_path, _inventory("agentic")) + plan_path = tmp_path / "plan.json" + plan_path.write_text( + json.dumps({"components": [{"component_id": "component-1", "members": ["solo"], "decision": "agentic"}]}), + encoding="utf-8", + ) + authored_path = tmp_path / "authored.json" + authored_path.write_text(json.dumps([_authored_pipeline()]), encoding="utf-8") + + from flowx.adapter.__main__ import main as adapter_main + + assert adapter_main(["route", "--output-dir", str(tmp_path), "--plan-path", str(plan_path)]) == 0 + assert ( + adapter_main( + [ + "fill-agentic", + "combine", + "--output-dir", + str(tmp_path), + "--members", + "solo", + "--pipelines-path", + str(authored_path), + ] + ) + == 0 + ) + + combines_path = tmp_path / "metadata" / "agentic_combines.json" + combines = json.loads(combines_path.read_text(encoding="utf-8")) + combines["component-1"]["pipelines"][0]["name"] = "tampered_name" + combines_path.write_text(json.dumps(combines, indent=2), encoding="utf-8") + + assert _package(tmp_path) == 1 + error = capsys.readouterr().err + assert "combine" in error.lower() or "re-run" in error.lower() + assert not (tmp_path / "databricks.yml").exists() + + +def test_package_refuses_stale_stamped_report(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + """Package refuses a stamped report whose record no longer matches the rebuild.""" + report_path = _setup(tmp_path, _inventory("agentic")) + plan_path = tmp_path / "plan.json" + plan_path.write_text( + json.dumps({"components": [{"component_id": "component-1", "members": ["solo"], "decision": "agentic"}]}), + encoding="utf-8", + ) + authored_path = tmp_path / "authored.json" + authored_path.write_text(json.dumps([_authored_pipeline()]), encoding="utf-8") + + from flowx.adapter.__main__ import main as adapter_main + + assert adapter_main(["route", "--output-dir", str(tmp_path), "--plan-path", str(plan_path)]) == 0 + assert ( + adapter_main( + [ + "fill-agentic", + "combine", + "--output-dir", + str(tmp_path), + "--members", + "solo", + "--pipelines-path", + str(authored_path), + ] + ) + == 0 + ) + assert adapter_main(["modify", str(report_path), "--output-dir", str(tmp_path)]) == 0 + + stamped_path = tmp_path / ".work" / "translation_report.stamped.json" + stamped_report = json.loads(stamped_path.read_text(encoding="utf-8")) + record = stamped_report["_routing_record"] + record["components"]["component-1"]["outcome"] = "agentic-not-viable" + stamped_path.write_text(json.dumps(stamped_report, indent=2), encoding="utf-8") + + assert _package(tmp_path) == 1 + assert "out of date; re-run modify" in capsys.readouterr().err + assert not (tmp_path / "databricks.yml").exists() diff --git a/tests/unit/test_route_agentic_flow.py b/tests/unit/test_route_agentic_flow.py index dc82d856..35842c94 100644 --- a/tests/unit/test_route_agentic_flow.py +++ b/tests/unit/test_route_agentic_flow.py @@ -808,3 +808,88 @@ def test_combine_requires_a_recorded_plan(tmp_path: Path) -> None: def test_apply_combine_fill_has_no_validation_bypass() -> None: # There must be no surface that writes the combined report without structural validation. assert "validate" not in inspect.signature(apply_combine_fill).parameters + + +def test_combine_same_hash_returns_already_combined_true_with_unchanged_message(tmp_path: Path) -> None: + """Same authored pipelines hash returns already_combined: True with message 'already applied, unchanged'.""" + _setup_routed_agentic(tmp_path, decision="agentic") + report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME + combines_path = tmp_path / "metadata" / "agentic_combines.json" + + first = apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()]) + assert first["ok"] is True and first["already_combined"] is False + report_before = report_path.read_bytes() + combines_before = combines_path.read_bytes() + + second = apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()]) + assert second["ok"] is True + assert second["already_combined"] is True + assert second.get("message") == "already applied, unchanged" + + assert report_path.read_bytes() == report_before + assert combines_path.read_bytes() == combines_before + + +def test_combine_empty_pipelines_list_is_refused(tmp_path: Path) -> None: + """Empty pipelines list is refused and writes nothing.""" + _setup_routed_agentic(tmp_path, decision="agentic") + report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME + combines_path = tmp_path / "metadata" / "agentic_combines.json" + report_before = report_path.read_bytes() + + result = apply_combine_fill(tmp_path, ["parent", "child"], []) + + assert result["ok"] is False + assert "cannot be empty" in result["error"] + assert report_path.read_bytes() == report_before + assert not combines_path.exists() + + +def test_combine_validation_failure_writes_nothing(tmp_path: Path) -> None: + """When structural validation fails, neither the store nor the report is written.""" + _setup_routed_agentic(tmp_path, decision="agentic") + report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME + combines_path = tmp_path / "metadata" / "agentic_combines.json" + report_before = report_path.read_bytes() + + bad_pipeline = copy.deepcopy(_lfc_pipeline()) + bad_pipeline["name"] = "orders_lfc_bad" + bad_task = bad_pipeline["tasks"][0] + bad_task["depends_on"] = [{"task_key": "nonexistent"}] + + result = apply_combine_fill(tmp_path, ["parent", "child"], [bad_pipeline]) + + assert result["ok"] is False + assert "violations" in result + assert report_path.read_bytes() == report_before + assert not combines_path.exists() + + +def test_combine_re_apply_after_reroute_rebuilds_unchanged(tmp_path: Path) -> None: + """After a re-route, re-applying the same combine rebuilds from baseline.""" + _setup_routed_agentic(tmp_path, decision="agentic") + report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME + + first = apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()]) + assert first["ok"] is True + report_after_combine = report_path.read_bytes() + + plan = json.loads((tmp_path / "metadata" / "conversion_plan.json").read_text(encoding="utf-8")) + apply_plan_to_report(tmp_path, plan) + report_after_reroute = report_path.read_bytes() + + assert report_after_combine == report_after_reroute + + +def test_merge_agentic_refuses_routed_agentic_pipeline(tmp_path: Path) -> None: + """merge_agentic refuses a result landing in a routed-agentic pipeline.""" + _setup_routed_agentic(tmp_path, decision="agentic") + apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()]) + + report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME + results_dir = tmp_path / "agentic_results" + _write_merge_result(results_dir, "parent", "Extract", "extract") + + with pytest.raises(ValueError) as exc: + merge_agentic_results(report_path, results_dir) + assert "routed agentic" in str(exc.value) From 2c429916bd3f13daba36aec32c134ed8dc4cb72a Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Wed, 7 Oct 2026 12:12:29 +0100 Subject: [PATCH 027/122] Document the replay model: routing, combine, and preflight behavior Update all documentation to reflect the replay model: - Route rebuilds from immutable baseline, plan, and stored combines (deterministic, idempotent) - Plan is freely editable; re-routes under different decisions are allowed - Combine: same hash is 'already applied, unchanged' (no writes); different hash replaces; validation before any writes (all-or-nothing) - merge_agentic after routing updates both live report and baseline - Package replays rebuild to verify report matches (refuses stale reports) - route_audit.json records decision, outcome, fingerprint, combine_sha256, and replacements Updated documentation in: - AGENTS.md: route_agentic.py description - skills/flowx-route/SKILL.md: routing plan, combine behavior, route_audit fields - skills/flowx-migrate/references/workflow.md: routing flow overview - skills/flowx-convert/SKILL.md: merge_agentic updates baseline after routing - src/flowx/mcp/server.py: route, fill_agentic, merge_agentic tool descriptions Co-authored-by: Isaac --- AGENTS.md | 2 +- skills/flowx-convert/SKILL.md | 2 +- skills/flowx-migrate/references/workflow.md | 27 +++++---- skills/flowx-route/SKILL.md | 67 +++++++++++++-------- src/flowx/mcp/server.py | 36 +++++------ 5 files changed, 77 insertions(+), 57 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index d01aec2f..e70c41c6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,7 +93,7 @@ intermediates under `.work/` (pruned by `package`). | `reporting/results.py` | Writes per-run coverage to a UC table (run_id/run_date/run_by) via the SDK | | `reporting/dashboard.py` | Installs + publishes an AI/BI coverage dashboard over the results table | | `routing.py` | Groups pipelines into connected components over control lineage; recommends deterministic/agentic per component (both options) and records the user's decision as `metadata/conversion_plan.json` (additive; convert untouched). Agentic decisions are ADF-only | -| `route_agentic.py` | Applies a routing decision after convert: rewrites the translation report for agentic-routed groups (placeholder tasks + per-task `AgenticGap`), keeping the fill in-engine | +| `route_agentic.py` | Rebuilds the report from the immutable deterministic baseline, the routing plan, and stored combines; applies combines deterministically and idempotently. | | `models/conversion_plan.py` | Typed `ConversionPlan` (schema 2): the library's load/validate/record API for `conversion_plan.json`, one decision per component, bound to the inventory, `source_graphs.json` and `source_insights.json` hashes; package fails closed on a mismatch | ## Activity Types diff --git a/skills/flowx-convert/SKILL.md b/skills/flowx-convert/SKILL.md index 2586eb4f..3be376e2 100644 --- a/skills/flowx-convert/SKILL.md +++ b/skills/flowx-convert/SKILL.md @@ -70,7 +70,7 @@ across ADF and Airflow. `show_when` condition). Walk it locally; ask an option only when its `show_when` is satisfied. - `modify --output-dir --answer OPTION_ID=VALUE ...` — validate and apply collected answers, writing `.work/translation_report.stamped.json` + `metadata/configuration.json`. -- `merge_agentic --report --agentic-results ` — **ADF only**. Fold agent-produced per-activity translations into an ADF report. It refuses, writing nothing, any result landing in a pipeline the routing record routes agentic (fill those with `fill-agentic combine`). Airflow's legacy name-based merge is disabled. +- `merge_agentic --report --agentic-results ` — **ADF only**. Fold agent-produced per-activity translations into an ADF report, filling convert's own agentic gaps. It refuses, writing nothing, any result landing in a pipeline the routing record routes agentic (fill those with `fill-agentic combine`). After routing, merges into non-routed pipelines are applied to both the live report and the stored immutable baseline (so the next rebuild keeps them) and the routing record's baseline hash is refreshed. Airflow's legacy name-based merge is disabled. ## Output artifacts (shared, transient under `/.work/`) diff --git a/skills/flowx-migrate/references/workflow.md b/skills/flowx-migrate/references/workflow.md index 2ffbee6b..ee8eafd2 100644 --- a/skills/flowx-migrate/references/workflow.md +++ b/skills/flowx-migrate/references/workflow.md @@ -74,18 +74,21 @@ merges: rebuilds `metadata/inventory.json` with it under a single additive `insights` key, leaving every deterministic key byte-identical. Skippable for a deterministic-only, headless pass. - **Route:** groups pipelines into connected components over control lineage and, per component, - records a `deterministic` or `agentic` decision as the fingerprint-bound - `metadata/conversion_plan.json`. Recording an agentic decision edits `.work/translation_report.json` - so those groups' tasks become placeholder gaps and stamps a routing record onto it (per component: - members, decision and outcome, plus the plan and baseline report hashes); a fully-deterministic - plan (or no plan) leaves convert/package behaving exactly as before — the non-breaking guarantee. - Combine and `modify` carry the record forward; re-routing under different decisions is refused - (re-run convert, then route), and package refuses a report whose record no longer matches the plan. - -Routed-agentic groups are then filled only by `fill-agentic combine`, which replaces each group with -authored pipelines (N→M, e.g. a Lakeflow Connect collapse, or one same-named pipeline to stay 1:1), -typically using `AgenticComponentActivity` nodes. `convert --merge-agentic` fills convert's own gaps -and refuses routed-agentic pipelines. + records a freely-editable `deterministic` or `agentic` decision as the fingerprint-bound + `metadata/conversion_plan.json`. Route rebuilds `.work/translation_report.json` and `gaps.json` + from an immutable deterministic baseline, the routing plan, and stored combines: deterministic + components stay as-is; agentic components with a stored combine get the combine applied; agentic + without a combine become placeholders. A fully-deterministic plan (or no plan) leaves the report + byte-identical to the baseline — the non-breaking guarantee. Route stamps a routing record onto + the report (per component: members, decision, outcome, combine hash, and fingerprint) and every + re-route rebuilds deterministically and idempotently from the baseline and current plan. + +Routed-agentic groups are then filled only by `fill-agentic combine`, which applies authored pipelines +(N→M, e.g. a Lakeflow Connect collapse, or one same-named pipeline to stay 1:1), typically using +`AgenticComponentActivity` nodes. Same authored pipelines as already applied returns "already applied, +unchanged" and writes nothing (idempotent). Different pipelines replace and rebuild. `convert --merge-agentic` +fills convert's own gaps and refuses routed-agentic pipelines. After routing, merging convert's own gaps +updates both the live report and the stored baseline so the next rebuild keeps them. ## Phase 2: Convert diff --git a/skills/flowx-route/SKILL.md b/skills/flowx-route/SKILL.md index 5c12e4d4..8de23074 100644 --- a/skills/flowx-route/SKILL.md +++ b/skills/flowx-route/SKILL.md @@ -94,19 +94,24 @@ option's patterns (flag any `has_simplification` re-architecture prominently). ## Step 2 — Decide and record the plan -Take the per-component decision and record it. `route` validates the plan, writes the fingerprint-bound -`metadata/conversion_plan.json`, and then edits `.work/translation_report.json` + `gaps.json` so every -routed-**agentic** group's tasks become placeholder gaps (deterministic groups are left byte-identical; -when nothing is routed agentic the report is untouched — the non-breaking guarantee). +Take the per-component decision and record it. The plan is **freely editable** — change a component's +decision from agentic to deterministic, or vice versa, without restriction. `route` validates the plan, +writes the fingerprint-bound `metadata/conversion_plan.json`, and then rebuilds `.work/translation_report.json` ++ `gaps.json` from the immutable deterministic baseline, the plan, and any stored combines: + +- **deterministic** components: the baseline pipelines stay (deterministic outcome). +- **agentic components with a stored combine** whose members match: the combine's pipelines replace the + members; gaps are dropped (agentic-applied outcome). +- **agentic components without a matching combine**: placeholders and gaps as before (agentic-not-viable). +- **switching back to deterministic**: restores that component's baseline exactly (deterministic outcome, + no record when the whole plan is deterministic — the non-breaking guarantee). When it edits the report, route also stamps a **routing record** onto it (the top-level -`_routing_record` key): the plan's hash, the hash of the deterministic report it started from, and one -entry per `component_id` with its `members`, `decision` and `outcome` — `deterministic`, -`agentic-not-viable` (nothing filled yet) or `agentic-applied` (`fill-agentic combine` filled it). -Every later rewrite (combine and `modify`) carries the record forward and only combine updates -outcomes. Re-running route under the **same** decisions only refreshes the record's -plan hash (in the report and in the stamped copy `modify` wrote); under different components or -decisions it refuses and writes nothing — re-run convert, then route. +`_routing_record` key): the plan's hash, the hashes of the immutable deterministic baseline report and +gaps it started from, and one entry per `component_id` with its `members`, `decision`, `outcome`, +`combine_sha256`, `fingerprint`, and `replacements` (tracking changes to a component's fingerprint +across re-routes). Every re-route rebuilds from the baseline and current plan; identical input gives +identical output (byte-for-byte, deterministic and idempotent). There are three ways to supply the decision: @@ -208,21 +213,24 @@ MCP: `flowx(command="fill_agentic", parameters={"output_dir": ..., "members": [. `source_graphs_sha256` and `source_insights_sha256` must still match the current inventory. A partial group, a superset, a typo, or a deterministic component is refused — you can't swap pipelines the plan didn't route agentic. The report must carry a routing record that matches that plan (route has applied it). -- Combine marks the component `agentic-applied`. Re-running it with the same authored pipelines as - those now in the report is a no-op (`already_combined: true`), even when authored names reuse - member names, so the authored pipelines are never appended twice. A combine with different - authored pipelines on an applied component is refused with "component already filled; re-run - convert and route to start again" and writes nothing. +- **Same authored pipelines** (hashed): `already_combined: true` with the message "already applied, + unchanged"; nothing is written. This is a no-op even after a re-route, allowing idempotent tooling. + If the report no longer reflects the stored combine (e.g. after a re-route with different decisions), + rebuilds from the baseline and stored combines to restore the combine, validating and writing the result. +- **Different authored pipelines**: replaces the stored combine for that component only, rebuilds, + and writes the result (if structural validation passes). The `route_audit.json` records the replacement, + showing the old `fingerprint` in `replacements` and the new `fingerprint`. - `--pipelines-path` is a JSON **list** of pipeline IR dicts (the authored replacements), typically - carrying `AgenticComponentActivity` nodes (see below). + carrying `AgenticComponentActivity` nodes (see below). Empty `pipelines` list is refused. - Each authored pipeline **must** carry the source tag `"tags": {"source": "adf"}` (routing/agentic conversion is ADF-only). Combine asserts this up front and **fails closed** (nothing written, with a clear message) on a missing or non-`adf` tag, so a mis-tagged pipeline is caught here rather than surviving to the package preflight. - The merged report is **always** validated with the structural bundle invariants (a real - `prepare → write_bundle` pass) before it is written — no bypass — so a duplicate key, dangling - dependency, cycle, or dangling pipeline/run_job reference can never land on disk. On any violation, - `ok` is `false`, `violations` lists them, and nothing is written. + `prepare → write_bundle` pass) **before it is written** — no bypass — so a duplicate key, dangling + dependency, cycle, or dangling pipeline/run_job reference can never land on disk. Validation happens + first; if any violation is found, `ok` is `false`, `violations` lists them, and nothing is written + (all-or-nothing). ### Authoring an `AgenticComponentActivity` (the escape hatch) @@ -293,12 +301,19 @@ just-in-time configuration (`inspect`/`modify`) as usual, then `flowx-package`. `metadata/conversion_plan.json` is kept alongside `inventory.json` as the decision of record. It is the library's typed `ConversionPlan` (schema 2): one decision per component, bound to the inventory fingerprint, the saved `source_graphs.json` and the saved `source_insights.json` it was decided on, -with a reserved, empty `assignments` list per component for per-node routing later. Package refuses -to run when any of those no longer match, or when the packaged report's routing record no longer -matches the plan (a different plan hash, different components, members or decisions, or no record -although the plan routes a component agentic). `metadata/route_audit.json` keeps each component's -decision and outcome from the record, the baseline report hash, and the gaps in agentic-routed -pipelines. +with a reserved, empty `assignments` list per component for per-node routing later. + +Package verifies the routing plan is still current against the inventory, then replays `rebuild` +from the baseline (reusing it if it exists), the plan, and stored combines. For an unstamped report +(not through `modify`), it must match the rebuild exactly; otherwise it refuses with "re-run route" +(or "re-run fill-agentic combine" if the combine store changed). For a stamped report from `modify`, +the routing record must match the rebuild; otherwise it refuses with "the configured report is out of +date; re-run modify". + +`metadata/route_audit.json` records per component the `decision`, `outcome`, `fingerprint`, +`combine_sha256`, and `replacements` (changes to the fingerprint across re-routes), plus the hashes +of the plan, the baseline report and gaps, the inventory, and the packaged report. It survives the +prune of `.work/` and documents what was packaged and how. ## Reference diff --git a/src/flowx/mcp/server.py b/src/flowx/mcp/server.py index b4afcf9f..f9edebbf 100644 --- a/src/flowx/mcp/server.py +++ b/src/flowx/mcp/server.py @@ -679,7 +679,9 @@ def flowx(command: str, parameters: dict[str, Any] | None = None) -> dict[str, A - "merge_agentic": source(req: "adf"), report_path(req), agentic_results_dir(req), output_path — merge ADF agent results for convert's own agentic gaps. Refuses (writing nothing) any result landing in a pipeline the routing record routes agentic — fill those with - fill_agentic. Airflow's legacy name-based merge is disabled; use resolve_agentic. + fill_agentic. After routing, merges into non-routed pipelines are applied to both the live + report and the stored immutable baseline (so the next rebuild keeps them) and the routing + record's baseline hash is refreshed. Airflow's legacy name-based merge is disabled; use resolve_agentic. - "resolve_agentic": source(req: "airflow"), action(req: prepare | stage | apply), output_dir, airflow_source_path, report_path, gap_id, candidates, replace, accept_gap | accept_gaps, accept_all, review_complete, review_manifest, reset — @@ -696,24 +698,24 @@ def flowx(command: str, parameters: dict[str, Any] | None = None) -> dict[str, A and routes each. With NO plan it emits the components with BOTH conversion options (deterministic capability + motif/coverage evidence, and the agentic recommended patterns with any simplification pattern surfaced), plus a ready-to-record default plan — the dry run - to read first. With a plan it validates + records metadata/conversion_plan.json AND edits the - translation report so every routed-agentic group's tasks become placeholder gaps - (deterministic groups untouched; convert's deterministic translation is never modified, and - with no agentic decision the report is byte-identical to today). The edit stamps a routing - record onto the report (per component_id: members, decision, outcome; plus the plan and - baseline report hashes) that fill_agentic and apply_answers carry forward; re-routing under - different decisions is refused (re-run convert, then route). Routed-agentic groups are - filled only by fill_agentic. + to read first. With a plan it validates + records metadata/conversion_plan.json AND rebuilds + the translation report from an immutable baseline, the plan, and stored combines: deterministic + components stay as-is, agentic components with a stored combine get it applied, agentic without + a combine become placeholders (with no agentic decision the report stays byte-identical to the + baseline). The plan is freely editable — change any component's route without restriction. + Route stamps a routing record onto the report (per component_id: members, decision, outcome, + combine_sha256, fingerprint, replacements) that fill_agentic carries forward; every re-route + rebuilds deterministically and idempotently from the baseline and current plan. - "fill_agentic": output_dir(req), members(req: list of pipeline names or comma-separated string), one of pipelines(inline list of pipeline IR dicts) | pipelines_path — cross-pipeline - COMBINE fill: replace a routed-agentic group's pipelines with the agent-authored pipeline(s) - (typically AgenticComponentActivity nodes). `members` must exactly match a routed-agentic - component in the recorded, fingerprint-bound conversion_plan.json and the report's routing - record, and the merged report is always validated structurally before writing (no bypass); - the component's outcome becomes agentic-applied. It is the only fill for a routed-agentic - group (to keep a pipeline 1:1, author one same-named pipeline). Re-running it with the same - authored pipelines returns already_combined; different ones are refused ("component already - filled; re-run convert and route to start again"). + COMBINE fill: apply agent-authored pipeline(s) to a routed-agentic component (typically + AgenticComponentActivity nodes). `members` must exactly match a routed-agentic component in + the recorded, fingerprint-bound conversion_plan.json and the report's routing record. Same + authored pipelines as already stored returns already_combined: true (idempotent, no writes). + Different pipelines replace the stored combine and rebuild. The merged report is always + validated structurally before writing (all-or-nothing: any violation and nothing is written). + It is the only fill for a routed-agentic group (to keep a pipeline 1:1, author one + same-named pipeline). - "inspect": report_path(req) — return the full translation-option schema (every option with a `show_when` condition) for the agent to walk locally. See "Collecting options" below. - "apply_answers": report_path(req), answers(req, list of "ID=VALUE"), output_dir, lookup_csv. From 832fa0acaae148933354a02e73e8cc05762bae37 Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Wed, 7 Oct 2026 12:18:58 +0100 Subject: [PATCH 028/122] Check the route audit's per-component and baseline-gaps fields The audit test now compares route_audit.json with the report's routing record: baseline_gaps_sha256, and each component's decision, outcome, fingerprint and combine_sha256. Co-authored-by: Isaac --- tests/unit/test_package_routing_preflight.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/unit/test_package_routing_preflight.py b/tests/unit/test_package_routing_preflight.py index aae5ca47..c96985f9 100644 --- a/tests/unit/test_package_routing_preflight.py +++ b/tests/unit/test_package_routing_preflight.py @@ -235,6 +235,8 @@ def test_package_with_a_current_plan_writes_an_audit_with_hashes(tmp_path: Path) _record(tmp_path, "agentic") plan = json.loads((tmp_path / "metadata" / "conversion_plan.json").read_text(encoding="utf-8")) apply_plan_to_report(tmp_path, plan) + record = routing_record(json.loads(report_path.read_text(encoding="utf-8"))) + assert record is not None assert _package(tmp_path) in (0, 1) # 1 only signals bundle-invariant warnings, not the preflight assert (tmp_path / "databricks.yml").exists() @@ -246,7 +248,14 @@ def test_package_with_a_current_plan_writes_an_audit_with_hashes(tmp_path: Path) == hashlib.sha256((tmp_path / "metadata" / "conversion_plan.json").read_bytes()).hexdigest() ) assert audit["baseline_report_sha256"] == baseline_sha256 + assert audit["baseline_gaps_sha256"] == record["baseline_gaps_sha256"] assert audit["translation_report_sha256"] == hashlib.sha256(report_path.read_bytes()).hexdigest() + (component,) = audit["components"] + recorded = record["components"][component["component_id"]] + assert component["decision"] == "agentic" + assert component["outcome"] == recorded["outcome"] == "agentic-not-viable" + assert component["fingerprint"] == recorded["fingerprint"] + assert component["combine_sha256"] is None def test_package_refuses_tampered_baseline(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: From 1c772db1b116ebac2531fd75eea8b6f7e4bb4341 Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Wed, 7 Oct 2026 12:55:33 +0100 Subject: [PATCH 029/122] no-mistakes(review): Restore baseline on switch-back; verify combines; never rewrite stamped report --- skills/flowx-migrate/SKILL.md | 48 ++- skills/flowx-migrate/references/workflow.md | 6 +- skills/flowx-route/SKILL.md | 43 ++- src/flowx/adapter/__main__.py | 17 +- src/flowx/bundler/dab_writer.py | 92 ++--- src/flowx/ir_serde.py | 127 +++---- src/flowx/route_agentic.py | 363 ++++++++----------- tests/unit/test_package_routing_preflight.py | 59 ++- tests/unit/test_route_agentic_flow.py | 199 +++++++++- 9 files changed, 522 insertions(+), 432 deletions(-) diff --git a/skills/flowx-migrate/SKILL.md b/skills/flowx-migrate/SKILL.md index 2a18ac0e..fad191c3 100644 --- a/skills/flowx-migrate/SKILL.md +++ b/skills/flowx-migrate/SKILL.md @@ -41,8 +41,8 @@ This is the top-level orchestration skill. It runs the full migration pipeline: 5. **Fill agentic gaps** — Replace each routed-agentic group with `fill-agentic combine` (one same-named authored pipeline keeps a pipeline 1:1); `convert --merge-agentic` (ADF only) fills convert's own gaps and refuses routed-agentic pipelines; Airflow gaps go through - `flowx-resolve-airflow-gaps`. **Never re-run a plain `convert` after routing** — it - rewrites the report and erases the placeholders + `flowx-resolve-airflow-gaps`. If you re-run `convert` after routing, re-run `route` next: it + takes the fresh report as its new baseline and re-applies the plan and stored combines 6. **Package** — Generate Databricks Declarative Automation Bundles for deployment Each phase builds on the output of the previous phase. The user is shown a summary and asked to confirm before proceeding to the next phase. @@ -122,8 +122,8 @@ flowx(command="discover", parameters={"source": "adf", "adf_definitions": {...}, flowx(command="enrich", parameters={"output_dir": ..., "insights": {...}}) # default: author + merge the insights layer (see flowx-enrich) flowx(command="convert", parameters={"source": "adf", "output_dir": ..., "pipeline": ...}) # deterministic baseline, BEFORE route flowx(command="route", parameters={"output_dir": ...}) # recommend; re-call with "plan": {...} to record + edit the report (see flowx-route) -flowx(command="merge_agentic", parameters={"source": "adf", "report_path": ..., "agentic_results_dir": ..., "output_path": ...}) # ADF only, per-pipeline agentic fill -flowx(command="fill_agentic", parameters={"output_dir": ..., "members": [...], "pipelines": [...]}) # cross-pipeline combine of a routed-agentic group +flowx(command="merge_agentic", parameters={"source": "adf", "report_path": ..., "agentic_results_dir": ..., "output_path": ...}) # ADF only, fills convert's own gaps +flowx(command="fill_agentic", parameters={"output_dir": ..., "members": [...], "pipelines": [...]}) # the only fill for a routed-agentic group flowx(command="inspect", parameters={"report_path": ...}) flowx(command="apply_answers", parameters={"report_path": ..., "answers": [...], "output_dir": ...}) flowx(command="package", parameters={"output_dir": ..., "catalog": ..., "schema": ...}) @@ -255,9 +255,10 @@ reads and edits this report. Invoke the `flowx:flowx-convert` skill with: - Source path: the original source path (same one discover used) - Output dir: the same shared `` (convert writes its report to `/.work/`) -Run this deterministic convert **exactly once, here, before routing.** The only convert *after* -routing is the additive `convert --merge-agentic` fill in Step 8 — a second plain `convert` would -rewrite `.work/translation_report.json` and erase the routed-agentic placeholders. +Run this deterministic convert **before routing.** If you re-run it after routing (for example with +different options), re-run `route` straight after: a fresh report carries no routing record, so route +takes it as the new deterministic baseline and re-applies the plan and the stored combines. Merges of +convert's own gaps made before that convert must be merged again. Wait for the translation to complete and present the summary: @@ -301,28 +302,37 @@ untouched — non-breaking. If Step 5 was skipped and the report is missing, `route` can trigger the convert phase in-process once (pass `--source` / `--source-path`); it never runs a second plain convert once a report exists. +Re-running `route` is always allowed: it rebuilds the report from the saved deterministic baseline, +the current plan and the stored combines, so switching a component back to deterministic restores +convert's output for it. If the user wants a straight deterministic migration, they accept the all-deterministic recommendation here and the rest of the flow is unchanged (no placeholders, so Step 8 is a no-op). ### Step 8 — Fill routed-agentic gaps If Step 7 routed any component **agentic**, its pipelines' tasks are now `PlaceholderActivity` nodes -with one tagged gap each. Author the replacements and merge them (no LLM in the library — it validates -and merges) **before** the just-in-time config below, so the filled tasks get configuration-stamped +with one tagged gap each. Author the replacement and apply it (no LLM in the library — it validates +and records) **before** the just-in-time config below, so the filled tasks get configuration-stamped and packaged: -- **Per-pipeline** (the pipeline stays 1:1, **ADF only**): author one result JSON per gap and merge - with `convert --source adf --merge-agentic --report .work/translation_report.json --agentic-results ` - (`--source adf` is mandatory; the convert phase exits 2 without it). The merge replaces, per result, - the **first task whose `name` matches `activity_name`** — it matches by name and does **not** check - the target is a placeholder, so make sure each name targets the intended agentic gap. Airflow's - `--merge-agentic` is disabled (exits 2) — for Airflow per-gap fills use the - **`flowx-resolve-airflow-gaps`** skill instead. -- **Cross-pipeline COMBINE** (N pipelines → M, e.g. one Lakeflow Connect pipeline): author the - replacement pipeline IR (typically with `AgenticComponentActivity` nodes) and run +- **COMBINE is the only fill for a routed-agentic group** (N pipelines → M, e.g. one Lakeflow Connect + pipeline, or one same-named pipeline to keep a pipeline 1:1): author the replacement pipeline IR + (typically with `AgenticComponentActivity` nodes) and run `fill-agentic combine --output-dir --members "" --pipelines-path `; the members must exactly match the routed-agentic component and the merged report is validated - structurally before it is written. + structurally before it is written. The authored pipelines are stored in + `metadata/agentic_combines.json`, so a later re-route re-applies them. +- **Convert's own agentic gaps** (in pipelines not routed agentic, **ADF only**): author one result + JSON per gap and merge with + `convert --source adf --merge-agentic --report .work/translation_report.json --agentic-results ` + (`--source adf` is mandatory; the convert phase exits 2 without it). The merge replaces, per result, + the **first task whose `name` matches `activity_name`**, so make sure each name targets the intended + gap. It refuses, writing nothing, any result that lands in a routed-agentic pipeline. After routing + it updates the saved baseline too, so a re-route keeps the merge. Airflow's `--merge-agentic` is + disabled (exits 2) — for Airflow per-gap fills use the **`flowx-resolve-airflow-gaps`** skill. + +Run `modify` (Step 9) after the last route, combine or merge: none of them rewrites modify's +configured report, and package asks you to re-run modify when it is out of date. Skip this step entirely when no component was routed agentic. See the **`flowx-route`** skill for the full fill details and the `AgenticComponentActivity` shape. diff --git a/skills/flowx-migrate/references/workflow.md b/skills/flowx-migrate/references/workflow.md index ee8eafd2..ac9e00e3 100644 --- a/skills/flowx-migrate/references/workflow.md +++ b/skills/flowx-migrate/references/workflow.md @@ -81,14 +81,16 @@ merges: without a combine become placeholders. A fully-deterministic plan (or no plan) leaves the report byte-identical to the baseline — the non-breaking guarantee. Route stamps a routing record onto the report (per component: members, decision, outcome, combine hash, and fingerprint) and every - re-route rebuilds deterministically and idempotently from the baseline and current plan. + re-route rebuilds deterministically and idempotently from the baseline and current plan. A fresh + convert (a report without a routing record) becomes the new baseline on the next route. Routed-agentic groups are then filled only by `fill-agentic combine`, which applies authored pipelines (N→M, e.g. a Lakeflow Connect collapse, or one same-named pipeline to stay 1:1), typically using `AgenticComponentActivity` nodes. Same authored pipelines as already applied returns "already applied, unchanged" and writes nothing (idempotent). Different pipelines replace and rebuild. `convert --merge-agentic` fills convert's own gaps and refuses routed-agentic pipelines. After routing, merging convert's own gaps -updates both the live report and the stored baseline so the next rebuild keeps them. +updates both the live report and the stored baseline so the next rebuild keeps them. Route, combine and +merge never rewrite modify's configured report; package asks to re-run `modify` when it is out of date. ## Phase 2: Convert diff --git a/skills/flowx-route/SKILL.md b/skills/flowx-route/SKILL.md index 8de23074..e6394f11 100644 --- a/skills/flowx-route/SKILL.md +++ b/skills/flowx-route/SKILL.md @@ -51,9 +51,10 @@ discover → enrich (default) → convert (deterministic baseline) → route (de ``` Convert builds the deterministic baseline report **before** routing (route can trigger it in-process -via `--source` / `--source-path`); routing then edits that report. After routing has recorded agentic -placeholders, never run a second plain `convert`: it would overwrite the report and erase the -placeholders. +via `--source` / `--source-path`); routing then edits that report. If you run `convert` again after +routing, run `route` straight after it: a fresh report carries no routing record, so route takes it +as the new baseline and re-applies the plan and the stored combines (merges of convert's own gaps +made before that convert must be merged again). Pipelines are grouped into weak/undirected **connected components** over the inventory's control lineage (`lineage.control_edges`), so mutually-referencing pipelines are decided together and a @@ -101,10 +102,11 @@ writes the fingerprint-bound `metadata/conversion_plan.json`, and then rebuilds - **deterministic** components: the baseline pipelines stay (deterministic outcome). - **agentic components with a stored combine** whose members match: the combine's pipelines replace the - members; gaps are dropped (agentic-applied outcome). + members; `gaps.json` still lists the members' routed tasks (agentic-applied outcome). - **agentic components without a matching combine**: placeholders and gaps as before (agentic-not-viable). -- **switching back to deterministic**: restores that component's baseline exactly (deterministic outcome, - no record when the whole plan is deterministic — the non-breaking guarantee). +- **switching back to deterministic**: restores that component's baseline exactly (deterministic outcome). + When the whole plan is deterministic, route writes the baseline report and gaps back byte for byte, + with no record — the non-breaking guarantee. When it edits the report, route also stamps a **routing record** onto it (the top-level `_routing_record` key): the plan's hash, the hashes of the immutable deterministic baseline report and @@ -171,8 +173,8 @@ the convert phase in-process first. Otherwise run `flowx-convert` before routing [--source adf --source-path ] # only needed to trigger convert when the report is missing ``` -Exit 1 (nothing written) on a missing report it cannot produce, a plan that fails validation, or a -report already routed under different components or decisions. +Exit 1 (nothing written) on a missing report it cannot produce or a plan that fails validation. +Re-routing under different decisions is always allowed; it rebuilds from the baseline. ## Step 3 — Fill the routed-agentic groups @@ -213,13 +215,16 @@ MCP: `flowx(command="fill_agentic", parameters={"output_dir": ..., "members": [. `source_graphs_sha256` and `source_insights_sha256` must still match the current inventory. A partial group, a superset, a typo, or a deterministic component is refused — you can't swap pipelines the plan didn't route agentic. The report must carry a routing record that matches that plan (route has applied it). -- **Same authored pipelines** (hashed): `already_combined: true` with the message "already applied, - unchanged"; nothing is written. This is a no-op even after a re-route, allowing idempotent tooling. - If the report no longer reflects the stored combine (e.g. after a re-route with different decisions), - rebuilds from the baseline and stored combines to restore the combine, validating and writing the result. +- **Same authored pipelines** (the canonical hash covers every field of every authored pipeline): + `already_combined: true` with the message "already applied, unchanged"; nothing is written. This + is a no-op even after a re-route, allowing idempotent tooling. If the report no longer reflects the + stored combine, or the stored entry was edited by hand, combine stores the pipelines again, + rebuilds, validates and writes the result. - **Different authored pipelines**: replaces the stored combine for that component only, rebuilds, and writes the result (if structural validation passes). The `route_audit.json` records the replacement, showing the old `fingerprint` in `replacements` and the new `fingerprint`. +- **Unique names**: an authored pipeline name may reuse a member of this component, but not another + authored name or any other pipeline in the report; a clash is refused and nothing is written. - `--pipelines-path` is a JSON **list** of pipeline IR dicts (the authored replacements), typically carrying `AgenticComponentActivity` nodes (see below). Empty `pipelines` list is refused. - Each authored pipeline **must** carry the source tag `"tags": {"source": "adf"}` (routing/agentic @@ -303,12 +308,14 @@ library's typed `ConversionPlan` (schema 2): one decision per component, bound t fingerprint, the saved `source_graphs.json` and the saved `source_insights.json` it was decided on, with a reserved, empty `assignments` list per component for per-node routing later. -Package verifies the routing plan is still current against the inventory, then replays `rebuild` -from the baseline (reusing it if it exists), the plan, and stored combines. For an unstamped report -(not through `modify`), it must match the rebuild exactly; otherwise it refuses with "re-run route" -(or "re-run fill-agentic combine" if the combine store changed). For a stamped report from `modify`, -the routing record must match the rebuild; otherwise it refuses with "the configured report is out of -date; re-run modify". +Package verifies the routing plan is still current against the inventory, checks the saved baseline +still has the hashes the routing record names ("re-run convert, then route" otherwise) and that no +stored combine was edited ("re-run fill-agentic combine"), then replays `rebuild` from the baseline, +the plan, and the stored combines. The live `.work/translation_report.json` must match the rebuild +exactly, record included; otherwise it refuses with "re-run route". Modify's configured +`.work/translation_report.stamped.json` must carry the rebuilt routing record; otherwise it refuses +with "the configured report is out of date; re-run modify". Route, combine and merge never rewrite +the configured report, so run `modify` again after any of them. `metadata/route_audit.json` records per component the `decision`, `outcome`, `fingerprint`, `combine_sha256`, and `replacements` (changes to the fingerprint across re-routes), plus the hashes diff --git a/src/flowx/adapter/__main__.py b/src/flowx/adapter/__main__.py index b71aeea4..9f5442f1 100644 --- a/src/flowx/adapter/__main__.py +++ b/src/flowx/adapter/__main__.py @@ -177,14 +177,14 @@ def _run_route(args: argparse.Namespace) -> int: findings + a ready-to-record default plan) and exits -- the dry run an agent reads first; * with a **decision** -- ``--plan-path FILE`` (or ``--plan-path -`` to read the plan from stdin), or an interactive TTY prompt -- it validates and records the fingerprint-bound - ``metadata/conversion_plan.json`` and then edits ``.work/translation_report.json`` + - ``gaps.json`` so every routed-agentic group's tasks become placeholder gaps (deterministic - groups untouched; nothing routed agentic leaves the report byte-identical). + ``metadata/conversion_plan.json`` and then rebuilds ``.work/translation_report.json`` + + ``gaps.json`` from the deterministic baseline, so every routed-agentic group's tasks become + placeholder gaps or its stored combine (deterministic groups keep the baseline; nothing routed + agentic leaves convert's report byte-identical). Re-routing under any decisions is allowed. Triggers the convert phase in-process when the report is missing and ``--source`` / - ``--source-path`` are supplied. Returns 1 on a missing report it cannot produce, on a plan that - fails validation, or when the report was already routed under different components or decisions - (report + plan left untouched in each case). + ``--source-path`` are supplied. Returns 1 on a missing report it cannot produce or on a plan that + fails validation (report + plan left untouched in each case). """ from flowx import routing from flowx.models.conversion_plan import ConversionPlan @@ -291,7 +291,7 @@ def _run_fill_agentic(args: argparse.Namespace) -> int: typically carrying ``AgenticComponentActivity`` nodes). The membership must exactly match a routed-agentic component in the recorded, fingerprint-bound ``metadata/conversion_plan.json``, and the merged report is always validated structurally before it is written back -- there is no bypass. - Per-pipeline agentic fills reuse ``convert --merge-agentic`` instead and are not handled here. + It is the only fill for a routed-agentic group; ``convert --merge-agentic`` fills convert's own gaps. """ from flowx.route_agentic import apply_combine_fill @@ -729,7 +729,8 @@ def _build_parser() -> argparse.ArgumentParser: "fill-agentic", help=( "Cross-pipeline COMBINE fill: replace a routed group's pipelines with agent-authored " - "pipeline(s), validated structurally before writing. Per-pipeline fills use convert --merge-agentic." + "pipeline(s), validated structurally before writing. It is the only fill for a routed-agentic " + "group; convert --merge-agentic fills convert's own gaps." ), ) fill_agentic.add_argument("action", choices=("combine",), help="'combine' performs the pipeline-grain fill.") diff --git a/src/flowx/bundler/dab_writer.py b/src/flowx/bundler/dab_writer.py index 1ce0c678..e0b371a8 100644 --- a/src/flowx/bundler/dab_writer.py +++ b/src/flowx/bundler/dab_writer.py @@ -385,20 +385,24 @@ def _routing_plan_failures(output_dir: Path, report_path: Path) -> list[str]: record disagrees with the plan: a different plan hash, different components, members or decisions, or no record although the plan routes a component agentic. - When the plan has an agentic component or the report has a routing record, also verifies that the - immutable baseline exists and its hashes match the record, then replays rebuild to check that the - packaged report matches (or for a stamped report, that its record matches). + When the report has a routing record, also verifies that the saved deterministic baseline still + has the hashes the record names and that no stored combine was edited, then replays the rebuild. + The live ``.work/translation_report.json`` (or any report other than modify's configured copy) + must equal the rebuild, record included ("re-run route"). Modify's configured copy can't be + replayed, because modify changes its pipelines, so its record must equal the rebuilt record + ("re-run modify"). Returns one message per problem; an empty list means there is no recorded plan or it still matches. """ from flowx.discovery_serde import canonical_sha256 from flowx.models.conversion_plan import DECISION_AGENTIC, ConversionPlan from flowx.route_agentic import ( - BASELINE_DIRNAME, - BASELINE_GAPS_FILENAME, - BASELINE_REPORT_FILENAME, + COMBINES_FILENAME, + ROUTING_RECORD_KEY, STAMPED_REPORT_FILENAME, WORK_DIRNAME, + load_baseline, + load_combines, rebuild, routing_record, routing_record_mismatches, @@ -432,7 +436,8 @@ def _routing_plan_failures(output_dir: Path, report_path: Path) -> list[str]: except (OSError, json.JSONDecodeError): return [] report_name = Path(report_path).name - remedy = "re-run convert, then route" + configured = Path(report_path).resolve() == (Path(output_dir) / WORK_DIRNAME / STAMPED_REPORT_FILENAME).resolve() + remedy = "re-run modify" if configured else "re-run route" record = routing_record(report) if record is None: if any(component.decision == DECISION_AGENTIC for component in plan.components): @@ -446,44 +451,26 @@ def _routing_plan_failures(output_dir: Path, report_path: Path) -> list[str]: if failures: return failures - has_agentic = any(component.decision == DECISION_AGENTIC for component in plan.components) - if not (has_agentic or record): - return [] - - work_dir = Path(output_dir) / WORK_DIRNAME - baseline_dir = work_dir / BASELINE_DIRNAME - baseline_report_path = baseline_dir / BASELINE_REPORT_FILENAME - baseline_gaps_path = baseline_dir / BASELINE_GAPS_FILENAME - - if not baseline_report_path.exists() or not baseline_gaps_path.exists(): + try: + baseline_report, baseline_gaps, baseline_report_bytes, baseline_gaps_bytes = load_baseline( + Path(output_dir), fresh=False + ) + except (OSError, ValueError): return ["the deterministic baseline is missing or changed; re-run convert, then route"] - - recorded_report_sha256 = record.get("baseline_report_sha256") - recorded_gaps_sha256 = record.get("baseline_gaps_sha256") - actual_report_sha256 = _file_sha256(baseline_report_path) - actual_gaps_sha256 = _file_sha256(baseline_gaps_path) - - if recorded_report_sha256 != actual_report_sha256 or recorded_gaps_sha256 != actual_gaps_sha256: + recorded_hashes = (record.get("baseline_report_sha256"), record.get("baseline_gaps_sha256")) + actual_hashes = (hashlib.sha256(baseline_report_bytes).hexdigest(), hashlib.sha256(baseline_gaps_bytes).hexdigest()) + if recorded_hashes != actual_hashes: return ["the deterministic baseline is missing or changed; re-run convert, then route"] try: - baseline_report = json.loads(baseline_report_path.read_text(encoding="utf-8")) - baseline_gaps = json.loads(baseline_gaps_path.read_text(encoding="utf-8")) - baseline_report_bytes = baseline_report_path.read_bytes() - baseline_gaps_bytes = baseline_gaps_path.read_bytes() - except (OSError, json.JSONDecodeError) as error: - return [f"Failed to load baseline files: {error}"] - - combines_path = metadata_dir / "agentic_combines.json" - combines = {} - if combines_path.exists(): - try: - combines_raw = json.loads(combines_path.read_text(encoding="utf-8")) - combines = combines_raw if isinstance(combines_raw, dict) else {} - except (OSError, json.JSONDecodeError): - pass + stored, combines = load_combines(Path(output_dir)) + except (OSError, ValueError) as error: + return [f"metadata/{COMBINES_FILENAME} is unreadable: {error}"] + edited = sorted(set(stored) - set(combines)) + if edited: + return [f"the combine store has been edited ({', '.join(edited)}); re-run fill-agentic combine"] - rebuilt_report, rebuilt_gaps, rebuilt_record = rebuild( + rebuilt_report, _, rebuilt_record = rebuild( baseline_report, baseline_gaps, plan_document, @@ -492,26 +479,11 @@ def _routing_plan_failures(output_dir: Path, report_path: Path) -> list[str]: baseline_report_bytes, baseline_gaps_bytes, ) - - stamped_report_path = work_dir / STAMPED_REPORT_FILENAME - if stamped_report_path.exists(): - try: - stamped_report = json.loads(stamped_report_path.read_text(encoding="utf-8")) - stamped_record = routing_record(stamped_report) - if stamped_record != rebuilt_record: - return ["the configured report is out of date; re-run modify"] - except (OSError, json.JSONDecodeError): - pass - else: - report_without_record = {k: v for k, v in report.items() if k != "_routing_record"} - rebuilt_without_record = {k: v for k, v in rebuilt_report.items() if k != "_routing_record"} - report_json = json.dumps(report_without_record, separators=(",", ":"), sort_keys=True, default=str) - rebuilt_json = json.dumps(rebuilt_without_record, separators=(",", ":"), sort_keys=True, default=str) - if report_json != rebuilt_json: - if combines: - return ["the combine store has been edited; re-run fill-agentic combine"] - return [f"{report_name} does not match a fresh rebuild; {remedy}"] - + if configured: + return [] if rebuilt_record == record else ["the configured report is out of date; re-run modify"] + report_without_record = {key: value for key, value in report.items() if key != ROUTING_RECORD_KEY} + if rebuilt_record != record or report_without_record != rebuilt_report: + return [f"{report_name} does not match a fresh rebuild; re-run route"] return [] diff --git a/src/flowx/ir_serde.py b/src/flowx/ir_serde.py index 04c9440e..2f6de272 100644 --- a/src/flowx/ir_serde.py +++ b/src/flowx/ir_serde.py @@ -626,16 +626,6 @@ def _find_and_replace_task(tasks: list[dict[str, Any]], activity_name: str, repl return True -def _merge_target(pipelines: list[dict[str, Any]], wanted: Any, activity_name: Any) -> Any: - """The pipeline a merge result lands in: the one it names, else the first holding its activity.""" - if wanted: - return wanted - for pipeline in pipelines: - if _locate_task(pipeline.get("tasks", []), activity_name) is not None: - return pipeline.get("name") - return None - - def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Path | None = None) -> tuple[int, int]: """Merge agent-produced per-activity translations into a translation report. @@ -653,12 +643,13 @@ def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Pat wrote to the workspace; the prepare phase then references it directly. This fills convert's own agentic gaps. A pipeline the routing record routes agentic is filled - only by ``fill-agentic combine``, so a result that names one, or an untargeted result whose - activity would first land in one, is refused before anything is written. + only by ``fill-agentic combine``, so a result that names one, or an untargeted result that would + land in one, is refused and nothing is written. Results apply one by one in file-name order, and + each is checked against the pipeline it actually lands in. - When the report has a routing record, merges into non-routed pipelines are applied to both the - live report and the stored baseline report (so the next rebuild keeps them), and the record's - baseline_report_sha256 is refreshed. All merges are validated first (all-or-nothing). + When the report has a routing record, every merge is applied to the same pipeline of route's + saved deterministic baseline too, so the next rebuild keeps it, and the record's + ``baseline_report_sha256`` is refreshed. Nothing is written until every result has been applied. Args: report_path: ``translation_report.json`` produced by the translate phase. @@ -670,7 +661,8 @@ def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Pat ``(merged, unmatched)`` counts. Raises: - ValueError: A result would land in a pipeline the routing record routes agentic. + ValueError: A result would land in a pipeline the routing record routes agentic, or the + routing baseline is missing or lacks the task a merge replaced. """ import hashlib @@ -678,28 +670,28 @@ def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Pat BASELINE_DIRNAME, BASELINE_REPORT_FILENAME, ROUTED_AGENTIC_MERGE_REFUSED, - ROUTING_RECORD_KEY, - STAMPED_REPORT_FILENAME, WORK_DIRNAME, + load_baseline, routed_agentic_pipelines, routing_record, ) report = json.loads(report_path.read_text(encoding="utf-8")) pipelines = report["pipelines"] if isinstance(report, dict) and "pipelines" in report else [report] - results = [ - (result_file, json.loads(result_file.read_text(encoding="utf-8"))) - for result_file in sorted(results_dir.glob("*.json")) - ] routed = routed_agentic_pipelines(report) - for _, data in results: - activity_name = data.get("activity_name") or data.get("activity") - if routed and _merge_target(pipelines, data.get("pipeline"), activity_name) in routed: - raise ValueError(ROUTED_AGENTIC_MERGE_REFUSED) + record = routing_record(report) + output_dir = report_path.parent.parent + baseline_report: Any = None + baseline_by_name: dict[Any, dict[str, Any]] = {} + if record is not None: + baseline_report = load_baseline(output_dir, fresh=False)[0] + baseline_pipelines = baseline_report["pipelines"] if "pipelines" in baseline_report else [baseline_report] + baseline_by_name = {pipeline.get("name"): pipeline for pipeline in baseline_pipelines} merged = 0 unmatched = 0 - for result_file, data in results: + for result_file in sorted(results_dir.glob("*.json")): + data = json.loads(result_file.read_text(encoding="utf-8")) activity_name = data.get("activity_name") or data.get("activity") task = data.get("task") or data.get("ir") if not activity_name or not isinstance(task, dict): @@ -707,61 +699,40 @@ def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Pat unmatched += 1 continue wanted = data.get("pipeline") - candidates = [pipeline for pipeline in pipelines if not wanted or pipeline.get("name") == wanted] - if any(_find_and_replace_task(pipeline.get("tasks", []), activity_name, dict(task)) for pipeline in candidates): - merged += 1 - logger.info("Merged agentic result for '%s' from %s", activity_name, result_file.name) - else: + target = next( + ( + pipeline + for pipeline in pipelines + if (not wanted or pipeline.get("name") == wanted) + and _locate_task(pipeline.get("tasks", []), activity_name) is not None + ), + None, + ) + landing = wanted or (target.get("name") if target is not None else None) + if landing in routed: + raise ValueError(ROUTED_AGENTIC_MERGE_REFUSED) + if target is None: logger.warning("No placeholder named '%s' found for %s", activity_name, result_file.name) unmatched += 1 + continue + _find_and_replace_task(target["tasks"], activity_name, dict(task)) + if record is not None: + baseline_pipeline = baseline_by_name.get(target.get("name")) + if baseline_pipeline is None or not _find_and_replace_task( + baseline_pipeline.get("tasks", []), activity_name, dict(task) + ): + raise ValueError( + f"the routing baseline has no task {activity_name!r} in pipeline {target.get('name')!r}; " + "re-run route, then merge again" + ) + merged += 1 + logger.info("Merged agentic result for '%s' from %s", activity_name, result_file.name) + if record is not None and merged: + baseline_bytes = json.dumps(baseline_report, indent=2, default=str).encode("utf-8") + (output_dir / WORK_DIRNAME / BASELINE_DIRNAME / BASELINE_REPORT_FILENAME).write_bytes(baseline_bytes) + record["baseline_report_sha256"] = hashlib.sha256(baseline_bytes).hexdigest() destination = output_path or report_path destination.write_text(json.dumps(report, indent=2, default=str), encoding="utf-8") logger.info("Wrote merged report to %s (%d merged, %d unmatched)", destination, merged, unmatched) - - record = routing_record(report) - if record is not None and merged > 0: - output_dir = report_path.parent.parent - baseline_dir = output_dir / WORK_DIRNAME / BASELINE_DIRNAME - baseline_report_path = baseline_dir / BASELINE_REPORT_FILENAME - - if baseline_report_path.exists(): - try: - baseline_report = json.loads(baseline_report_path.read_text(encoding="utf-8")) - baseline_pipelines = ( - baseline_report["pipelines"] - if isinstance(baseline_report, dict) and "pipelines" in baseline_report - else [baseline_report] - ) - - for result_file in results_dir.glob("*.json"): - data = json.loads(result_file.read_text(encoding="utf-8")) - activity_name = data.get("activity_name") or data.get("activity") - task = data.get("task") or data.get("ir") - if activity_name and isinstance(task, dict): - wanted = data.get("pipeline") - candidates = [ - pipeline for pipeline in baseline_pipelines if not wanted or pipeline.get("name") == wanted - ] - _find_and_replace_task( - [task_item for pipeline in candidates for task_item in pipeline.get("tasks", [])], - activity_name, - dict(task), - ) - - baseline_report_bytes = json.dumps(baseline_report, indent=2, default=str).encode("utf-8") - baseline_report_path.write_bytes(baseline_report_bytes) - - record["baseline_report_sha256"] = hashlib.sha256(baseline_report_bytes).hexdigest() - report[ROUTING_RECORD_KEY] = record - destination.write_text(json.dumps(report, indent=2, default=str), encoding="utf-8") - - stamped_path = report_path.parent / STAMPED_REPORT_FILENAME - if stamped_path.exists(): - stamped_report = json.loads(stamped_path.read_text(encoding="utf-8")) - stamped_report[ROUTING_RECORD_KEY] = record - stamped_path.write_text(json.dumps(stamped_report, indent=2, default=str), encoding="utf-8") - except (OSError, json.JSONDecodeError) as error: - logger.warning("Could not update baseline with merges: %s", error) - return merged, unmatched diff --git a/src/flowx/route_agentic.py b/src/flowx/route_agentic.py index f662438e..af9b0529 100644 --- a/src/flowx/route_agentic.py +++ b/src/flowx/route_agentic.py @@ -8,11 +8,13 @@ pattern similar to :mod:`flowx.agentic`, always keeping the work in-engine so the package phase, structural validation, and provenance all still apply: -* When route runs with at least one agentic component on a fresh convert (no routing record), it saves - an immutable copy of ``.work/translation_report.json`` and ``.work/gaps.json`` to - ``.work/route_baseline/``, records their hashes, and never modifies them. Later routes verify and - reuse the baseline. Combines are stored in ``metadata/agentic_combines.json`` and are idempotent: - the same combine is a no-op, a different combine replaces the stored entry. +* A report without a routing record is a fresh convert. When route first routes a component agentic + from one, it saves an exact copy of ``.work/translation_report.json`` and ``.work/gaps.json`` to + ``.work/route_baseline/`` and records their hashes; later routes rebuild from that copy (only a + merge of convert's own gaps changes it). Combines are stored in ``metadata/agentic_combines.json`` + with a canonical hash of their pipelines and are idempotent: the same combine is a no-op, a + different combine replaces the stored entry. Switching every component back to deterministic writes + the baseline back byte for byte. Route and combine never write modify's configured copy. * :func:`rebuild` is a pure function that takes the immutable baseline, the routing plan, and stored combines, and produces edited report + gaps + routing record. It implements both re-routing and re-applying combines deterministically: every rebuild call with the same inputs produces identical @@ -47,6 +49,7 @@ import os import sys import tempfile +from collections import Counter from collections.abc import Callable, Iterable from pathlib import Path from typing import Any @@ -79,10 +82,6 @@ OUTCOME_AGENTIC_APPLIED = "agentic-applied" OUTCOME_AGENTIC_NOT_VIABLE = "agentic-not-viable" -# Legacy: kept for backward compatibility with tests, but no longer used by route. -REROUTED_UNDER_DIFFERENT_PLAN = "the report was routed under a different plan; re-run convert, then route" - -COMPONENT_ALREADY_FILLED = "component already filled; re-run convert and route to start again" ROUTED_AGENTIC_MERGE_REFUSED = ( "this pipeline is routed agentic; fill it with fill-agentic combine " "(to change the approach, change the plan, re-run convert and route, then combine again)" @@ -130,43 +129,6 @@ def _planned_components(plan: dict[str, Any]) -> dict[str, dict[str, Any]]: } -def build_routing_record( - plan: dict[str, Any], - baseline_report_bytes: bytes, - baseline_gaps_bytes: bytes, - combines: dict[str, dict[str, Any]], -) -> dict[str, Any]: - """Build a routing record for a plan applied to a fresh convert. - - Records the plan hash, baseline file hashes, and per-component information (members, decision, - outcome, combine hash, and fingerprint). - """ - planned = _planned_components(plan) - components: dict[str, dict[str, Any]] = {} - for component_id, entry in planned.items(): - combine_data = combines.get(component_id) - combine_sha256 = combine_data.get("combine_sha256") if combine_data else None - decision = entry["decision"] - outcome = OUTCOME_AGENTIC_NOT_VIABLE if decision == DECISION_AGENTIC else OUTCOME_DETERMINISTIC - fingerprint_input = { - "members": entry["members"], - "decision": decision, - "combine_sha256": combine_sha256, - } - components[component_id] = { - **entry, - "outcome": outcome, - "combine_sha256": combine_sha256, - "fingerprint": canonical_sha256(fingerprint_input), - } - return { - "conversion_plan_sha256": canonical_sha256(plan), - "baseline_report_sha256": hashlib.sha256(baseline_report_bytes).hexdigest(), - "baseline_gaps_sha256": hashlib.sha256(baseline_gaps_bytes).hexdigest(), - "components": components, - } - - def routing_record(report: Any) -> dict[str, Any] | None: """The routing record a report carries, or ``None`` when route has not edited it.""" record = report.get(ROUTING_RECORD_KEY) if isinstance(report, dict) else None @@ -230,42 +192,33 @@ def routed_agentic_pipelines(report: Any) -> set[str]: # --------------------------------------------------------------------------- # -def _baseline_exists(output_dir: Path) -> bool: - """Check whether a baseline has been saved.""" - baseline_dir = Path(output_dir) / WORK_DIRNAME / BASELINE_DIRNAME - return (baseline_dir / BASELINE_REPORT_FILENAME).exists() or (baseline_dir / BASELINE_GAPS_FILENAME).exists() - +def load_baseline(output_dir: Path, *, fresh: bool) -> tuple[dict[str, Any], list[dict[str, Any]], bytes, bytes]: + """Read the deterministic baseline a rebuild starts from: ``(report, gaps, report_bytes, gaps_bytes)``. -def _load_baseline(output_dir: Path) -> tuple[dict[str, Any], list[dict[str, Any]], bytes, bytes, str | None]: - """Load the immutable baseline report and gaps, creating it from current live ones if needed. + With ``fresh`` (the live report carries no routing record, so convert just wrote it) the live + report and gaps *are* the baseline. Otherwise this reads the copy route saved under + ``.work/route_baseline/``. The exact bytes are kept so the routing record can hash them. - Returns ``(report, gaps, report_bytes, gaps_bytes, error)`` where error is set when trying to - load an existing baseline and it fails to read. The bytes are used to compute baseline hashes - for the routing record. + Raises: + ValueError: The saved baseline is missing or is not valid JSON. """ - baseline_dir = Path(output_dir) / WORK_DIRNAME / BASELINE_DIRNAME - baseline_report_path = baseline_dir / BASELINE_REPORT_FILENAME - baseline_gaps_path = baseline_dir / BASELINE_GAPS_FILENAME - if baseline_report_path.exists() or baseline_gaps_path.exists(): - try: - report_bytes = baseline_report_path.read_bytes() if baseline_report_path.exists() else b"{}" - gaps_bytes = baseline_gaps_path.read_bytes() if baseline_gaps_path.exists() else b"[]" - report = json.loads(report_bytes) if report_bytes else {} - gaps = json.loads(gaps_bytes) if gaps_bytes else [] - return report, gaps if isinstance(gaps, list) else [], report_bytes, gaps_bytes, None - except (OSError, json.JSONDecodeError) as error: - return {}, [], b"{}", b"[]", str(error) - work_dir = Path(output_dir) / WORK_DIRNAME - live_report_path = work_dir / REPORT_FILENAME - live_gaps_path = work_dir / GAPS_FILENAME + work = Path(output_dir) / WORK_DIRNAME + if fresh: + report_path, gaps_path = work / REPORT_FILENAME, work / GAPS_FILENAME + else: + report_path = work / BASELINE_DIRNAME / BASELINE_REPORT_FILENAME + gaps_path = work / BASELINE_DIRNAME / BASELINE_GAPS_FILENAME + if not report_path.exists() or not gaps_path.exists(): + raise ValueError("the deterministic baseline is missing; re-run convert, then route") + report_bytes = report_path.read_bytes() + gaps_bytes = gaps_path.read_bytes() if gaps_path.exists() else b"[]" try: - report_bytes = live_report_path.read_bytes() if live_report_path.exists() else b"{}" - gaps_bytes = live_gaps_path.read_bytes() if live_gaps_path.exists() else b"[]" - report = json.loads(report_bytes) if report_bytes else {} - gaps = json.loads(gaps_bytes) if gaps_bytes else [] - return report, gaps if isinstance(gaps, list) else [], report_bytes, gaps_bytes, None - except (OSError, json.JSONDecodeError): - return {}, [], b"{}", b"[]", None + report = json.loads(report_bytes) + gaps = json.loads(gaps_bytes) + except json.JSONDecodeError as error: + message = f"the deterministic baseline is not valid JSON ({error}); re-run convert, then route" + raise ValueError(message) from error + return report, gaps if isinstance(gaps, list) else [], report_bytes, gaps_bytes def _save_baseline_bytes(output_dir: Path, report_bytes: bytes, gaps_bytes: bytes) -> None: @@ -278,16 +231,39 @@ def _save_baseline_bytes(output_dir: Path, report_bytes: bytes, gaps_bytes: byte baseline_gaps_path.write_bytes(gaps_bytes) -def _load_combines(output_dir: Path) -> dict[str, dict[str, Any]] | None: - """Load the agentic combines store, or ``None`` when it doesn't exist.""" +def _combine_checks_out(entry: Any) -> bool: + """Whether a stored combine is as combine wrote it: sorted members, and a hash matching its pipelines.""" + if not isinstance(entry, dict): + return False + members = entry.get("members") + pipelines = entry.get("pipelines") + return ( + isinstance(members, list) + and all(isinstance(member, str) for member in members) + and members == sorted(set(members)) + and isinstance(pipelines, list) + and bool(pipelines) + and entry.get("combine_sha256") == canonical_sha256(pipelines) + ) + + +def load_combines(output_dir: Path) -> tuple[dict[str, Any], dict[str, dict[str, Any]]]: + """Read ``metadata/agentic_combines.json``: every entry as stored, and the entries that check out. + + Route, combine and package all read the store here. A rebuild applies only the entries that check + out, so an edited entry is never applied: package refuses while one exists, and re-running combine + for that component replaces it. Both are empty when there is no store yet. + + Raises: + ValueError: The store is not valid JSON or not a JSON object. + """ combines_path = Path(output_dir) / METADATA_DIRNAME / COMBINES_FILENAME if not combines_path.exists(): - return None - try: - raw = json.loads(combines_path.read_text(encoding="utf-8")) - return raw if isinstance(raw, dict) else None - except (OSError, json.JSONDecodeError): - return None + return {}, {} + stored = json.loads(combines_path.read_text(encoding="utf-8")) + if not isinstance(stored, dict): + raise ValueError(f"{COMBINES_FILENAME} must contain a JSON object") + return stored, {component_id: entry for component_id, entry in stored.items() if _combine_checks_out(entry)} def _save_combines(output_dir: Path, combines: dict[str, dict[str, Any]]) -> None: @@ -361,8 +337,8 @@ def rebuild( plan: dict[str, Any], combines: dict[str, dict[str, Any]] | None, previous_record: dict[str, Any] | None, - baseline_report_bytes: bytes | None = None, - baseline_gaps_bytes: bytes | None = None, + baseline_report_bytes: bytes, + baseline_gaps_bytes: bytes, ) -> tuple[dict[str, Any], list[dict[str, Any]], dict[str, Any]]: """Pure rebuild: derive the routed report, gaps, and record from baseline and plan. @@ -381,12 +357,10 @@ def rebuild( baseline_report: Parsed baseline report JSON. baseline_gaps: Parsed baseline gaps JSON. plan: Routing plan dict. - combines: Stored combines from metadata/agentic_combines.json. + combines: The stored combines that check out (see :func:`load_combines`). previous_record: Prior routing record for change tracking. - baseline_report_bytes: Raw bytes of baseline report file for hashing. If not provided, - computed from baseline_report as compact JSON. - baseline_gaps_bytes: Raw bytes of baseline gaps file for hashing. If not provided, - computed from baseline_gaps as compact JSON. + baseline_report_bytes: Raw bytes of the baseline report file, hashed into the record. + baseline_gaps_bytes: Raw bytes of the baseline gaps file, hashed into the record. """ combines_store = combines or {} planned = _planned_components(plan) @@ -398,10 +372,6 @@ def rebuild( new_report, new_gaps = alter_report(report, gaps, agentic_names) if not isinstance(new_report.get("pipelines"), list): new_report = {"pipelines": [new_report]} - if baseline_report_bytes is None: - baseline_report_bytes = json.dumps(baseline_report, separators=(",", ":")).encode("utf-8") - if baseline_gaps_bytes is None: - baseline_gaps_bytes = json.dumps(baseline_gaps, separators=(",", ":")).encode("utf-8") components: dict[str, dict[str, Any]] = {} for component_id, planned_entry in planned.items(): decision = planned_entry["decision"] @@ -563,58 +533,48 @@ def alter_report( return report, kept_gaps + fresh_gaps -def _write_json_atomic(path: Path, document: Any) -> None: - """Write JSON atomically (temp file + ``os.replace``), matching the report's 2-space indentation.""" +def _write_atomic(path: Path, content: bytes) -> None: + """Write bytes atomically (temp file + ``os.replace``).""" path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_name(f".{path.name}.tmp") - temporary.write_text(json.dumps(document, indent=2, default=str), encoding="utf-8") + temporary.write_bytes(content) os.replace(temporary, path) -def _refresh_record_plan_hash(report_path: Path, plan: dict[str, Any]) -> None: - """Bring a report copy's recorded plan hash up to date when it was routed under these decisions. - - A copy without a record, or routed under other decisions, is left for package to refuse. - """ - if not report_path.exists(): - return - report = json.loads(report_path.read_text(encoding="utf-8")) - record = routing_record(report) - plan_sha256 = canonical_sha256(plan) - if record is None or routing_record_mismatches(record, plan) or record.get("conversion_plan_sha256") == plan_sha256: - return - record["conversion_plan_sha256"] = plan_sha256 - _write_json_atomic(report_path, report) +def _write_json_atomic(path: Path, document: Any) -> None: + """Write JSON atomically, matching the report's 2-space indentation.""" + _write_atomic(path, json.dumps(document, indent=2, default=str).encode("utf-8")) def apply_plan_to_report(output_dir: Path, plan: dict[str, Any]) -> dict[str, Any]: - """Apply a routing decision to the report on disk by rebuilding from baseline. + """Apply a routing decision to the report on disk by rebuilding it from the deterministic baseline. - Reads or creates the immutable baseline at ``.work/route_baseline/``, loads stored combines from - ``metadata/agentic_combines.json``, rebuilds the report + gaps + record deterministically, and - writes them atomically. When no component is routed agentic and no record exists, files are left - untouched (non-breaking guarantee). Re-routing with different decisions rebuilds (not refused). + A report without a routing record is a fresh convert, so it becomes the baseline: the first time + the plan routes a component agentic, route saves it and its gaps to ``.work/route_baseline/``. A + routed report is rebuilt from that saved copy. The rebuild applies the plan and the stored combines + from ``metadata/agentic_combines.json``, and the report + gaps are written atomically. When no + component is routed agentic the baseline bytes are written back unchanged, so switching back to + deterministic restores convert's output exactly and a never-routed report is left untouched. Route + never writes modify's configured copy; package asks for ``modify`` again when it is out of date. Returns a summary dict with the altered pipeline names and the resulting gap count. Raises: FileNotFoundError: when the translation report is missing (run convert first). + ValueError: when a routed report's saved baseline or the combine store cannot be read. """ work = Path(output_dir) / WORK_DIRNAME report_path = work / REPORT_FILENAME + gaps_path = work / GAPS_FILENAME if not report_path.exists(): raise FileNotFoundError(f"No {REPORT_FILENAME} under {work}; run the convert phase first.") - baseline_report, baseline_gaps, baseline_report_bytes, baseline_gaps_bytes, baseline_error = _load_baseline( - output_dir - ) - if baseline_error is not None: - raise ValueError(f"Failed to load baseline: {baseline_error}") - old_report_bytes = report_path.read_bytes() - report = json.loads(old_report_bytes) - old_gaps_bytes = (work / GAPS_FILENAME).read_bytes() if (work / GAPS_FILENAME).exists() else b"[]" + report = json.loads(report_path.read_bytes()) record = routing_record(report) - combines = _load_combines(output_dir) or {} + baseline_report, baseline_gaps, baseline_report_bytes, baseline_gaps_bytes = load_baseline( + output_dir, fresh=record is None + ) + _, combines = load_combines(output_dir) agentic = agentic_pipeline_names(plan) new_report, new_gaps, new_record = rebuild( baseline_report, @@ -627,32 +587,18 @@ def apply_plan_to_report(output_dir: Path, plan: dict[str, Any]) -> dict[str, An ) if not new_record: if record is not None: - report_without_record = copy.deepcopy(report) - report_without_record.pop(ROUTING_RECORD_KEY, None) - _write_json_atomic(report_path, report_without_record) - _write_json_atomic(work / GAPS_FILENAME, baseline_gaps) - if (work / STAMPED_REPORT_FILENAME).exists(): - _write_json_atomic(work / STAMPED_REPORT_FILENAME, report_without_record) + _write_atomic(report_path, baseline_report_bytes) + _write_atomic(gaps_path, baseline_gaps_bytes) return {"agentic_pipelines": sorted(agentic), "gaps": len(baseline_gaps), "altered": record is not None} - if not _baseline_exists(output_dir): + if record is None: _save_baseline_bytes(output_dir, baseline_report_bytes, baseline_gaps_bytes) - new_report[ROUTING_RECORD_KEY] = new_record - old_report_without_record = copy.deepcopy(report) - old_report_without_record.pop(ROUTING_RECORD_KEY, None) + old_gaps_bytes = gaps_path.read_bytes() if gaps_path.exists() else b"[]" + old_report_without_record = {key: value for key, value in report.items() if key != ROUTING_RECORD_KEY} new_gaps_bytes = json.dumps(new_gaps, indent=2, default=str).encode("utf-8") - altered = ( - json.dumps(old_report_without_record, separators=(",", ":"), sort_keys=True, default=str) - != json.dumps( - {k: v for k, v in new_report.items() if k != ROUTING_RECORD_KEY}, - separators=(",", ":"), - sort_keys=True, - default=str, - ) - ) or (old_gaps_bytes != new_gaps_bytes) + altered = old_report_without_record != new_report or old_gaps_bytes != new_gaps_bytes + new_report[ROUTING_RECORD_KEY] = new_record _write_json_atomic(report_path, new_report) - _write_json_atomic(work / GAPS_FILENAME, new_gaps) - if (work / STAMPED_REPORT_FILENAME).exists(): - _write_json_atomic(work / STAMPED_REPORT_FILENAME, new_report) + _write_atomic(gaps_path, new_gaps_bytes) return {"agentic_pipelines": sorted(agentic), "gaps": len(new_gaps), "altered": altered} @@ -660,10 +606,10 @@ def apply_plan(output_dir: Path, plan: ConversionPlan) -> dict[str, Any]: """Apply a recorded, typed plan to the IR after convert: the library's one routing entry point. Checks the plan still matches the inventory, source graphs and source insights it was decided - on, then placeholders the routed-agentic components in the translation report (see - :func:`apply_plan_to_report`); the fills (``convert --merge-agentic`` per pipeline, - ``fill-agentic combine`` across pipelines) then replace those placeholders. Phase 1 decides whole - components, so the reserved per-node assignments are not read here. + on, then rebuilds the translation report from the deterministic baseline (see + :func:`apply_plan_to_report`). Each routed-agentic component is then filled only by + ``fill-agentic combine``; ``convert --merge-agentic`` stays for convert's own gaps. Phase 1 decides + whole components, so the reserved per-node assignments are not read here. Raises: FileNotFoundError: The inventory or translation report is missing. @@ -798,18 +744,18 @@ def apply_combine_fill( Validates the group's membership against the recorded plan, stores the authored pipelines in ``metadata/agentic_combines.json``, and rebuilds (applying it deterministically). The group's membership must exactly match a routed-agentic component in the plan, and every authored pipeline - must carry the correct source tag. + must carry the correct source tag and a name no other pipeline in the rebuilt report uses. Returns ``{"ok", "violations", "error", "component_id", "pipelines", "already_combined", "message"}``. - ``ok`` is ``False`` (and nothing written) on a plan/membership error, source tag violation, - structural validation failure, or any other error. Empty ``authored_pipelines`` is refused. - - When the combine's hash matches the stored one and is already reflected in the live report, - returns ``ok: true, already_combined: true, message: 'already applied, unchanged'`` and writes - nothing. After a re-route, if the report no longer reflects the stored combine, rebuilds from - baseline and stored combines and writes the result. A different hash replaces the stored entry - for that component only and rebuilds. Nothing is written until structural validation passes - (all-or-nothing). + ``ok`` is ``False`` (and nothing written) on a plan/membership error, source tag violation, name + clash, structural validation failure, or any other error. Empty ``authored_pipelines`` is refused. + + The stored entry carries the canonical hash of the complete authored pipelines. When that hash + matches the stored one and the live report already holds those pipelines, this returns + ``ok: true, already_combined: true, message: 'already applied, unchanged'`` and writes nothing. + Otherwise (different pipelines, an edited stored entry, or a report that no longer reflects the + combine) it replaces that component's entry only and rebuilds. Nothing is written until + structural validation passes (all-or-nothing). Modify's configured copy is never written. """ members = {str(member) for member in group_members} if not authored_pipelines: @@ -835,55 +781,43 @@ def apply_combine_fill( error = f"{REPORT_FILENAME} carries no routing record; run `route` to apply the plan before filling." return {"ok": False, "error": error, "violations": [], "pipelines": 0} if routing_record_mismatches(record, plan): - error = "the report was routed under a different plan; re-run convert, then route, then combine again" + error = "the report was routed under a different plan; re-run route, then combine again" return {"ok": False, "error": error, "violations": [], "pipelines": 0} - new_combine_sha256 = canonical_sha256([pipeline_to_store(p) for p in authored_pipelines]) - combines = _load_combines(output_dir) or {} - stored_combine = combines.get(component_id, {}) - stored_sha256 = stored_combine.get("combine_sha256") + combine_sha256 = canonical_sha256(authored_pipelines) + stored, combines = load_combines(output_dir) in_report = {pipeline.get("name"): pipeline for pipeline in _report_pipelines(report)} - - if stored_sha256 == new_combine_sha256: - if all(in_report.get(pipeline.get("name")) == pipeline for pipeline in authored_pipelines): - return { - "ok": True, - "error": None, - "violations": [], - "component_id": component_id, - "pipelines": len(_report_pipelines(report)), - "already_combined": True, - "message": "already applied, unchanged", - } - baseline_report, baseline_gaps, baseline_report_bytes, baseline_gaps_bytes, _ = _load_baseline(output_dir) - new_report, new_gaps, new_record = rebuild( - baseline_report, - baseline_gaps, - plan, - combines, - record, - baseline_report_bytes, - baseline_gaps_bytes, - ) - else: - combines[component_id] = { - "members": list(members), - "pipelines": authored_pipelines, - "combine_sha256": new_combine_sha256, + if (combines.get(component_id) or {}).get("combine_sha256") == combine_sha256 and all( + in_report.get(pipeline.get("name")) == pipeline for pipeline in authored_pipelines + ): + return { + "ok": True, + "error": None, + "violations": [], + "component_id": component_id, + "pipelines": len(_report_pipelines(report)), + "already_combined": True, + "message": "already applied, unchanged", } - baseline_report, baseline_gaps, baseline_report_bytes, baseline_gaps_bytes, _ = _load_baseline(output_dir) - new_report, new_gaps, new_record = rebuild( - baseline_report, - baseline_gaps, - plan, - combines, - record, - baseline_report_bytes, - baseline_gaps_bytes, - ) - if not isinstance(new_report.get("pipelines"), list): - new_report = {"pipelines": [new_report]} + entry = {"members": sorted(members), "pipelines": authored_pipelines, "combine_sha256": combine_sha256} + stored[component_id] = entry + combines[component_id] = entry + baseline_report, baseline_gaps, baseline_report_bytes, baseline_gaps_bytes = load_baseline(output_dir, fresh=False) + new_report, new_gaps, new_record = rebuild( + baseline_report, + baseline_gaps, + plan, + combines, + record, + baseline_report_bytes, + baseline_gaps_bytes, + ) + name_counts = Counter(str(pipeline.get("name")) for pipeline in _report_pipelines(new_report)) + clashes = sorted(name for name, count in name_counts.items() if count > 1) + if clashes: + error = f"authored pipeline names {clashes} clash with another pipeline in the report; give each a unique name" + return {"ok": False, "error": error, "violations": [], "pipelines": 0} new_report[ROUTING_RECORD_KEY] = new_record result = validate_report_structurally(new_report) @@ -891,11 +825,9 @@ def apply_combine_fill( violations = [f"[{finding.code}] {finding.location}: {finding.message}" for finding in result.violations] return {"ok": False, "error": None, "violations": violations, "pipelines": 0} - _save_combines(output_dir, combines) + _save_combines(output_dir, stored) _write_json_atomic(report_path, new_report) _write_json_atomic(work / GAPS_FILENAME, new_gaps) - if (work / STAMPED_REPORT_FILENAME).exists(): - _write_json_atomic(work / STAMPED_REPORT_FILENAME, new_report) return { "ok": True, @@ -907,15 +839,6 @@ def apply_combine_fill( } -def pipeline_to_store(pipeline: dict[str, Any]) -> dict[str, Any]: - """Extract the storable parts of a pipeline IR dict for hashing (excluding dynamic fields).""" - return { - "name": pipeline.get("name"), - "tags": pipeline.get("tags"), - "tasks": pipeline.get("tasks"), - } - - # --------------------------------------------------------------------------- # # Structural validation via the existing bundle invariants. # --------------------------------------------------------------------------- # diff --git a/tests/unit/test_package_routing_preflight.py b/tests/unit/test_package_routing_preflight.py index c96985f9..ee3ec839 100644 --- a/tests/unit/test_package_routing_preflight.py +++ b/tests/unit/test_package_routing_preflight.py @@ -105,7 +105,7 @@ def test_package_refuses_an_agentic_plan_when_the_report_has_no_routing_record( _record(tmp_path, "agentic") # recorded, but never applied to the report assert _package(tmp_path) == 1 - assert "carries no routing record; re-run convert, then route" in capsys.readouterr().err + assert "carries no routing record; re-run route" in capsys.readouterr().err assert not (tmp_path / "databricks.yml").exists() @@ -184,7 +184,9 @@ def test_route_combine_modify_then_package_keeps_the_routing_record(tmp_path: Pa assert audit["translation_report_sha256"] == hashlib.sha256(stamped_path.read_bytes()).hexdigest() -def test_rerouting_the_same_decisions_after_enrich_refreshes_the_stamped_report(tmp_path: Path) -> None: +def test_route_after_modify_leaves_the_stamped_report_and_package_asks_to_re_run_modify( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: report_path = _setup(tmp_path, _inventory("agentic")) plan_path = tmp_path / "plan.json" plan_path.write_text( @@ -194,6 +196,7 @@ def test_rerouting_the_same_decisions_after_enrich_refreshes_the_stamped_report( authored_path = tmp_path / "authored.json" authored_path.write_text(json.dumps([_authored_pipeline()]), encoding="utf-8") route = ["route", "--output-dir", str(tmp_path), "--plan-path", str(plan_path)] + stamped_path = tmp_path / WORK_DIRNAME / "translation_report.stamped.json" assert adapter_main(route) == 0 combine = ["fill-agentic", "combine", "--output-dir", str(tmp_path), "--members", "solo"] @@ -201,8 +204,16 @@ def test_rerouting_the_same_decisions_after_enrich_refreshes_the_stamped_report( assert adapter_main(["modify", str(report_path), "--output-dir", str(tmp_path)]) == 0 assert enrich_inventory(tmp_path, insights={"overview": "Solo loads one table."})["ok"] is True assert _package(tmp_path) == 1 # the plan is now stale against the new insights + stamped_before = stamped_path.read_bytes() + capsys.readouterr() assert adapter_main(route) == 0 + assert stamped_path.read_bytes() == stamped_before + assert _package(tmp_path) == 1 + assert "re-run modify" in capsys.readouterr().err + assert not (tmp_path / "databricks.yml").exists() + + assert adapter_main(["modify", str(report_path), "--output-dir", str(tmp_path)]) == 0 assert _package(tmp_path) == 0 audit = json.loads((tmp_path / "metadata" / "route_audit.json").read_text(encoding="utf-8")) assert audit["components"][0]["outcome"] == "agentic-applied" @@ -211,8 +222,9 @@ def test_rerouting_the_same_decisions_after_enrich_refreshes_the_stamped_report( def test_route_agentic_then_deterministic_switches_back_to_deterministic( tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: - """Re-routing from agentic to deterministic is allowed; the record is cleared (spec change).""" + """Re-routing from agentic to deterministic writes convert's report and gaps back byte for byte.""" report_path = _setup(tmp_path, _inventory("agentic")) + converted = report_path.read_bytes() plan_path = tmp_path / "plan.json" def _route(decision: str) -> int: @@ -225,8 +237,10 @@ def _route(decision: str) -> int: assert _route("deterministic") == 0 capsys.readouterr() - deterministic_report = json.loads(report_path.read_text(encoding="utf-8")) - assert "_routing_record" not in deterministic_report + assert report_path.read_bytes() == converted + baseline_gaps = tmp_path / WORK_DIRNAME / "route_baseline" / "gaps.json" + assert (tmp_path / WORK_DIRNAME / "gaps.json").read_bytes() == baseline_gaps.read_bytes() + assert _package(tmp_path) == 0 def test_package_with_a_current_plan_writes_an_audit_with_hashes(tmp_path: Path) -> None: @@ -287,8 +301,7 @@ def test_package_refuses_stale_unstamped_report(tmp_path: Path, capsys: pytest.C report_path.write_text(json.dumps(report, indent=2), encoding="utf-8") assert _package(tmp_path) == 1 - error = capsys.readouterr().err - assert "does not match" in error or "re-run" in error.lower() + assert "translation_report.json does not match a fresh rebuild; re-run route" in capsys.readouterr().err assert not (tmp_path / "databricks.yml").exists() @@ -328,10 +341,13 @@ def test_package_refuses_edited_combine_store(tmp_path: Path, capsys: pytest.Cap combines_path.write_text(json.dumps(combines, indent=2), encoding="utf-8") assert _package(tmp_path) == 1 - error = capsys.readouterr().err - assert "combine" in error.lower() or "re-run" in error.lower() + assert "the combine store has been edited (component-1); re-run fill-agentic combine" in capsys.readouterr().err assert not (tmp_path / "databricks.yml").exists() + combine = ["fill-agentic", "combine", "--output-dir", str(tmp_path), "--members", "solo"] + assert adapter_main([*combine, "--pipelines-path", str(authored_path)]) == 0 + assert _package(tmp_path) == 0 + def test_package_refuses_stale_stamped_report(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: """Package refuses a stamped report whose record no longer matches the rebuild.""" @@ -373,3 +389,28 @@ def test_package_refuses_stale_stamped_report(tmp_path: Path, capsys: pytest.Cap assert _package(tmp_path) == 1 assert "out of date; re-run modify" in capsys.readouterr().err assert not (tmp_path / "databricks.yml").exists() + + +def test_package_replays_an_explicit_live_report_even_when_a_stamped_copy_exists( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + report_path = _setup(tmp_path, _inventory("agentic")) + plan_path = tmp_path / "plan.json" + plan_path.write_text( + json.dumps({"components": [{"component_id": "component-1", "members": ["solo"], "decision": "agentic"}]}), + encoding="utf-8", + ) + assert adapter_main(["route", "--output-dir", str(tmp_path), "--plan-path", str(plan_path)]) == 0 + assert adapter_main(["modify", str(report_path), "--output-dir", str(tmp_path)]) == 0 + report = json.loads(report_path.read_text(encoding="utf-8")) + report["pipelines"][0]["tasks"] = [] + report_path.write_text(json.dumps(report, indent=2), encoding="utf-8") + capsys.readouterr() + + exit_code = package_main( + ["--output-dir", str(tmp_path), "--report", str(report_path), "--no-download-workspace-files"] + ) + + assert exit_code == 1 + assert "translation_report.json does not match a fresh rebuild; re-run route" in capsys.readouterr().err + assert not (tmp_path / "databricks.yml").exists() diff --git a/tests/unit/test_route_agentic_flow.py b/tests/unit/test_route_agentic_flow.py index 35842c94..59b08da9 100644 --- a/tests/unit/test_route_agentic_flow.py +++ b/tests/unit/test_route_agentic_flow.py @@ -9,10 +9,10 @@ ``PlaceholderActivity`` entries, and appends one ``AgenticGap`` per task to ``gaps.json``. Pipelines in deterministic components are left byte-identical, and with no agentic decision the report and gaps are untouched (the non-breaking guarantee). -* the agent then authors the fill. **Per-pipeline** agentic reuses the existing name-matched - :func:`flowx.ir_serde.merge_agentic_results`. **Cross-pipeline COMBINE** (N pipelines -> M, e.g. one - Lakeflow Connect pipeline) uses a new pipeline-grain fill that swaps the routed group's pipelines - for the agent-authored pipeline(s), which carry ``AgenticComponentActivity`` nodes. +* the agent then authors the fill. A routed-agentic group is filled only by the **cross-pipeline + COMBINE** (N pipelines -> M, e.g. one Lakeflow Connect pipeline), which swaps the group's pipelines + for the agent-authored pipeline(s), typically carrying ``AgenticComponentActivity`` nodes. The + name-matched :func:`flowx.ir_serde.merge_agentic_results` fills only convert's own gaps. * the merged report is validated structurally via the existing :func:`flowx.validate.bundle_invariants.check_bundle_dir` (unique keys, no dangling deps, acyclic, no dangling pipeline/run_job references). @@ -21,7 +21,6 @@ from __future__ import annotations import copy -import inspect import json import re from pathlib import Path @@ -29,6 +28,7 @@ import pytest +from flowx.bundler.dab_writer import main as package_main from flowx.discovery_inventory import STRATEGY_PROPERTY, build_source_inventory from flowx.ir_serde import merge_agentic_results from flowx.models.conversion_plan import ConversionPlan @@ -50,7 +50,7 @@ routing_record, validate_report_structurally, ) -from flowx.routing import record_plan +from flowx.routing import build_recommendation, record_plan from flowx.sources.adf.translate import main as translate_main # --------------------------------------------------------------------------- # @@ -187,6 +187,38 @@ def _setup_routed_agentic(output_dir: Path, *, decision: str = "agentic") -> Non apply_plan(output_dir, recorded) +def _two_component_inventory() -> dict[str, Any]: + """Inventory with no control edge, so 'parent' and 'child' are separate components.""" + parent = SourceGraph(name="parent", source="unit", tasks=[_node("extract", "Copy")]) + child = SourceGraph(name="child", source="unit", tasks=[_node("load", "Script")]) + return build_source_inventory([parent, child], source="adf", source_dir="/tmp/src") + + +def _route_parent_agentic(output_dir: Path, report: dict[str, Any]) -> None: + """Write ``report``, record a plan routing 'parent' agentic and 'child' deterministic, and apply it.""" + _write_work(output_dir, report, gaps=[]) + metadata = output_dir / "metadata" + metadata.mkdir(parents=True, exist_ok=True) + inventory = _two_component_inventory() + (metadata / "inventory.json").write_text(json.dumps(inventory, indent=2), encoding="utf-8") + plan = build_recommendation(inventory)["default_plan"] + for component in plan["components"]: + component["decision"] = "agentic" if component["members"] == ["parent"] else "deterministic" + assert record_plan(output_dir, plan=plan)["ok"] is True + recorded = ConversionPlan.load(output_dir) + assert recorded is not None + apply_plan(output_dir, recorded) + + +def _report_with_child_gap() -> dict[str, Any]: + """The two-pipeline report where convert left 'child''s top-level 'Load' as an agentic gap.""" + report = _report_two_pipelines() + report["pipelines"][1]["tasks"] = [ + {"name": "Load", "task_key": "load", "type": "PlaceholderActivity", "original_type": "Script"} + ] + return report + + def _record_outcomes(report_path: Path) -> dict[str, str]: record = routing_record(json.loads(report_path.read_text(encoding="utf-8"))) assert record is not None @@ -414,11 +446,7 @@ def test_merge_into_a_routed_agentic_pipeline_is_refused(tmp_path: Path, pipelin @pytest.mark.parametrize("pipeline", ["child", None]) def test_merge_still_fills_convert_gaps_in_a_deterministic_pipeline(tmp_path: Path, pipeline: str | None) -> None: - report = _report_two_pipelines() - report["pipelines"][1]["tasks"] = [ - {"name": "Load", "task_key": "load", "type": "PlaceholderActivity", "original_type": "Script"} - ] - _write_work(tmp_path, report, gaps=[]) + _write_work(tmp_path, _report_with_child_gap(), gaps=[]) apply_plan_to_report(tmp_path, _plan(parent="agentic", child="deterministic")) report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME results_dir = tmp_path / "agentic_results" @@ -436,6 +464,52 @@ def test_merge_still_fills_convert_gaps_in_a_deterministic_pipeline(tmp_path: Pa assert _record_outcomes(report_path) == {"component-1": "agentic-not-viable", "component-2": "deterministic"} +def test_a_merged_top_level_gap_survives_a_re_route_and_package_accepts_it( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + report = _report_with_child_gap() + for pipeline in report["pipelines"]: + pipeline["tags"] = {"source": "adf"} + _route_parent_agentic(tmp_path, report) + report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME + results_dir = tmp_path / "agentic_results" + _write_merge_result(results_dir, "child", "Load", "load") + assert merge_agentic_results(report_path, results_dir) == (1, 0) + + recorded = ConversionPlan.load(tmp_path) + assert recorded is not None + assert apply_plan(tmp_path, recorded)["altered"] is False + for path in (report_path, tmp_path / WORK_DIRNAME / "route_baseline" / REPORT_FILENAME): + child = next(p for p in json.loads(path.read_text(encoding="utf-8"))["pipelines"] if p["name"] == "child") + assert child["tasks"][0]["type"] == "NotebookActivity" + capsys.readouterr() + package_main(["--output-dir", str(tmp_path), "--no-download-workspace-files", "--keep-intermediates"]) + assert "preflight failed" not in capsys.readouterr().err + assert (tmp_path / "child" / "databricks.yml").exists() + + +def test_an_untargeted_merge_is_checked_where_it_lands_after_earlier_merges(tmp_path: Path) -> None: + report = { + "pipelines": [ + {"name": "child", "tasks": [{"name": "Script1", "task_key": "s", "type": "PlaceholderActivity"}]}, + {"name": "parent", "tasks": [_copy_task("Script1", "s")]}, + ] + } + _write_work(tmp_path, report, gaps=[]) + apply_plan_to_report(tmp_path, _plan(parent="agentic", child="deterministic")) + report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME + routed_report = report_path.read_bytes() + results_dir = tmp_path / "agentic_results" + results_dir.mkdir() + for file_name, task_name in (("a.json", "Script1_done"), ("b.json", "Script1")): + result = {"activity_name": "Script1", "task": _notebook_task(task_name, "s")} + (results_dir / file_name).write_text(json.dumps(result), encoding="utf-8") + + with pytest.raises(ValueError, match=re.escape(ROUTED_AGENTIC_MERGE_REFUSED)): + merge_agentic_results(report_path, results_dir) + assert report_path.read_bytes() == routed_report + + # --------------------------------------------------------------------------- # # The routing record: re-routing under different decisions is refused. # --------------------------------------------------------------------------- # @@ -452,17 +526,38 @@ def test_route_writes_no_record_when_nothing_is_routed_agentic(tmp_path: Path) - def test_reroute_to_different_decision_rebuilds_the_report(tmp_path: Path) -> None: - """Re-routing with different decisions rebuilds deterministically; no longer refused.""" + """Re-routing with different decisions rebuilds deterministically; switching back restores convert's bytes.""" _write_work(tmp_path, _report_two_pipelines(), gaps=[]) - apply_plan_to_report(tmp_path, _plan(parent="agentic", child="deterministic")) report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME + gaps_path = tmp_path / WORK_DIRNAME / GAPS_FILENAME + converted_report = report_path.read_bytes() + converted_gaps = gaps_path.read_bytes() + apply_plan_to_report(tmp_path, _plan(parent="agentic", child="deterministic")) first = json.loads(report_path.read_text(encoding="utf-8")) first_outcomes = {cid: entry["outcome"] for cid, entry in first[ROUTING_RECORD_KEY]["components"].items()} assert first_outcomes["component-1"] == "agentic-not-viable" assert first_outcomes["component-2"] == "deterministic" + apply_plan_to_report(tmp_path, _plan(parent="deterministic", child="deterministic")) - second = json.loads(report_path.read_text(encoding="utf-8")) - assert ROUTING_RECORD_KEY not in second + + assert report_path.read_bytes() == converted_report + assert gaps_path.read_bytes() == converted_gaps + + +def test_route_takes_a_fresh_convert_as_the_new_baseline(tmp_path: Path) -> None: + _write_work(tmp_path, _report_two_pipelines(), gaps=[]) + apply_plan_to_report(tmp_path, _plan(parent="agentic", child="deterministic")) + reconverted = _report_two_pipelines() + reconverted["pipelines"][1]["tasks"] = [_notebook_task("Load", "load", "/Workspace/Shared/reconverted")] + _write_work(tmp_path, reconverted, gaps=[]) + + apply_plan_to_report(tmp_path, _plan(parent="agentic", child="deterministic")) + + baseline = tmp_path / WORK_DIRNAME / "route_baseline" / REPORT_FILENAME + assert json.loads(baseline.read_text(encoding="utf-8")) == reconverted + report = json.loads((tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_text(encoding="utf-8")) + child = next(p for p in report["pipelines"] if p["name"] == "child") + assert child["tasks"][0]["notebook_path"] == "/Workspace/Shared/reconverted" def test_reroute_under_the_same_decisions_keeps_fills_and_updates_the_plan_hash(tmp_path: Path) -> None: @@ -805,9 +900,77 @@ def test_combine_requires_a_recorded_plan(tmp_path: Path) -> None: assert "conversion_plan.json" in result["error"] -def test_apply_combine_fill_has_no_validation_bypass() -> None: - # There must be no surface that writes the combined report without structural validation. - assert "validate" not in inspect.signature(apply_combine_fill).parameters +def test_an_invalid_replacement_combine_is_refused_and_keeps_the_applied_one(tmp_path: Path) -> None: + _setup_routed_agentic(tmp_path, decision="agentic") + assert apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()])["ok"] is True + report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME + combines_path = tmp_path / "metadata" / "agentic_combines.json" + report_before = report_path.read_bytes() + combines_before = combines_path.read_bytes() + dangling = _named_lfc_pipeline("orders_lfc_v2") + dangling["tasks"][0]["task"] = {"pipeline_task": {"pipeline_id": "${resources.pipelines.ghost.id}"}} + + result = apply_combine_fill(tmp_path, ["parent", "child"], [dangling]) + + assert result["ok"] is False + assert any("dangling_pipeline_reference" in violation for violation in result["violations"]) + assert report_path.read_bytes() == report_before + assert combines_path.read_bytes() == combines_before + + +def test_combine_stores_sorted_members_and_hashes_every_pipeline_field(tmp_path: Path) -> None: + _setup_routed_agentic(tmp_path, decision="agentic") + assert apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()])["ok"] is True + described = _lfc_pipeline() + described["description"] = "Ingest orders through Lakeflow Connect" + + result = apply_combine_fill(tmp_path, ["parent", "child"], [described]) + + assert result["ok"] is True and result["already_combined"] is False + combines = json.loads((tmp_path / "metadata" / "agentic_combines.json").read_text(encoding="utf-8")) + assert combines["component-1"]["members"] == ["child", "parent"] + assert combines["component-1"]["pipelines"] == [described] + report = json.loads((tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_text(encoding="utf-8")) + assert report["pipelines"] == [described] + + +def test_combine_replaces_a_hand_edited_store_entry(tmp_path: Path) -> None: + _setup_routed_agentic(tmp_path, decision="agentic") + assert apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()])["ok"] is True + combines_path = tmp_path / "metadata" / "agentic_combines.json" + combined = combines_path.read_bytes() + edited = json.loads(combined) + edited["component-1"]["pipelines"][0]["tasks"] = [] + combines_path.write_text(json.dumps(edited, indent=2), encoding="utf-8") + + result = apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()]) + + assert result["ok"] is True and result["already_combined"] is False + assert combines_path.read_bytes() == combined + + +@pytest.mark.parametrize( + "authored_names", + [["child"], ["orders_lfc", "orders_lfc"]], + ids=["outside-the-component", "within-the-authored-list"], +) +def test_combine_refuses_authored_names_that_clash(tmp_path: Path, authored_names: list[str]) -> None: + _route_parent_agentic(tmp_path, _report_two_pipelines()) + report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME + report_before = report_path.read_bytes() + authored = [_named_lfc_pipeline(name) for name in authored_names] + for index, pipeline in enumerate(authored): + pipeline["tasks"][0]["resources"][0]["resource_key"] = f"orders_ingestion_{index}" + pipeline["tasks"][0]["task"] = { + "pipeline_task": {"pipeline_id": f"${{resources.pipelines.orders_ingestion_{index}.id}}"} + } + + result = apply_combine_fill(tmp_path, ["parent"], authored) + + assert result["ok"] is False + assert "clash" in result["error"] + assert report_path.read_bytes() == report_before + assert not (tmp_path / "metadata" / "agentic_combines.json").exists() def test_combine_same_hash_returns_already_combined_true_with_unchanged_message(tmp_path: Path) -> None: From 8ef71c2598f4ee9c341cfcc9700b3f60d961bd30 Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Wed, 7 Oct 2026 13:31:27 +0100 Subject: [PATCH 030/122] no-mistakes(review): Check configured reports against live record; tighten combine and merge --- skills/flowx-convert/SKILL.md | 2 +- skills/flowx-convert/sources/adf.md | 4 +- skills/flowx-migrate/SKILL.md | 3 +- skills/flowx-migrate/references/workflow.md | 3 +- skills/flowx-route/SKILL.md | 15 ++-- src/flowx/bundler/dab_writer.py | 86 ++++++++++++-------- src/flowx/ir_serde.py | 22 +++-- src/flowx/mcp/server.py | 7 +- src/flowx/route_agentic.py | 50 ++++++++---- tests/unit/test_package_routing_preflight.py | 46 +++++++++++ tests/unit/test_route_agentic_flow.py | 75 +++++++++++++++-- 11 files changed, 239 insertions(+), 74 deletions(-) diff --git a/skills/flowx-convert/SKILL.md b/skills/flowx-convert/SKILL.md index 3be376e2..214bd008 100644 --- a/skills/flowx-convert/SKILL.md +++ b/skills/flowx-convert/SKILL.md @@ -70,7 +70,7 @@ across ADF and Airflow. `show_when` condition). Walk it locally; ask an option only when its `show_when` is satisfied. - `modify --output-dir --answer OPTION_ID=VALUE ...` — validate and apply collected answers, writing `.work/translation_report.stamped.json` + `metadata/configuration.json`. -- `merge_agentic --report --agentic-results ` — **ADF only**. Fold agent-produced per-activity translations into an ADF report, filling convert's own agentic gaps. It refuses, writing nothing, any result landing in a pipeline the routing record routes agentic (fill those with `fill-agentic combine`). After routing, merges into non-routed pipelines are applied to both the live report and the stored immutable baseline (so the next rebuild keeps them) and the routing record's baseline hash is refreshed. Airflow's legacy name-based merge is disabled. +- `merge_agentic --report --agentic-results ` — **ADF only**. Fold agent-produced per-activity translations into an ADF report, filling convert's own agentic gaps. It refuses, writing nothing, any result landing in a pipeline the routing record routes agentic (fill those with `fill-agentic combine`). After routing, a merge written in place into the live `.work/translation_report.json` is applied to both that report and the stored deterministic baseline (so the next rebuild keeps it) and the routing record's baseline hash is refreshed; a merge written to a separate `--output` copy leaves the baseline alone, so a re-route does not keep it. Airflow's legacy name-based merge is disabled. ## Output artifacts (shared, transient under `/.work/`) diff --git a/skills/flowx-convert/sources/adf.md b/skills/flowx-convert/sources/adf.md index 2a605f33..54e33d84 100644 --- a/skills/flowx-convert/sources/adf.md +++ b/skills/flowx-convert/sources/adf.md @@ -84,7 +84,9 @@ are inherited from the placeholder when omitted, preserving dependency edges. ``` Placeholders are replaced in place, status → `translated`. Exits non-zero if any result can't be -matched. Add `--output ` to write a copy instead of overwriting. +matched. Add `--output ` to write a copy instead of overwriting. After routing, only an +in-place merge into `.work/translation_report.json` also updates route's saved deterministic baseline +(so a re-route keeps it); a merge written to an `--output` copy leaves the baseline untouched. ## Step 6.1 — Just-in-time translation configuration diff --git a/skills/flowx-migrate/SKILL.md b/skills/flowx-migrate/SKILL.md index fad191c3..d117799b 100644 --- a/skills/flowx-migrate/SKILL.md +++ b/skills/flowx-migrate/SKILL.md @@ -328,7 +328,8 @@ and packaged: (`--source adf` is mandatory; the convert phase exits 2 without it). The merge replaces, per result, the **first task whose `name` matches `activity_name`**, so make sure each name targets the intended gap. It refuses, writing nothing, any result that lands in a routed-agentic pipeline. After routing - it updates the saved baseline too, so a re-route keeps the merge. Airflow's `--merge-agentic` is + an in-place merge into `.work/translation_report.json` updates the saved baseline too, so a re-route + keeps the merge (a merge written to an `--output` copy does not). Airflow's `--merge-agentic` is disabled (exits 2) — for Airflow per-gap fills use the **`flowx-resolve-airflow-gaps`** skill. Run `modify` (Step 9) after the last route, combine or merge: none of them rewrites modify's diff --git a/skills/flowx-migrate/references/workflow.md b/skills/flowx-migrate/references/workflow.md index ac9e00e3..ba35e778 100644 --- a/skills/flowx-migrate/references/workflow.md +++ b/skills/flowx-migrate/references/workflow.md @@ -89,7 +89,8 @@ Routed-agentic groups are then filled only by `fill-agentic combine`, which appl `AgenticComponentActivity` nodes. Same authored pipelines as already applied returns "already applied, unchanged" and writes nothing (idempotent). Different pipelines replace and rebuild. `convert --merge-agentic` fills convert's own gaps and refuses routed-agentic pipelines. After routing, merging convert's own gaps -updates both the live report and the stored baseline so the next rebuild keeps them. Route, combine and +in place into `.work/translation_report.json` updates both that report and the stored baseline so the +next rebuild keeps them. Route, combine and merge never rewrite modify's configured report; package asks to re-run `modify` when it is out of date. ## Phase 2: Convert diff --git a/skills/flowx-route/SKILL.md b/skills/flowx-route/SKILL.md index e6394f11..4798222b 100644 --- a/skills/flowx-route/SKILL.md +++ b/skills/flowx-route/SKILL.md @@ -111,8 +111,9 @@ writes the fingerprint-bound `metadata/conversion_plan.json`, and then rebuilds When it edits the report, route also stamps a **routing record** onto it (the top-level `_routing_record` key): the plan's hash, the hashes of the immutable deterministic baseline report and gaps it started from, and one entry per `component_id` with its `members`, `decision`, `outcome`, -`combine_sha256`, `fingerprint`, and `replacements` (tracking changes to a component's fingerprint -across re-routes). Every re-route rebuilds from the baseline and current plan; identical input gives +`combine_sha256` (set only while that component's stored combine is applied, otherwise `null`), +`fingerprint`, and `replacements` (tracking changes to a component's fingerprint across re-routes). +Every re-route rebuilds from the baseline and current plan; identical input gives identical output (byte-for-byte, deterministic and idempotent). There are three ways to supply the decision: @@ -224,7 +225,8 @@ MCP: `flowx(command="fill_agentic", parameters={"output_dir": ..., "members": [. and writes the result (if structural validation passes). The `route_audit.json` records the replacement, showing the old `fingerprint` in `replacements` and the new `fingerprint`. - **Unique names**: an authored pipeline name may reuse a member of this component, but not another - authored name or any other pipeline in the report; a clash is refused and nothing is written. + name in the same list, a member of another component, or a pipeline another component's combine + authored; a clash is refused and nothing is written. - `--pipelines-path` is a JSON **list** of pipeline IR dicts (the authored replacements), typically carrying `AgenticComponentActivity` nodes (see below). Empty `pipelines` list is refused. - Each authored pipeline **must** carry the source tag `"tags": {"source": "adf"}` (routing/agentic @@ -312,9 +314,10 @@ Package verifies the routing plan is still current against the inventory, checks still has the hashes the routing record names ("re-run convert, then route" otherwise) and that no stored combine was edited ("re-run fill-agentic combine"), then replays `rebuild` from the baseline, the plan, and the stored combines. The live `.work/translation_report.json` must match the rebuild -exactly, record included; otherwise it refuses with "re-run route". Modify's configured -`.work/translation_report.stamped.json` must carry the rebuilt routing record; otherwise it refuses -with "the configured report is out of date; re-run modify". Route, combine and merge never rewrite +exactly, record included; otherwise it refuses with "re-run route". Any other packaged report — +modify's configured `.work/translation_report.stamped.json`, or a `modify --out` copy — must carry +the live report's routing record; otherwise it refuses with "the configured report is out of date; +re-run modify". Route, combine and merge never rewrite the configured report, so run `modify` again after any of them. `metadata/route_audit.json` records per component the `decision`, `outcome`, `fingerprint`, diff --git a/src/flowx/bundler/dab_writer.py b/src/flowx/bundler/dab_writer.py index e0b371a8..99b8ec99 100644 --- a/src/flowx/bundler/dab_writer.py +++ b/src/flowx/bundler/dab_writer.py @@ -385,28 +385,17 @@ def _routing_plan_failures(output_dir: Path, report_path: Path) -> list[str]: record disagrees with the plan: a different plan hash, different components, members or decisions, or no record although the plan routes a component agentic. - When the report has a routing record, also verifies that the saved deterministic baseline still - has the hashes the record names and that no stored combine was edited, then replays the rebuild. - The live ``.work/translation_report.json`` (or any report other than modify's configured copy) - must equal the rebuild, record included ("re-run route"). Modify's configured copy can't be - replayed, because modify changes its pipelines, so its record must equal the rebuilt record - ("re-run modify"). + The live ``.work/translation_report.json`` is the report route, combine and merge keep in step + with the saved deterministic baseline, so it is always the one replayed (see + :func:`_live_report_failures`; its remedy is "re-run route"). Any other packaged report -- modify's + configured copy, or one written with ``modify --out`` -- can't be replayed, because modify changes + its pipelines, so its routing record must equal the live one; otherwise package refuses with "the + configured report is out of date; re-run modify". Returns one message per problem; an empty list means there is no recorded plan or it still matches. """ - from flowx.discovery_serde import canonical_sha256 - from flowx.models.conversion_plan import DECISION_AGENTIC, ConversionPlan - from flowx.route_agentic import ( - COMBINES_FILENAME, - ROUTING_RECORD_KEY, - STAMPED_REPORT_FILENAME, - WORK_DIRNAME, - load_baseline, - load_combines, - rebuild, - routing_record, - routing_record_mismatches, - ) + from flowx.models.conversion_plan import ConversionPlan + from flowx.route_agentic import REPORT_FILENAME, WORK_DIRNAME, routing_record from flowx.routing import plan_binding_violations metadata_dir = Path(output_dir) / "metadata" @@ -435,25 +424,60 @@ def _routing_plan_failures(output_dir: Path, report_path: Path) -> list[str]: report = json.loads(Path(report_path).read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): return [] - report_name = Path(report_path).name - configured = Path(report_path).resolve() == (Path(output_dir) / WORK_DIRNAME / STAMPED_REPORT_FILENAME).resolve() - remedy = "re-run modify" if configured else "re-run route" + live_path = Path(output_dir) / WORK_DIRNAME / REPORT_FILENAME + configured = Path(report_path).resolve() != live_path.resolve() + live_report = report + if configured: + try: + live_report = json.loads(live_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + live_report = {} + failures = _live_report_failures(Path(output_dir), plan.to_dict(), live_report) + if failures: + return failures + if configured and routing_record(report) != routing_record(live_report): + return ["the configured report is out of date; re-run modify"] + return [] + + +def _live_report_failures(output_dir: Path, plan_document: dict[str, Any], report: Any) -> list[str]: + """Check the live translation report still equals a fresh replay of the rebuild. + + The report's routing record must match the plan, the saved deterministic baseline must still have + the hashes the record names, and no stored combine may have been edited. The rebuild from the + baseline, the plan and the stored combines must then reproduce the report exactly, record included. + A report without a record passes only when the plan routes nothing agentic. + """ + from flowx.discovery_serde import canonical_sha256 + from flowx.models.conversion_plan import DECISION_AGENTIC + from flowx.route_agentic import ( + COMBINES_FILENAME, + REPORT_FILENAME, + ROUTING_RECORD_KEY, + load_baseline, + load_combines, + rebuild, + routing_record, + routing_record_mismatches, + ) + record = routing_record(report) if record is None: - if any(component.decision == DECISION_AGENTIC for component in plan.components): - return [f"the plan routes a component agentic but {report_name} carries no routing record; {remedy}"] + if any(component.get("decision") == DECISION_AGENTIC for component in plan_document.get("components", [])): + return [ + f"the plan routes a component agentic but {REPORT_FILENAME} carries no routing record; re-run route" + ] return [] - plan_document = plan.to_dict() failures: list[str] = [] if record.get("conversion_plan_sha256") != canonical_sha256(plan_document): - failures.append(f"{report_name} was routed under a different conversion plan; {remedy}") - failures.extend(f"{mismatch}; {remedy}" for mismatch in routing_record_mismatches(record, plan_document)) + failures.append(f"{REPORT_FILENAME} was routed under a different conversion plan; re-run route") + failures.extend(f"{mismatch}; re-run route" for mismatch in routing_record_mismatches(record, plan_document)) if failures: return failures try: baseline_report, baseline_gaps, baseline_report_bytes, baseline_gaps_bytes = load_baseline( - Path(output_dir), fresh=False + output_dir, fresh=False ) except (OSError, ValueError): return ["the deterministic baseline is missing or changed; re-run convert, then route"] @@ -463,7 +487,7 @@ def _routing_plan_failures(output_dir: Path, report_path: Path) -> list[str]: return ["the deterministic baseline is missing or changed; re-run convert, then route"] try: - stored, combines = load_combines(Path(output_dir)) + stored, combines = load_combines(output_dir) except (OSError, ValueError) as error: return [f"metadata/{COMBINES_FILENAME} is unreadable: {error}"] edited = sorted(set(stored) - set(combines)) @@ -479,11 +503,9 @@ def _routing_plan_failures(output_dir: Path, report_path: Path) -> list[str]: baseline_report_bytes, baseline_gaps_bytes, ) - if configured: - return [] if rebuilt_record == record else ["the configured report is out of date; re-run modify"] report_without_record = {key: value for key, value in report.items() if key != ROUTING_RECORD_KEY} if rebuilt_record != record or report_without_record != rebuilt_report: - return [f"{report_name} does not match a fresh rebuild; re-run route"] + return [f"{REPORT_FILENAME} does not match a fresh rebuild; re-run route"] return [] diff --git a/src/flowx/ir_serde.py b/src/flowx/ir_serde.py index 2f6de272..f23bc66e 100644 --- a/src/flowx/ir_serde.py +++ b/src/flowx/ir_serde.py @@ -647,9 +647,12 @@ def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Pat land in one, is refused and nothing is written. Results apply one by one in file-name order, and each is checked against the pipeline it actually lands in. - When the report has a routing record, every merge is applied to the same pipeline of route's - saved deterministic baseline too, so the next rebuild keeps it, and the record's - ``baseline_report_sha256`` is refreshed. Nothing is written until every result has been applied. + When the report has a routing record and the merge is written in place into the live + ``/.work/translation_report.json``, every merge is applied to the same pipeline of + route's saved deterministic baseline too, so the next rebuild keeps it, and the record's + ``baseline_report_sha256`` is refreshed. A merge written anywhere else (``output_path``, or into + a copy such as modify's configured report) leaves the baseline alone. Nothing is written until + every result has been applied. Args: report_path: ``translation_report.json`` produced by the translate phase. @@ -669,6 +672,7 @@ def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Pat from flowx.route_agentic import ( BASELINE_DIRNAME, BASELINE_REPORT_FILENAME, + REPORT_FILENAME, ROUTED_AGENTIC_MERGE_REFUSED, WORK_DIRNAME, load_baseline, @@ -681,9 +685,14 @@ def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Pat routed = routed_agentic_pipelines(report) record = routing_record(report) output_dir = report_path.parent.parent + destination = output_path or report_path + live_report_path = output_dir / WORK_DIRNAME / REPORT_FILENAME + updates_baseline = ( + record is not None and destination.resolve() == report_path.resolve() == live_report_path.resolve() + ) baseline_report: Any = None baseline_by_name: dict[Any, dict[str, Any]] = {} - if record is not None: + if updates_baseline: baseline_report = load_baseline(output_dir, fresh=False)[0] baseline_pipelines = baseline_report["pipelines"] if "pipelines" in baseline_report else [baseline_report] baseline_by_name = {pipeline.get("name"): pipeline for pipeline in baseline_pipelines} @@ -716,7 +725,7 @@ def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Pat unmatched += 1 continue _find_and_replace_task(target["tasks"], activity_name, dict(task)) - if record is not None: + if updates_baseline: baseline_pipeline = baseline_by_name.get(target.get("name")) if baseline_pipeline is None or not _find_and_replace_task( baseline_pipeline.get("tasks", []), activity_name, dict(task) @@ -728,11 +737,10 @@ def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Pat merged += 1 logger.info("Merged agentic result for '%s' from %s", activity_name, result_file.name) - if record is not None and merged: + if record is not None and updates_baseline and merged: baseline_bytes = json.dumps(baseline_report, indent=2, default=str).encode("utf-8") (output_dir / WORK_DIRNAME / BASELINE_DIRNAME / BASELINE_REPORT_FILENAME).write_bytes(baseline_bytes) record["baseline_report_sha256"] = hashlib.sha256(baseline_bytes).hexdigest() - destination = output_path or report_path destination.write_text(json.dumps(report, indent=2, default=str), encoding="utf-8") logger.info("Wrote merged report to %s (%d merged, %d unmatched)", destination, merged, unmatched) return merged, unmatched diff --git a/src/flowx/mcp/server.py b/src/flowx/mcp/server.py index f9edebbf..88c0d395 100644 --- a/src/flowx/mcp/server.py +++ b/src/flowx/mcp/server.py @@ -679,9 +679,10 @@ def flowx(command: str, parameters: dict[str, Any] | None = None) -> dict[str, A - "merge_agentic": source(req: "adf"), report_path(req), agentic_results_dir(req), output_path — merge ADF agent results for convert's own agentic gaps. Refuses (writing nothing) any result landing in a pipeline the routing record routes agentic — fill those with - fill_agentic. After routing, merges into non-routed pipelines are applied to both the live - report and the stored immutable baseline (so the next rebuild keeps them) and the routing - record's baseline hash is refreshed. Airflow's legacy name-based merge is disabled; use resolve_agentic. + fill_agentic. After routing, a merge written in place into the live + .work/translation_report.json also updates the stored deterministic baseline (so the next + rebuild keeps it) and the routing record's baseline hash; a merge written to output_path + leaves the baseline alone. Airflow's legacy name-based merge is disabled; use resolve_agentic. - "resolve_agentic": source(req: "airflow"), action(req: prepare | stage | apply), output_dir, airflow_source_path, report_path, gap_id, candidates, replace, accept_gap | accept_gaps, accept_all, review_complete, review_manifest, reset — diff --git a/src/flowx/route_agentic.py b/src/flowx/route_agentic.py index af9b0529..9864b672 100644 --- a/src/flowx/route_agentic.py +++ b/src/flowx/route_agentic.py @@ -23,9 +23,9 @@ * The agent authors the fill. A routed-agentic component is filled **only** by the cross-pipeline COMBINE (N pipelines -> M, e.g. one Lakeflow Connect pipeline, or one same-named pipeline to keep it 1:1): stored in ``metadata/agentic_combines.json`` and applied via rebuild. The name-matched - :func:`flowx.ir_serde.merge_agentic_results` stays for convert's own gaps (after routing, merges - update both the live report and the stored baseline so the next rebuild keeps them) and refuses - routed-agentic pipelines. + :func:`flowx.ir_serde.merge_agentic_results` stays for convert's own gaps (after routing, a merge + written in place into the live report also updates the stored baseline so the next rebuild keeps + it) and refuses routed-agentic pipelines. * :func:`validate_report_structurally` packages the merged report through the same ``prepare -> write_bundle`` path the package phase uses and runs the existing :func:`flowx.validate.bundle_invariants.check_bundle_dir` over the output, so a fill can never @@ -49,7 +49,6 @@ import os import sys import tempfile -from collections import Counter from collections.abc import Callable, Iterable from pathlib import Path from typing import Any @@ -60,8 +59,6 @@ # The report + gaps live under the shared output dir's transient .work/ folder, beside the pipeline IR. WORK_DIRNAME = ".work" REPORT_FILENAME = "translation_report.json" -# The configuration-stamped copy modify writes beside the report; package reads it when present. -STAMPED_REPORT_FILENAME = "translation_report.stamped.json" GAPS_FILENAME = "gaps.json" # The immutable baseline copies, saved on first route with agentic components. BASELINE_DIRNAME = "route_baseline" @@ -349,6 +346,9 @@ def rebuild( * **agentic without a matching combine:** placeholders and tagged gaps as ``alter_report`` does. Outcome is ``agentic-not-viable``. + Only a combine that is applied puts its ``combine_sha256`` in the record and the fingerprint; every + other component records ``None``, whatever the store holds for it. + The record holds per-component members, decision, outcome, combine_sha256, and fingerprint. When fingerprint differs from previous_record's, a replacement entry is appended to track the change. An all-deterministic plan returns no record (byte-identical to baseline). @@ -376,7 +376,12 @@ def rebuild( for component_id, planned_entry in planned.items(): decision = planned_entry["decision"] members_set = set(planned_entry["members"]) - combine_data = combines_store.get(component_id) + stored_combine = combines_store.get(component_id) + combine_data = ( + stored_combine + if decision == DECISION_AGENTIC and stored_combine and set(stored_combine.get("members", [])) == members_set + else None + ) combine_sha256 = combine_data.get("combine_sha256") if combine_data else None fingerprint_input = { "members": planned_entry["members"], @@ -384,7 +389,7 @@ def rebuild( "combine_sha256": combine_sha256, } fingerprint = canonical_sha256(fingerprint_input) - if decision == DECISION_AGENTIC and combine_data and set(combine_data.get("members", [])) == members_set: + if combine_data: outcome = OUTCOME_AGENTIC_APPLIED new_report = combine_group_fill(new_report, members_set, combine_data.get("pipelines", [])) elif decision == DECISION_AGENTIC: @@ -744,7 +749,9 @@ def apply_combine_fill( Validates the group's membership against the recorded plan, stores the authored pipelines in ``metadata/agentic_combines.json``, and rebuilds (applying it deterministically). The group's membership must exactly match a routed-agentic component in the plan, and every authored pipeline - must carry the correct source tag and a name no other pipeline in the rebuilt report uses. + must carry the correct source tag and a name of its own: no repeat within the list, no member of + another component, and no pipeline another component's combine authored. A name may reuse a member + of this component, since the combine replaces it. Returns ``{"ok", "violations", "error", "component_id", "pipelines", "already_combined", "message"}``. ``ok`` is ``False`` (and nothing written) on a plan/membership error, source tag violation, name @@ -786,6 +793,26 @@ def apply_combine_fill( combine_sha256 = canonical_sha256(authored_pipelines) stored, combines = load_combines(output_dir) + authored_names = [pipeline.get("name") for pipeline in authored_pipelines] + taken_names = { + member + for component in plan.get("components", []) + if isinstance(component, dict) and component.get("component_id") != component_id + for member in component.get("members", []) + } | { + pipeline.get("name") + for other_id, entry in combines.items() + if other_id != component_id + for pipeline in entry["pipelines"] + if isinstance(pipeline, dict) + } + clashes = sorted({str(name) for name in authored_names if name in taken_names or authored_names.count(name) > 1}) + if clashes: + error = ( + f"authored pipeline names {clashes} repeat or clash with a pipeline of another component; " + "give each a unique name" + ) + return {"ok": False, "error": error, "violations": [], "pipelines": 0} in_report = {pipeline.get("name"): pipeline for pipeline in _report_pipelines(report)} if (combines.get(component_id) or {}).get("combine_sha256") == combine_sha256 and all( in_report.get(pipeline.get("name")) == pipeline for pipeline in authored_pipelines @@ -813,11 +840,6 @@ def apply_combine_fill( baseline_report_bytes, baseline_gaps_bytes, ) - name_counts = Counter(str(pipeline.get("name")) for pipeline in _report_pipelines(new_report)) - clashes = sorted(name for name, count in name_counts.items() if count > 1) - if clashes: - error = f"authored pipeline names {clashes} clash with another pipeline in the report; give each a unique name" - return {"ok": False, "error": error, "violations": [], "pipelines": 0} new_report[ROUTING_RECORD_KEY] = new_record result = validate_report_structurally(new_report) diff --git a/tests/unit/test_package_routing_preflight.py b/tests/unit/test_package_routing_preflight.py index ee3ec839..c287fe01 100644 --- a/tests/unit/test_package_routing_preflight.py +++ b/tests/unit/test_package_routing_preflight.py @@ -414,3 +414,49 @@ def test_package_replays_an_explicit_live_report_even_when_a_stamped_copy_exists assert exit_code == 1 assert "translation_report.json does not match a fresh rebuild; re-run route" in capsys.readouterr().err assert not (tmp_path / "databricks.yml").exists() + + +def _route_solo_agentic_and_modify(output_dir: Path, report_path: Path, *modify_args: str) -> list[str]: + """Route 'solo' agentic through the CLI, then run modify; returns the route command for re-runs.""" + plan_path = output_dir / "plan.json" + plan_path.write_text( + json.dumps({"components": [{"component_id": "component-1", "members": ["solo"], "decision": "agentic"}]}), + encoding="utf-8", + ) + route = ["route", "--output-dir", str(output_dir), "--plan-path", str(plan_path)] + assert adapter_main(route) == 0 + assert adapter_main(["modify", str(report_path), "--output-dir", str(output_dir), *modify_args]) == 0 + return route + + +def test_package_asks_to_re_run_modify_after_a_re_convert_and_re_route( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + report_path = _setup(tmp_path, _inventory("agentic")) + route = _route_solo_agentic_and_modify(tmp_path, report_path) + _setup(tmp_path, _inventory("agentic")) + report = json.loads(report_path.read_text(encoding="utf-8")) + report["pipelines"][0]["tasks"][0]["notebook_path"] = "/Workspace/Shared/reconverted" + report_path.write_text(json.dumps(report, indent=2), encoding="utf-8") + assert adapter_main(route) == 0 + capsys.readouterr() + + assert _package(tmp_path) == 1 + assert "the configured report is out of date; re-run modify" in capsys.readouterr().err + assert not (tmp_path / "databricks.yml").exists() + + assert adapter_main(["modify", str(report_path), "--output-dir", str(tmp_path)]) == 0 + assert _package(tmp_path) == 0 + + +def test_package_checks_a_modify_out_copy_by_its_routing_record(tmp_path: Path) -> None: + report_path = _setup(tmp_path, _inventory("agentic")) + configured_path = tmp_path / "configured.json" + _route_solo_agentic_and_modify(tmp_path, report_path, "--out", str(configured_path)) + + exit_code = package_main( + ["--output-dir", str(tmp_path), "--report", str(configured_path), "--no-download-workspace-files"] + ) + + assert exit_code == 0 + assert (tmp_path / "databricks.yml").exists() diff --git a/tests/unit/test_route_agentic_flow.py b/tests/unit/test_route_agentic_flow.py index 59b08da9..3b19da3f 100644 --- a/tests/unit/test_route_agentic_flow.py +++ b/tests/unit/test_route_agentic_flow.py @@ -194,22 +194,27 @@ def _two_component_inventory() -> dict[str, Any]: return build_source_inventory([parent, child], source="adf", source_dir="/tmp/src") -def _route_parent_agentic(output_dir: Path, report: dict[str, Any]) -> None: - """Write ``report``, record a plan routing 'parent' agentic and 'child' deterministic, and apply it.""" - _write_work(output_dir, report, gaps=[]) - metadata = output_dir / "metadata" - metadata.mkdir(parents=True, exist_ok=True) - inventory = _two_component_inventory() - (metadata / "inventory.json").write_text(json.dumps(inventory, indent=2), encoding="utf-8") +def _route_two_components(output_dir: Path, *, parent: str, child: str) -> None: + """Record a plan deciding 'parent' and 'child' (each its own component) as given, and apply it.""" + inventory = json.loads((output_dir / "metadata" / "inventory.json").read_text(encoding="utf-8")) plan = build_recommendation(inventory)["default_plan"] for component in plan["components"]: - component["decision"] = "agentic" if component["members"] == ["parent"] else "deterministic" + component["decision"] = parent if component["members"] == ["parent"] else child assert record_plan(output_dir, plan=plan)["ok"] is True recorded = ConversionPlan.load(output_dir) assert recorded is not None apply_plan(output_dir, recorded) +def _route_parent_agentic(output_dir: Path, report: dict[str, Any], *, child: str = "deterministic") -> None: + """Write ``report`` and the two-component inventory, then route 'parent' agentic and 'child' as given.""" + _write_work(output_dir, report, gaps=[]) + metadata = output_dir / "metadata" + metadata.mkdir(parents=True, exist_ok=True) + (metadata / "inventory.json").write_text(json.dumps(_two_component_inventory(), indent=2), encoding="utf-8") + _route_two_components(output_dir, parent="agentic", child=child) + + def _report_with_child_gap() -> dict[str, Any]: """The two-pipeline report where convert left 'child''s top-level 'Load' as an agentic gap.""" report = _report_two_pipelines() @@ -488,6 +493,26 @@ def test_a_merged_top_level_gap_survives_a_re_route_and_package_accepts_it( assert (tmp_path / "child" / "databricks.yml").exists() +def test_a_merge_written_to_a_separate_copy_leaves_the_baseline_and_live_report_alone(tmp_path: Path) -> None: + _route_parent_agentic(tmp_path, _report_with_child_gap()) + report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME + baseline_path = tmp_path / WORK_DIRNAME / "route_baseline" / REPORT_FILENAME + live_before = report_path.read_bytes() + baseline_before = baseline_path.read_bytes() + results_dir = tmp_path / "agentic_results" + _write_merge_result(results_dir, "child", "Load", "load") + copy_path = tmp_path / WORK_DIRNAME / "merged.json" + + assert merge_agentic_results(report_path, results_dir, copy_path) == (1, 0) + + assert report_path.read_bytes() == live_before + assert baseline_path.read_bytes() == baseline_before + merged_copy = json.loads(copy_path.read_text(encoding="utf-8")) + child = next(p for p in merged_copy["pipelines"] if p["name"] == "child") + assert child["tasks"][0]["type"] == "NotebookActivity" + assert routing_record(merged_copy) == routing_record(json.loads(live_before)) + + def test_an_untargeted_merge_is_checked_where_it_lands_after_earlier_merges(tmp_path: Path) -> None: report = { "pipelines": [ @@ -949,6 +974,40 @@ def test_combine_replaces_a_hand_edited_store_entry(tmp_path: Path) -> None: assert combines_path.read_bytes() == combined +def test_a_combine_routed_back_to_deterministic_records_no_combine_hash(tmp_path: Path) -> None: + _route_parent_agentic(tmp_path, _report_two_pipelines(), child="agentic") + assert apply_combine_fill(tmp_path, ["parent"], [_named_lfc_pipeline("parent_lfc")])["ok"] is True + + _route_two_components(tmp_path, parent="deterministic", child="agentic") + + report = json.loads((tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_text(encoding="utf-8")) + record = routing_record(report) + assert record is not None + (parent_entry,) = [entry for entry in record["components"].values() if entry["members"] == ["parent"]] + assert parent_entry["outcome"] == "deterministic" + assert parent_entry["combine_sha256"] is None + assert [pipeline["name"] for pipeline in report["pipelines"]] == ["parent", "child"] + + +def test_combine_refuses_a_name_that_another_components_combine_would_drop(tmp_path: Path) -> None: + _route_parent_agentic(tmp_path, _report_two_pipelines(), child="agentic") + child_lfc = _named_lfc_pipeline("child_lfc") + child_lfc["tasks"][0]["resources"][0]["resource_key"] = "child_ingestion" + child_lfc["tasks"][0]["task"] = {"pipeline_task": {"pipeline_id": "${resources.pipelines.child_ingestion.id}"}} + assert apply_combine_fill(tmp_path, ["child"], [child_lfc])["ok"] is True + report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME + combines_path = tmp_path / "metadata" / "agentic_combines.json" + report_before = report_path.read_bytes() + combines_before = combines_path.read_bytes() + + result = apply_combine_fill(tmp_path, ["parent"], [_named_lfc_pipeline("child")]) + + assert result["ok"] is False + assert "clash" in result["error"] + assert report_path.read_bytes() == report_before + assert combines_path.read_bytes() == combines_before + + @pytest.mark.parametrize( "authored_names", [["child"], ["orders_lfc", "orders_lfc"]], From 7cc139fb5d44c4f58b6e62c3049df254cf32d933 Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Wed, 7 Oct 2026 14:21:26 +0100 Subject: [PATCH 031/122] no-mistakes(document): Document route skill, MCP routing commands and routing artifacts --- AGENTS.md | 2 +- README.md | 3 +++ app/README.md | 3 +++ docs/content/docs/architecture.mdx | 2 +- docs/content/docs/installation.mdx | 2 +- src/flowx/bundler/dab_writer.py | 13 +++++++------ 6 files changed, 16 insertions(+), 9 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e70c41c6..7f04998d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,7 +93,7 @@ intermediates under `.work/` (pruned by `package`). | `reporting/results.py` | Writes per-run coverage to a UC table (run_id/run_date/run_by) via the SDK | | `reporting/dashboard.py` | Installs + publishes an AI/BI coverage dashboard over the results table | | `routing.py` | Groups pipelines into connected components over control lineage; recommends deterministic/agentic per component (both options) and records the user's decision as `metadata/conversion_plan.json` (additive; convert untouched). Agentic decisions are ADF-only | -| `route_agentic.py` | Rebuilds the report from the immutable deterministic baseline, the routing plan, and stored combines; applies combines deterministically and idempotently. | +| `route_agentic.py` | Applies a routing decision after convert: rebuilds the report from the saved deterministic baseline (`.work/route_baseline/`), the plan and the combines stored in `metadata/agentic_combines.json`; `fill-agentic combine` is the only fill for routed-agentic groups | | `models/conversion_plan.py` | Typed `ConversionPlan` (schema 2): the library's load/validate/record API for `conversion_plan.json`, one decision per component, bound to the inventory, `source_graphs.json` and `source_insights.json` hashes; package fails closed on a mismatch | ## Activity Types diff --git a/README.md b/README.md index 862300f2..dfc4292c 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,7 @@ Or run individual phases: /flowx:flowx-discover # Parse the source (ADF JSON / Airflow DAGs), produce inventory + complexity report /flowx:flowx-enrich # Validate and record agent-authored insights on the inventory /flowx:flowx-convert # Deterministic + agentic translation +/flowx:flowx-route # Decide deterministic vs. agentic per pipeline group and fill the agentic groups (ADF) /flowx:flowx-package # Generate DABs project ``` @@ -263,6 +264,8 @@ flowx_output/ profile_report.csv # discover: per-pipeline complexity report .arm.json # discover: verbatim original ADF/ARM source conversion_plan.json # route: the recorded per-component conversion decision + agentic_combines.json # fill-agentic combine (ADF): authored pipelines per routed-agentic component, re-applied by every route + route_audit.json # package: routing decisions, outcomes and hashes, written when a plan is recorded configuration.json # modify: collected configuration answers .work/ # transient intermediates (translation report, IR, gaps.json); pruned by package ``` diff --git a/app/README.md b/app/README.md index a0b3563e..8ca2aa4f 100644 --- a/app/README.md +++ b/app/README.md @@ -22,6 +22,9 @@ operation; `parameters` is its keyword-argument dict. | `convert` | `adapter convert` | ADF activities → Databricks IR | | `merge_agentic` | `adapter convert --merge-agentic` | Merge ADF agent-produced results into the report | | `resolve_agentic` | `adapter resolve-agentic` | Prepare, stage, and apply reviewed Airflow leaf-gap resolutions | +| `enrich` | `adapter enrich` | Validate and record agent-authored insights, rebuild the inventory | +| `route` | `adapter route` | Recommend deterministic vs. agentic per component; record the plan and rebuild the report | +| `fill_agentic` | `adapter fill-agentic combine` | Replace a routed-agentic group with agent-authored pipelines (ADF) | | `inspect` | `adapter inspect` | Surface pending translation options | | `apply_answers` | `adapter modify` | Apply answers → stamped IR | | `materialize_lookup` | `adapter materialize-lookup` | CSV → lookup-values JSON | diff --git a/docs/content/docs/architecture.mdx b/docs/content/docs/architecture.mdx index 14c122d9..5aa78260 100644 --- a/docs/content/docs/architecture.mdx +++ b/docs/content/docs/architecture.mdx @@ -59,7 +59,7 @@ The unified `flowx.adapter` CLI is the single contract. Both surfaces go through | `mcp/runner.py` | Subprocess bridge to `flowx.adapter` with artifact summarizers (for running translation without the `mcp` dependency) | | `mcp/__main__.py` | `python -m flowx.mcp` entry point (stdio default, `--http` for hosting) | -The `flowx` tool's `command` selects the adapter operation: `inputs`, `discover`, `convert`, `merge_agentic` (ADF only), `resolve_agentic` (Airflow only), `inspect`, `apply_answers`, `materialize_lookup`, `workspace_paths`, `package`, `migrate`, `record_results`, and `install_dashboard` (with `parameters` carrying that command's arguments). +The `flowx` tool's `command` selects the adapter operation, with `parameters` carrying that command's arguments; the [app README](https://github.com/databricks-solutions/flowx/tree/main/app#tool) lists every command. ## Deployment topology diff --git a/docs/content/docs/installation.mdx b/docs/content/docs/installation.mdx index b047e21e..7f5c8a9d 100644 --- a/docs/content/docs/installation.mdx +++ b/docs/content/docs/installation.mdx @@ -132,7 +132,7 @@ flowx is distributed through a Claude Code marketplace. It can be installed as a You can also copy the skill folders straight into your local skills directory: ```shell -cp -R skills/{flowx-setup,flowx-discover,flowx-enrich,flowx-convert,flowx-package,flowx-migrate} ~/.claude/skills/ +cp -R skills/{flowx-setup,flowx-discover,flowx-enrich,flowx-route,flowx-convert,flowx-resolve-airflow-gaps,flowx-package,flowx-migrate} ~/.claude/skills/ ``` diff --git a/src/flowx/bundler/dab_writer.py b/src/flowx/bundler/dab_writer.py index 99b8ec99..0b9a723e 100644 --- a/src/flowx/bundler/dab_writer.py +++ b/src/flowx/bundler/dab_writer.py @@ -522,12 +522,13 @@ def _write_route_audit(output_dir: Path, report_path: Path | None = None) -> Pat Package prunes the transient ``.work/`` folder (translation report + ``gaps.json``) by default, which erases the "what did routing change?" trail. When a routing decision was recorded - (``metadata/conversion_plan.json`` exists), summarise the routed components, their decisions and - their outcomes from the report's routing record, and the gaps routing introduced in agentic-routed - pipelines, into an additive ``metadata/`` artifact that survives the prune. It also records hashes - of the inventory, the saved source graphs and source insights the plan was bound to, the plan, and - the packaged report, plus the report route started from as the routing record names it, so the - trail can be checked against those files later. + (``metadata/conversion_plan.json`` exists), summarise the routed components -- their decisions, + and from the report's routing record their outcomes, fingerprints, applied combine hashes and + replacements -- and the gaps routing introduced in agentic-routed pipelines, into an additive + ``metadata/`` artifact that survives the prune. It also records hashes of the inventory, the saved + source graphs and source insights the plan was bound to, the plan, and the packaged report, plus + the baseline report and gaps route started from as the routing record names them, so the trail + can be checked against those files later. Returns the written path, or ``None`` when there is no recorded plan (no routing happened) -- so the no-route path writes nothing and stays byte-identical. From 9221edcfc861789dab28060ba02724271f748f07 Mon Sep 17 00:00:00 2001 From: Matthew Moorcroft Date: Fri, 9 Oct 2026 09:44:46 +0100 Subject: [PATCH 032/122] Route into the plan, accept groupings, package agentic outputs only Route now writes its recommendation straight into metadata/conversion_plan.json (schema 3) with every decision pending, plus the suggested groupings and an optional routing conversation, and draws it as metadata/routing_review.html, a standard eight-section review page. The agent edits the plan (or passes it) and route applies it once nothing is pending; library fields are recomputed. - Groupings: whole components linked by an inferred relationship or a shared simplification pattern in the insights; an accepted grouping is one agentic unit for the record, fill-agentic, package and the audit. - The agent's output moves to metadata/agentic_conversion.json (renamed from agentic_combines.json); `fill-agentic combine` becomes `fill-agentic`. - Convert's baseline is never rewritten: merges of convert's own gaps after routing are stored as gap fills and replayed on every rebuild. - Package refuses an unfilled agentic unit, a pending decision, a routed report with no plan, and a recorded plan that is not a complete decision. - Route, fill-agentic and package refuse when agentic_insights.json and inventory.json disagree (or the file fails its own hash), with recovery steps. - fill-agentic refuses authored names that share a bundle folder and authored pipelines that still hold a placeholder; route refuses a baseline converted from a different discover. - Applied units drop their routed gaps; the audit carries one canonical plan hash, the output replacement history (first fill is not a replacement), gap fills and the conversation. Deterministic-only discover/convert/package output on the ADF and Airflow fixtures is byte-identical to 839b2c6. Co-authored-by: Isaac --- AGENTS.md | 7 +- CLAUDE.md | 3 +- README.md | 7 +- app/README.md | 4 +- docs/content/docs/guide.mdx | 5 +- skills/flowx-convert/SKILL.md | 2 +- skills/flowx-convert/sources/adf.md | 5 +- skills/flowx-migrate/SKILL.md | 78 +- skills/flowx-migrate/references/workflow.md | 45 +- skills/flowx-route/SKILL.md | 387 +++++---- src/flowx/adapter/__main__.py | 156 ++-- src/flowx/bundler/dab_writer.py | 236 +++-- src/flowx/ir_serde.py | 32 +- src/flowx/mcp/server.py | 132 +-- src/flowx/models/conversion_plan.py | 178 +++- src/flowx/reporting/routing_review.py | 302 +++++++ .../reporting/routing_review_template.html | 83 ++ src/flowx/route_agentic.py | 818 ++++++++++++------ src/flowx/routing.py | 405 ++++++++- tests/unit/test_cli_route_agentic.py | 177 +++- tests/unit/test_conversion_plan_v2.py | 70 +- tests/unit/test_mcp_route.py | 29 +- tests/unit/test_package_routing_preflight.py | 124 ++- tests/unit/test_route_agentic_flow.py | 337 ++++++-- tests/unit/test_route_audit.py | 117 +-- tests/unit/test_routing_groupings.py | 274 ++++++ tests/unit/test_routing_insights_check.py | 165 ++++ tests/unit/test_routing_record.py | 50 +- tests/unit/test_routing_review_page.py | 140 +++ 29 files changed, 3303 insertions(+), 1065 deletions(-) mode change 120000 => 100644 CLAUDE.md create mode 100644 src/flowx/reporting/routing_review.py create mode 100644 src/flowx/reporting/routing_review_template.html create mode 100644 tests/unit/test_routing_groupings.py create mode 100644 tests/unit/test_routing_insights_check.py create mode 100644 tests/unit/test_routing_review_page.py diff --git a/AGENTS.md b/AGENTS.md index 029c1f47..7ef5df36 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -92,9 +92,10 @@ intermediates under `.work/` (pruned by `package`). | `reporting/coverage.py` | Builds per-pipeline coverage rows from `metadata/` | | `reporting/results.py` | Writes per-run coverage to a UC table (run_id/run_date/run_by) via the SDK | | `reporting/dashboard.py` | Installs + publishes an AI/BI coverage dashboard over the results table | -| `routing.py` | Groups pipelines into connected components over control lineage; recommends deterministic/agentic per component (both options) and records the user's decision as `metadata/conversion_plan.json` (additive; convert untouched). Agentic decisions are ADF-only | -| `route_agentic.py` | Applies a routing decision after convert: rebuilds the report from the saved deterministic baseline (`.work/route_baseline/`), the plan and the combines stored in `metadata/agentic_combines.json`; `fill-agentic combine` is the only fill for routed-agentic groups | -| `models/conversion_plan.py` | Typed `ConversionPlan` (schema 2): the library's load/validate/record API for `conversion_plan.json`, one decision per component, bound to the inventory, `source_graphs.json` and `agentic_insights.json` hashes; package fails closed on a mismatch | +| `reporting/routing_review.py` | Renders `metadata/routing_review.html`, the standard eight-section routing review page, from `routing_review_template.html` (stdlib only, no timestamps) | +| `routing.py` | Groups pipelines into connected components over control lineage; recommends deterministic/agentic per component (both options) and suggests groupings from the insights; writes the recommendation into `metadata/conversion_plan.json` with decisions pending and records the user's edits (additive; convert untouched). Agentic decisions are ADF-only; also the reader-side check that `agentic_insights.json` agrees with the inventory | +| `route_agentic.py` | Applies a routing decision after convert: rebuilds the report from the never-rewritten baseline (`.work/route_baseline/`), the plan, and the agent's output in `metadata/agentic_conversion.json` (per-unit pipelines + fills of convert's own gaps); `fill-agentic` is the only fill for routed-agentic units | +| `models/conversion_plan.py` | Typed `ConversionPlan` (schema 3): the library's load/validate/record API for `conversion_plan.json`, one decision per component (pending until made), suggested groupings, the routing conversation, bound to the inventory, `source_graphs.json` and `agentic_insights.json` hashes; package fails closed on a mismatch, a pending decision, or an unfilled agentic unit | ## Activity Types diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 120000 index 47dc3e3d..00000000 --- a/CLAUDE.md +++ /dev/null @@ -1 +0,0 @@ -AGENTS.md \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..a9d4d269 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,2 @@ + +@AGENTS.md diff --git a/README.md b/README.md index bba869c3..516f61df 100644 --- a/README.md +++ b/README.md @@ -263,9 +263,10 @@ flowx_output/ agentic_insights.json # enrich: validated agent-authored insights, rebuilt into inventory.json profile_report.csv # discover: per-pipeline complexity report .arm.json # discover: verbatim original ADF/ARM source - conversion_plan.json # route: the recorded per-component conversion decision - agentic_combines.json # fill-agentic combine (ADF): authored pipelines per routed-agentic component, re-applied by every route - route_audit.json # package: routing decisions, outcomes and hashes, written when a plan is recorded + conversion_plan.json # route: the recommendation, suggested groupings and the user's per-component decisions + routing_review.html # route: the standard review page drawn from the plan + agentic_conversion.json # fill-agentic (ADF): the agent's conversion output per routed-agentic unit, re-applied by every route + route_audit.json # package: routing decisions, outcomes, history, conversation and hashes, written when a plan is recorded configuration.json # modify: collected configuration answers .work/ # transient intermediates (translation report, IR, gaps.json); pruned by package ``` diff --git a/app/README.md b/app/README.md index 384135a9..6dfa8d2b 100644 --- a/app/README.md +++ b/app/README.md @@ -23,8 +23,8 @@ operation; `parameters` is its keyword-argument dict. | `merge_agentic` | `adapter convert --merge-agentic` | Merge ADF agent-produced results into the report | | `resolve_agentic` | `adapter resolve-agentic` | Prepare, stage, and apply reviewed Airflow leaf-gap resolutions | | `enrich` | `adapter enrich` | Return `inventory.json` (`prepare`), then validate and record agent-authored insights (`apply`) | -| `route` | `adapter route` | Recommend deterministic vs. agentic per component; record the plan and rebuild the report | -| `fill_agentic` | `adapter fill-agentic combine` | Replace a routed-agentic group with agent-authored pipelines (ADF) | +| `route` | `adapter route` | Write the recommendation into `conversion_plan.json` and the review page; apply the plan once decided | +| `fill_agentic` | `adapter fill-agentic` | Fill a routed-agentic unit with agent-authored pipelines (ADF) | | `inspect` | `adapter inspect` | Surface pending translation options | | `apply_answers` | `adapter modify` | Apply answers → stamped IR | | `materialize_lookup` | `adapter materialize-lookup` | CSV → lookup-values JSON | diff --git a/docs/content/docs/guide.mdx b/docs/content/docs/guide.mdx index 7e967fce..59484480 100644 --- a/docs/content/docs/guide.mdx +++ b/docs/content/docs/guide.mdx @@ -81,7 +81,10 @@ flowx_output/ │ ├── profile_report.csv # profile: per-pipeline complexity report │ ├── .arm.json # discover: verbatim original ADF/ARM pipeline source │ ├── agentic_insights.json # enrich: agent-authored insights, bound to source_graphs.json -│ ├── conversion_plan.json # route: the recorded per-component conversion decision +│ ├── conversion_plan.json # route: the recommendation and the user's per-component decisions +│ ├── routing_review.html # route: the standard review page drawn from the plan +│ ├── agentic_conversion.json # fill-agentic: the agent's conversion output, re-applied by every route +│ ├── route_audit.json # package: what each routing unit was converted to, and the routing conversation │ └── configuration.json # modify: the collected configuration answers └── .work/ # transient intermediates (translation report, IR, gaps.json); pruned by prepare ``` diff --git a/skills/flowx-convert/SKILL.md b/skills/flowx-convert/SKILL.md index 214bd008..dc0b4500 100644 --- a/skills/flowx-convert/SKILL.md +++ b/skills/flowx-convert/SKILL.md @@ -70,7 +70,7 @@ across ADF and Airflow. `show_when` condition). Walk it locally; ask an option only when its `show_when` is satisfied. - `modify --output-dir --answer OPTION_ID=VALUE ...` — validate and apply collected answers, writing `.work/translation_report.stamped.json` + `metadata/configuration.json`. -- `merge_agentic --report --agentic-results ` — **ADF only**. Fold agent-produced per-activity translations into an ADF report, filling convert's own agentic gaps. It refuses, writing nothing, any result landing in a pipeline the routing record routes agentic (fill those with `fill-agentic combine`). After routing, a merge written in place into the live `.work/translation_report.json` is applied to both that report and the stored deterministic baseline (so the next rebuild keeps it) and the routing record's baseline hash is refreshed; a merge written to a separate `--output` copy leaves the baseline alone, so a re-route does not keep it. Airflow's legacy name-based merge is disabled. +- `merge_agentic --report --agentic-results ` — **ADF only**. Fold agent-produced per-activity translations into an ADF report, filling convert's own agentic gaps. It refuses, writing nothing, any result landing in a pipeline the routing record routes agentic (fill those with `fill-agentic`). After routing, a merge written in place into the live `.work/translation_report.json` is also stored as a gap fill in `metadata/agentic_conversion.json`, so the next rebuild re-applies it on top of route's unchanged deterministic baseline; a merge written to a separate `--output` copy stores nothing, so a re-route does not keep it. Airflow's legacy name-based merge is disabled. ## Output artifacts (shared, transient under `/.work/`) diff --git a/skills/flowx-convert/sources/adf.md b/skills/flowx-convert/sources/adf.md index cc68b971..fe8632f6 100644 --- a/skills/flowx-convert/sources/adf.md +++ b/skills/flowx-convert/sources/adf.md @@ -131,8 +131,9 @@ Where the fragment leaves a value out, flowx adds only the plumbing the task nee Placeholders are replaced in place, status → `translated`. Exits non-zero if any result can't be matched. Add `--output ` to write a copy instead of overwriting. After routing, only an -in-place merge into `.work/translation_report.json` also updates route's saved deterministic baseline -(so a re-route keeps it); a merge written to an `--output` copy leaves the baseline untouched. +in-place merge into `.work/translation_report.json` is also stored as a gap fill in +`metadata/agentic_conversion.json` (so a re-route keeps it, while route's saved deterministic baseline +stays as convert wrote it); a merge written to an `--output` copy stores nothing. ## Step 6.1 — Just-in-time translation configuration diff --git a/skills/flowx-migrate/SKILL.md b/skills/flowx-migrate/SKILL.md index d117799b..cb1ffe44 100644 --- a/skills/flowx-migrate/SKILL.md +++ b/skills/flowx-migrate/SKILL.md @@ -18,7 +18,7 @@ triggers: Orchestrate the complete migration of a source orchestrator's pipelines to Databricks Lakeflow Jobs via Declarative Automation Bundles. This skill runs the full flow in sequence: discover → enrich -(default) → convert (deterministic baseline) → route (decide + edit) → fill agentic gaps → package. +(default) → convert (deterministic baseline) → route (recommend → decide → apply) → fill agentic gaps → package. ## Context @@ -35,14 +35,16 @@ This is the top-level orchestration skill. It runs the full migration pipeline: (`flowx-enrich`); skippable for a deterministic-only pass 3. **Convert (deterministic baseline)** — Convert the source's tasks to Databricks IR, producing `.work/translation_report.json` (`flowx-convert`). Runs **before** routing -4. **Route** — Decide, per connected component, deterministic vs. agentic conversion; record the - fingerprint-bound `metadata/conversion_plan.json`; edit the baseline report so routed-agentic - groups become placeholder gaps (`flowx-route`) -5. **Fill agentic gaps** — Replace each routed-agentic group with `fill-agentic combine` (one - same-named authored pipeline keeps a pipeline 1:1); `convert --merge-agentic` (ADF only) fills - convert's own gaps and refuses routed-agentic pipelines; Airflow gaps go through - `flowx-resolve-airflow-gaps`. If you re-run `convert` after routing, re-run `route` next: it - takes the fresh report as its new baseline and re-applies the plan and stored combines +4. **Route** — Route writes its recommendation and the suggested groupings into + `metadata/conversion_plan.json` and the standard review page `metadata/routing_review.html`; the + customer decides deterministic vs. agentic per component (and which groupings to accept), and + route applies the plan to the baseline report so routed-agentic units become placeholder gaps + (`flowx-route`) +5. **Fill agentic gaps** — Fill each routed-agentic unit with `fill-agentic` (one same-named authored + pipeline keeps a pipeline 1:1); `convert --merge-agentic` (ADF only) fills convert's own gaps and + refuses routed-agentic pipelines; Airflow gaps go through `flowx-resolve-airflow-gaps`. Package + refuses until every agentic unit is filled. If you re-run `convert` after routing, re-run `route` + next: it takes the fresh report as its new baseline and re-applies the plan and the agent's outputs 6. **Package** — Generate Databricks Declarative Automation Bundles for deployment Each phase builds on the output of the previous phase. The user is shown a summary and asked to confirm before proceeding to the next phase. @@ -121,9 +123,9 @@ flowx(command="inputs", parameters={"phase": "discover", "source": "adf"}) # so flowx(command="discover", parameters={"source": "adf", "adf_definitions": {...}, "output_dir": ..., "pipeline": ...}) flowx(command="enrich", parameters={"output_dir": ..., "insights": {...}}) # default: author + merge the insights layer (see flowx-enrich) flowx(command="convert", parameters={"source": "adf", "output_dir": ..., "pipeline": ...}) # deterministic baseline, BEFORE route -flowx(command="route", parameters={"output_dir": ...}) # recommend; re-call with "plan": {...} to record + edit the report (see flowx-route) +flowx(command="route", parameters={"output_dir": ...}) # writes the plan + review page; re-call with the decided "plan": {...} to apply (see flowx-route) flowx(command="merge_agentic", parameters={"source": "adf", "report_path": ..., "agentic_results_dir": ..., "output_path": ...}) # ADF only, fills convert's own gaps -flowx(command="fill_agentic", parameters={"output_dir": ..., "members": [...], "pipelines": [...]}) # the only fill for a routed-agentic group +flowx(command="fill_agentic", parameters={"output_dir": ..., "members": [...], "pipelines": [...]}) # the only fill for a routed-agentic unit flowx(command="inspect", parameters={"report_path": ...}) flowx(command="apply_answers", parameters={"report_path": ..., "answers": [...], "output_dir": ...}) flowx(command="package", parameters={"output_dir": ..., "catalog": ..., "schema": ...}) @@ -257,8 +259,8 @@ reads and edits this report. Invoke the `flowx:flowx-convert` skill with: Run this deterministic convert **before routing.** If you re-run it after routing (for example with different options), re-run `route` straight after: a fresh report carries no routing record, so route -takes it as the new deterministic baseline and re-applies the plan and the stored combines. Merges of -convert's own gaps made before that convert must be merged again. +takes it as the new deterministic baseline and re-applies the plan and the agent's stored outputs. +Merges of convert's own gaps made against the earlier report are dropped and must be merged again. Wait for the translation to complete and present the summary: @@ -292,50 +294,52 @@ decision for an Airflow inventory; Airflow agentic conversion follows the Airflo contract. Routing reads the deterministic baseline report `.work/translation_report.json` (built in Step 5) and -edits it. Invoke the **`flowx:flowx-route`** skill to present the per-connected-component -recommendation, take the **customer's** decision (interactive, or an authored plan — the customer -decides deterministic-vs-agentic per component; the agent presents the options and serializes only the -approved decision), and record the fingerprint-bound `/metadata/conversion_plan.json`. -Recording a plan edits `.work/translation_report.json` so every routed-**agentic** group's tasks -become placeholder gaps; deterministic groups (and a no-agentic-route plan) leave the report -untouched — non-breaking. +edits it. Invoke the **`flowx:flowx-route`** skill: route first writes its per-component +recommendation and the suggested groupings into `/metadata/conversion_plan.json` with every +decision pending, and draws them as `/metadata/routing_review.html`. Present the page, ask +the customer what they want the conversion to achieve (record it as the plan's `conversation`), take +the **customer's** decisions (the agent presents the options and records only the approved decisions), +and route again. Once nothing is pending, route applies the plan: every routed-**agentic** unit's tasks +become placeholder gaps; deterministic units (and a no-agentic-route plan) leave the report untouched — +non-breaking. If Step 5 was skipped and the report is missing, `route` can trigger the convert phase in-process once (pass `--source` / `--source-path`); it never runs a second plain convert once a report exists. Re-running `route` is always allowed: it rebuilds the report from the saved deterministic baseline, -the current plan and the stored combines, so switching a component back to deterministic restores -convert's output for it. -If the user wants a straight deterministic migration, they accept the all-deterministic recommendation -here and the rest of the flow is unchanged (no placeholders, so Step 8 is a no-op). +the stored gap fills, the current plan and the agent's stored outputs, so switching a component back +to deterministic restores convert's output for it. +If the user wants a straight deterministic migration, they decide every component deterministic here +and the rest of the flow is unchanged (no placeholders, so Step 8 is a no-op). ### Step 8 — Fill routed-agentic gaps -If Step 7 routed any component **agentic**, its pipelines' tasks are now `PlaceholderActivity` nodes -with one tagged gap each. Author the replacement and apply it (no LLM in the library — it validates +If Step 7 routed any unit **agentic**, its pipelines' tasks are now `PlaceholderActivity` nodes +with one tagged gap each, and package refuses until each such unit is filled. Author the replacement and apply it (no LLM in the library — it validates and records) **before** the just-in-time config below, so the filled tasks get configuration-stamped and packaged: -- **COMBINE is the only fill for a routed-agentic group** (N pipelines → M, e.g. one Lakeflow Connect - pipeline, or one same-named pipeline to keep a pipeline 1:1): author the replacement pipeline IR - (typically with `AgenticComponentActivity` nodes) and run - `fill-agentic combine --output-dir --members "" --pipelines-path `; - the members must exactly match the routed-agentic component and the merged report is validated - structurally before it is written. The authored pipelines are stored in - `metadata/agentic_combines.json`, so a later re-route re-applies them. +- **`fill-agentic` is the only fill for a routed-agentic unit** (N pipelines → M, e.g. one Lakeflow + Connect pipeline, or one same-named pipeline to keep a pipeline 1:1): author the replacement + pipeline IR (typically with `AgenticComponentActivity` nodes) and run + `fill-agentic --output-dir --members "" --pipelines-path `; + the members must exactly match the routed-agentic component or accepted grouping and the merged + report is validated structurally before it is written. The authored pipelines are stored in + `metadata/agentic_conversion.json`, so a later re-route re-applies them. - **Convert's own agentic gaps** (in pipelines not routed agentic, **ADF only**): author one result JSON per gap and merge with `convert --source adf --merge-agentic --report .work/translation_report.json --agentic-results ` (`--source adf` is mandatory; the convert phase exits 2 without it). The merge replaces, per result, the **first task whose `name` matches `activity_name`**, so make sure each name targets the intended gap. It refuses, writing nothing, any result that lands in a routed-agentic pipeline. After routing - an in-place merge into `.work/translation_report.json` updates the saved baseline too, so a re-route - keeps the merge (a merge written to an `--output` copy does not). Airflow's `--merge-agentic` is + an in-place merge into `.work/translation_report.json` is also stored as a gap fill in + `metadata/agentic_conversion.json`, so a re-route keeps the merge while the saved baseline stays as + convert wrote it (a merge written to an `--output` copy stores nothing). Airflow's `--merge-agentic` is disabled (exits 2) — for Airflow per-gap fills use the **`flowx-resolve-airflow-gaps`** skill. -Run `modify` (Step 9) after the last route, combine or merge: none of them rewrites modify's +Run `modify` (Step 9) after the last route, fill-agentic or merge: none of them rewrites modify's configured report, and package asks you to re-run modify when it is out of date. -Skip this step entirely when no component was routed agentic. See the **`flowx-route`** skill for the +Skip this step entirely when no unit was routed agentic. See the **`flowx-route`** skill for the full fill details and the `AgenticComponentActivity` shape. ### Step 9 — Gather just-in-time translation configuration diff --git a/skills/flowx-migrate/references/workflow.md b/skills/flowx-migrate/references/workflow.md index 4fe88f68..41e7c844 100644 --- a/skills/flowx-migrate/references/workflow.md +++ b/skills/flowx-migrate/references/workflow.md @@ -73,25 +73,32 @@ merges: records it in `metadata/agentic_insights.json` (bound to the saved `source_graphs.json`) and rebuilds `metadata/inventory.json` with it under a single additive `insights` key, leaving every deterministic key byte-identical. Skippable for a deterministic-only, headless pass. -- **Route:** groups pipelines into connected components over control lineage and, per component, - records a freely-editable `deterministic` or `agentic` decision as the fingerprint-bound - `metadata/conversion_plan.json`. Route rebuilds `.work/translation_report.json` and `gaps.json` - from an immutable deterministic baseline, the routing plan, and stored combines: deterministic - components stay as-is; agentic components with a stored combine get the combine applied; agentic - without a combine become placeholders. A fully-deterministic plan (or no plan) leaves the report - byte-identical to the baseline — the non-breaking guarantee. Route stamps a routing record onto - the report (per component: members, decision, outcome, combine hash, and fingerprint) and every - re-route rebuilds deterministically and idempotently from the baseline and current plan. A fresh - convert (a report without a routing record) becomes the new baseline on the next route. - -Routed-agentic groups are then filled only by `fill-agentic combine`, which applies authored pipelines -(N→M, e.g. a Lakeflow Connect collapse, or one same-named pipeline to stay 1:1), typically using -`AgenticComponentActivity` nodes. Same authored pipelines as already applied returns "already applied, -unchanged" and writes nothing (idempotent). Different pipelines replace and rebuild. `convert --merge-agentic` -fills convert's own gaps and refuses routed-agentic pipelines. After routing, merging convert's own gaps -in place into `.work/translation_report.json` updates both that report and the stored baseline so the -next rebuild keeps them. Route, combine and -merge never rewrite modify's configured report; package asks to re-run `modify` when it is out of date. +- **Route:** groups pipelines into connected components over control lineage and writes its + per-component recommendation, plus the groupings the insights suggest (components linked by an + inferred relationship or a shared simplification pattern), straight into + `metadata/conversion_plan.json` with every decision pending, and draws it as the standard review + page `metadata/routing_review.html`. The customer decides `deterministic` or `agentic` per + component and which groupings to accept (an accepted grouping is converted as one agentic unit); + the agent records those decisions, and the routing conversation, in the plan and routes again. Once + nothing is pending, route rebuilds `.work/translation_report.json` and `gaps.json` from convert's + unchanged baseline, the stored fills of convert's own gaps, the plan, and the agent's outputs: + deterministic units stay as-is; agentic units with a stored output get it applied; agentic units + without one become placeholders. A fully-deterministic plan (or no plan) leaves the report + byte-identical to the baseline — the non-breaking guarantee. Route stamps a routing record onto the + report (per unit: members, decision, outcome, output hash, and fingerprint) and every re-route + rebuilds deterministically and idempotently. A fresh convert (a report without a routing record) + becomes the new baseline on the next route. + +Routed-agentic units are then filled only by `fill-agentic`, which applies authored pipelines (N→M, +e.g. a Lakeflow Connect collapse, or one same-named pipeline to stay 1:1), typically using +`AgenticComponentActivity` nodes, and stores them in `metadata/agentic_conversion.json`. Same authored +pipelines as already applied returns "already applied, unchanged" and writes nothing (idempotent). +Different pipelines replace the stored output, recorded in its `replaced` history, and rebuild. +`convert --merge-agentic` fills convert's own gaps and refuses routed-agentic pipelines; after routing, +an in-place merge into `.work/translation_report.json` is also stored as a gap fill, so the next +rebuild keeps it and the baseline stays as convert wrote it. Package refuses until every agentic unit +is filled. Route, fill-agentic and merge never rewrite modify's configured report; package asks to +re-run `modify` when it is out of date. ## Phase 2: Convert diff --git a/skills/flowx-route/SKILL.md b/skills/flowx-route/SKILL.md index b0623b12..c417b63c 100644 --- a/skills/flowx-route/SKILL.md +++ b/skills/flowx-route/SKILL.md @@ -2,16 +2,17 @@ name: flowx-route description: > Route each connected component of the discovered inventory to a deterministic (1:1 engine) or - agentic (LLM-assisted re-architecture) conversion, record the fingerprint-bound conversion plan, - and fill the routed-agentic groups with fill-agentic combine. Runs after enrich, before/with - convert. + agentic (LLM-assisted re-architecture) conversion: route writes its recommendation and the + suggested groupings into metadata/conversion_plan.json and a standard review page, the user decides, + route applies the plan, and fill-agentic fills each routed-agentic unit. Runs after enrich and + convert, before package. triggers: - "route pipelines" - "route conversion" - "conversion plan" - "deterministic or agentic" - "fill agentic" - - "combine pipelines" + - "group pipelines" - "agentic conversion" - "recommend conversion route" --- @@ -21,185 +22,175 @@ triggers: After discover (and, by default, `flowx-enrich`), routing decides **per connected component** whether each part of the factory converts **deterministically** (the typed engine's 1:1 translation) or **agentically** (an LLM-authored re-architecture, e.g. collapsing five extractors onto one Lakeflow -Connect pipeline). It records the decision as a fingerprint-bound `metadata/conversion_plan.json`, -edits the translation report so routed-agentic groups become placeholder gaps, and then you author -the fill. +Connect pipeline). Route writes its recommendation straight into `metadata/conversion_plan.json`, the +one readable record of the decisions, and draws it as `metadata/routing_review.html`. The user decides, +you edit the plan, route applies it to the translation report, and you author the fill. -**ADF only (current scope).** The discover → enrich → route → edit → fill agentic-conversion flow is +**ADF only (current scope).** The discover → enrich → route → fill agentic-conversion flow is supported for **ADF today**, and `route` enforces it: for an Airflow inventory it recommends and -records only `deterministic` decisions and rejects an `agentic` one (the plan is not recorded and the -report is left untouched). `convert --merge-agentic --source airflow` is disabled too. Airflow -DAGs are often one connected component, so Airflow agentic conversion follows the Airflow track's own -per-gap and patch contract rather than this whole-component flow. For **Airflow agentic gaps today, -use the `flowx-resolve-airflow-gaps` skill** (the strict per-gap resolver), not this routing flow. +records only `deterministic` decisions, suggests no groupings, and rejects an `agentic` decision (the +plan is not recorded and the report is left untouched). `convert --merge-agentic --source airflow` is +disabled too. Airflow DAGs are often one connected component, so Airflow agentic conversion follows the +Airflow track's own per-gap and patch contract rather than this whole-component flow. For **Airflow +agentic gaps today, use the `flowx-resolve-airflow-gaps` skill** (the strict per-gap resolver), not +this routing flow. **There is no LLM inside flowx.** The library computes the recommendation deterministically and only *validates and records* the decision and the authored fill — the same author → validate → merge contract `enrich` uses. This is additive and non-breaking: with **no recorded plan**, `convert` and `package` behave exactly as before. -**Who decides what.** The library **recommends** a route per component; the **customer decides** -deterministic-vs-agentic for each component; the **agent** presents the options and the -recommendation, serializes only the customer's *approved* decision into the plan, and authors the -agentic fill. The agent never picks the route on the customer's behalf — it records the customer's -choice and does the mechanical work of the approved fill. +**Who decides what.** The library **recommends** a route per component and **suggests** groupings; +the **customer decides** deterministic-vs-agentic for each component and which groupings to accept; +the **agent** presents the review page, asks the customer what they want the conversion to achieve, +records only the customer's decisions in the plan, and authors the agentic fill. The agent never picks +the route on the customer's behalf. Route leaves every decision **pending** until it is made, and +applies nothing while one is pending. + +## The files + +| Stage | File | Written by | +|---|---|---| +| describe | `metadata/source_graphs.json`, `metadata/agentic_insights.json` (optional), `metadata/inventory.json` | discover, enrich | +| decide | `metadata/conversion_plan.json` — components, decisions, suggested groupings, conversation | route writes it; you edit it | +| decide | `metadata/routing_review.html` — the standard review page (generated, never edited) | route, fill-agentic | +| convert | `.work/translation_report.json`, `.work/gaps.json` — the deterministic report with the agentic decision's edits | convert, then route | +| convert | `metadata/agentic_conversion.json` — the agent's conversion output (per unit, plus fills of convert's own gaps) | fill-agentic, merge_agentic | +| convert | `.work/route_baseline/` — convert's report as route first found it (internal, never rewritten) | route | +| after | the bundle, `metadata/route_audit.json` | package | ## Where routing sits ``` -discover → enrich (default) → convert (deterministic baseline) → route (decide + edit) → fill-agentic → package +discover → enrich (default) → convert (deterministic baseline) → route (recommend → decide → apply) → fill-agentic → package ``` Convert builds the deterministic baseline report **before** routing (route can trigger it in-process via `--source` / `--source-path`); routing then edits that report. If you run `convert` again after routing, run `route` straight after it: a fresh report carries no routing record, so route takes it -as the new baseline and re-applies the plan and the stored combines (merges of convert's own gaps -made before that convert must be merged again). +as the new baseline and re-applies the plan and the agent's stored outputs (fills of convert's own +gaps merged against the earlier report are dropped and must be merged again; route says how many). Pipelines are grouped into weak/undirected **connected components** over the inventory's control lineage (`lineage.control_edges`), so mutually-referencing pipelines are decided together and a caller/callee reference is never split across incompatible routes. Only resolved control edges join a component; an unresolved call is reported as a finding. ADF emits these control edges (from -`ExecutePipeline`) in discovery. Agentic routing is ADF-only today whatever lineage an inventory -carries (see the scope note above). +`ExecutePipeline`) in discovery. Run the **`setup`** skill first if you haven't. Everything below has an MCP-tool path (Genie Code, or a local stdio registration — call the single **`flowx`** tool, run no `python3`/`$PY`) and a venv-CLI path (local; `PY="$(cat /.migration-venv)"` and `export PYTHONPATH="/src"`). -## Step 1 — Recommend (the dry run you read first) +## Step 1 — Route writes the recommendation into the plan -Call `route` with **no decision** to get the recommendation: every component with its `members`, both -first-class conversion `options` (a *deterministic* option carrying the engine-capability assessment -+ any uncovered gaps, and an *agentic* option carrying the `recommended_patterns` from `enrich`, with -any `simplification_pattern` flagged), the `findings` (unresolved/dangling control edges kept, never -severed), and a ready-to-record `default_plan` proposing `decision == recommended` for every -component. - -- **MCP tool:** `flowx(command="route", parameters={"output_dir": ""})` -- **venv CLI:** - - ```bash - export PYTHONPATH="/src" - PY="$(cat /.migration-venv)" - "$PY" -m flowx.adapter route --output-dir - ``` - - On a non-TTY with no `--plan-path`, this emits the recommendation and exits 0 (it edits nothing). - `route` reads `metadata/inventory.json`; run discover first. `--out ` writes the JSON to a - file instead of stdout. - -`recommended` is the library's starting suggestion — `deterministic` when the whole component is -engine-capable, else `agentic`. Present each component's members, its recommendation, and the agentic -option's patterns (flag any `has_simplification` re-architecture prominently). - -## Step 2 — Decide and record the plan - -Take the per-component decision and record it. The plan is **freely editable** — change a component's -decision from agentic to deterministic, or vice versa, without restriction. `route` validates the plan, -writes the fingerprint-bound `metadata/conversion_plan.json`, and then rebuilds `.work/translation_report.json` -+ `gaps.json` from the immutable deterministic baseline, the plan, and any stored combines: - -- **deterministic** components: the baseline pipelines stay (deterministic outcome). -- **agentic components with a stored combine** whose members match: the combine's pipelines replace the - members; `gaps.json` still lists the members' routed tasks (agentic-applied outcome). -- **agentic components without a matching combine**: placeholders and gaps as before (agentic-not-viable). -- **switching back to deterministic**: restores that component's baseline exactly (deterministic outcome). - When the whole plan is deterministic, route writes the baseline report and gaps back byte for byte, - with no record — the non-breaking guarantee. - -When it edits the report, route also stamps a **routing record** onto it (the top-level -`_routing_record` key): the plan's hash, the hashes of the immutable deterministic baseline report and -gaps it started from, and one entry per `component_id` with its `members`, `decision`, `outcome`, -`combine_sha256` (set only while that component's stored combine is applied, otherwise `null`), -`fingerprint`, and `replacements` (tracking changes to a component's fingerprint across re-routes). -Every re-route rebuilds from the baseline and current plan; identical input gives -identical output (byte-for-byte, deterministic and idempotent). - -There are three ways to supply the decision: - -- **Interactive prompt (TTY).** Run `route` with no `--plan-path` on a real terminal and it asks, per - component, `route [d]eterministic / [a]gentic (default=)`. An empty answer accepts the - recommendation. This is the from-the-seat path. -- **Authored plan file** — `--plan-path ` (or `--plan-path -` to read the plan JSON from stdin). -- **MCP tool** — pass the plan inline as `plan` (or `plan_path`): - - ``` - flowx(command="route", parameters={"output_dir": "", "plan": { ...authored plan... }}) - ``` - -### The plan shape - -The plan carries **only** the decision (and an optional rationale) per component — and the decision is -the **customer's**, not yours. Your job is to present each component's members, its `recommended` -route, and both `options`, then serialize the customer's pick; you do not choose the route yourself. -The library recomputes `members`, `recommended`, and both `options` on record, so recorded facts -cannot drift from the inventory or be faked. Start from the recommendation's `default_plan` and flip -the components the customer chose to override: - -```json -{ - "components": [ - {"component_id": "component-1", "members": ["IngestSalesforce", "IngestWorkday"], "decision": "agentic", - "rationale": "Collapse both extractors onto one Lakeflow Connect pipeline"}, - {"component_id": "component-2", "members": ["BuildMart"], "decision": "deterministic"} - ] -} -``` - -Rules the validator enforces (all violations returned at once; nothing written on failure): - -- `decision` must be `"deterministic"` or `"agentic"`; `rationale` (optional) must be a non-empty - string when present. -- Every `member` must be a real inventory pipeline, and a component's `members` must **exactly match - one computed connected component** — a decision can never split a component or span two. -- The plan is a **bijection**: every component is decided exactly once (no partial plan, no - duplicate/conflicting decisions). -- `component_id` is **required** on every component — a non-empty string that must match the computed - component for those `members`. Start from the recommendation's `default_plan`, which already carries - the correct `component_id` for each component. +Run `route` with no plan. It writes `metadata/conversion_plan.json` and `metadata/routing_review.html` +and touches nothing else: -### Triggering convert if the report is missing +- **MCP tool:** `flowx(command="route", parameters={"output_dir": ""})` — the plan and the page + come back under `bundle` (or are uploaded to `output_volume_path` / `output_workspace_path`), because + a hosted agent cannot read the server's `output_dir`. +- **venv CLI:** `"$PY" -m flowx.adapter route --output-dir ` (`--out ` writes the result + JSON to a file). -`route` edits `.work/translation_report.json`, which the convert phase produces. If it is missing, -pass `--source ` **and** `--source-path ` (MCP: `source` plus the source-specific -path param — `adf_source_path` or `airflow_source_path`; a generic `source_path` is **ignored**, and -ADF also accepts `adf_definitions` / `adf_volume_path` / `adf_workspace_path`) and `route` triggers -the convert phase in-process first. Otherwise run `flowx-convert` before routing. - -### venv CLI +The plan lists every component with its `members`, the library's `recommended` route, both +first-class conversion `options` (a *deterministic* option carrying the engine-capability assessment +and any uncovered activities, and an *agentic* option carrying the `recommended_patterns` from enrich, +with any `simplification_pattern` flagged and release states disclosed), and `decision: null` +(pending). It also lists the `suggested_groupings`, an empty `conversation`, and the `findings` +(unresolved control edges kept, never severed). A decision already in the file for a component with +the same members is kept, so re-running route after a re-enrich or re-discover keeps your decisions +and leaves only new or regrouped components pending. + +### Present it with the review page + +`metadata/routing_review.html` is the standard way to show the customer what their source does and how +it will be converted. Every run has the same eight sections in the same order: summary, what the +source does (the insights), components, suggested groupings, how each unit will be converted, the +routing conversation, findings, and how to steer. Open it with the customer, or walk through the same +sections in that order. + +Ask open questions grounded in what discover and enrich found — what they want the conversion to +achieve, the databases behind the extractors, whether they are after a specific pattern such as +Lakeflow Connect — and record each question and answer in the plan's `conversation` +(`[{"question": "...", "answer": "..."}]`). Package copies it into `route_audit.json`. + +### Suggested groupings + +A **grouping** is two or more whole components the insights suggest converting together, agentically, +as **one unit**: one agent output replaces all of their pipelines (for example five independent +extractors → one Lakeflow Connect pipeline). It never splits a component. Route suggests one from two +kinds of link in the insights: + +- an `inferred` pipeline relationship (with its evidence and confidence) between pipelines of + different components; +- one simplification pattern (same pattern name, `simplification_pattern: true`) recommended for + pipelines in more than one component — the independent extractors one managed pattern could replace. + +Components joined by these links, directly or through each other, form one suggestion +(`grouping-`), so suggestions never overlap. Each lists its `components`, `members` and `basis` +(the relationships or the shared pattern behind it), and `accepted: false`. Only a suggested grouping +can be accepted; to group other components, record an inferred relationship through enrich and route +again. + +## Step 2 — Decide, then route again to apply + +Edit only these fields in `metadata/conversion_plan.json` (everything else is library-owned and +recomputed on every route, so edits to it are ignored; an unknown key is refused): + +- each component's `decision` — `"deterministic"` or `"agentic"` (the customer's choice) — and an + optional `rationale`; +- to accept a grouping, its `accepted: true`, and decide each of its components `"agentic"`; +- `conversation`. + +Then run `route` again (no plan). Once **no decision is pending** it validates the plan, records it, +and applies it. Other ways to supply the decisions: + +- **A plan** — `--plan-path ` (or `--plan-path -` for stdin), or MCP `plan` inline / `plan_path`: + the decisions alone (`{"components": [{component_id, members, decision}], "suggested_groupings": + [{grouping_id, accepted}], "conversation": [...]}`), or an edited copy of the recorded plan. This is + the hosted path: edit the plan route returned and send it back as `plan`. +- **Interactive prompt (TTY)** — `route` on a real terminal offers each suggested grouping whose + components are pending, then asks `route [d]eterministic / [a]gentic (default=)` for + each pending component. -```bash -"$PY" -m flowx.adapter route --output-dir --plan-path plan.json \ - [--source adf --source-path ] # only needed to trigger convert when the report is missing -``` +Rules the validator enforces (all violations returned at once; nothing written on failure): -Exit 1 (nothing written) on a missing report it cannot produce or a plan that fails validation. -Re-routing under different decisions is always allowed; it rebuilds from the baseline. +- `decision` is `"deterministic"`, `"agentic"`, or `null` while pending; `rationale` (optional) is a + non-empty string. +- Every component is listed exactly once, and its `members` must exactly match one computed + connected component; `component_id` must match too. +- An accepted grouping has no component decided `"deterministic"`; agentic decisions and groupings are + ADF-only. +- `conversation` entries are `{question, answer}` with non-empty strings. -## Step 3 — Fill the routed-agentic groups +### What applying does -Every routed-agentic pipeline's tasks are now `PlaceholderActivity` nodes with one pipeline-tagged -`AgenticGap` each. A routed-agentic group is filled **only** by `fill-agentic combine`, which replaces -the whole group with pipelines you author (no LLM in the library, which validates and records). +Route rebuilds `.work/translation_report.json` + `gaps.json` from convert's baseline, the stored fills +of convert's own gaps, the plan, and the agent's outputs in `metadata/agentic_conversion.json`. Each +**routing unit** (a component, or an accepted grouping) becomes: -### 3a — Keep a pipeline 1:1 +- **deterministic**: the baseline pipelines stay (outcome `deterministic`). +- **agentic with a stored output** whose members match: the output's pipelines replace the members, + and their routed gaps are dropped (outcome `agentic-applied`). +- **agentic without one**: every task becomes a `PlaceholderActivity` with one pipeline-tagged gap, + the deterministic translation kept as its `raw_definition` (outcome `agentic-not-viable` — package + refuses until it is filled). -To convert a routed-agentic pipeline agentically but keep it as one pipeline, combine its component -with one authored pipeline of the **same name** (see 3b). The per-pipeline -`convert --merge-agentic` is for convert's **own** agentic gaps only: it refuses, writing nothing, any -result that names a routed-agentic pipeline, or whose activity would first land in one, with "this -pipeline is routed agentic; fill it with fill-agentic combine (to change the approach, change the plan, -re-run convert and route, then combine again)". +When nothing is routed agentic (and no gap fill is stored), route writes convert's report and gaps back +byte for byte, with no record — the non-breaking guarantee. Otherwise it stamps a **routing record** +onto the report (`_routing_record`): the plan's hash, the baseline report and gaps hashes, and one +entry per unit with its `members`, `decision`, `outcome`, `output_sha256` (set only while its stored +output is applied) and `fingerprint` (a grouping also lists its `components`). The plan is freely +editable: change it and route again as often as needed; the same plan gives the same bytes, and a +changed unit is re-derived. -### 3b — Cross-pipeline COMBINE (N pipelines → M) +## Step 3 — Fill each routed-agentic unit with fill-agentic -When the decision is a re-architecture that changes pipeline count — e.g. five extractor pipelines -collapse onto **one** Lakeflow Connect pipeline — use `fill-agentic combine`. You author the -replacement pipeline(s) as IR dicts and the whole routed group is swapped for them. +A routed-agentic unit is filled **only** by `fill-agentic`, which replaces the whole unit with +pipelines you author (no LLM in the library, which validates and records). ```bash -"$PY" -m flowx.adapter fill-agentic combine \ +"$PY" -m flowx.adapter fill-agentic \ --output-dir \ --members "IngestSalesforce,IngestWorkday" \ --pipelines-path authored_pipelines.json \ @@ -209,35 +200,39 @@ replacement pipeline(s) as IR dicts and the whole routed group is swapped for th MCP: `flowx(command="fill_agentic", parameters={"output_dir": ..., "members": [...], "pipelines": [...]})` (pass `pipelines` inline as a list, or `pipelines_path`). -`combine` is the only action. Its guarantees: - -- `--members` (comma-separated; MCP accepts a list) must **exactly match** a routed-**agentic** - component in the recorded `metadata/conversion_plan.json`, whose `inventory_sha256`, - `source_graphs_sha256` and `agentic_insights_sha256` must still match the current inventory. A partial group, a superset, a typo, or a deterministic component is refused - — you can't swap pipelines the plan didn't route agentic. The report must carry a routing record - that matches that plan (route has applied it). +To keep a pipeline 1:1, fill its component with one authored pipeline of the **same name**. The +per-pipeline `convert --merge-agentic` is for convert's **own** agentic gaps only: it refuses, writing +nothing, any result that names a routed-agentic pipeline, or whose activity would first land in one, +with "this pipeline is routed agentic; fill its component with fill-agentic (run fill-agentic again +with new pipelines to change the approach)". After routing, a merge into the live report is also +stored as a gap fill in `metadata/agentic_conversion.json`, so every rebuild re-applies it on top of +the unchanged baseline. + +Its guarantees: + +- `--members` (comma-separated; MCP accepts a list) must **exactly match** a routed-**agentic** unit — + a component, or an accepted grouping's full member list — in the recorded plan, whose + `inventory_sha256`, `source_graphs_sha256` and `agentic_insights_sha256` must still match the + current inventory, with nothing pending. A partial group, a superset, a typo, or a deterministic + component is refused. The report must carry a routing record that matches that plan. - **Same authored pipelines** (the canonical hash covers every field of every authored pipeline): - `already_combined: true` with the message "already applied, unchanged"; nothing is written. This - is a no-op even after a re-route, allowing idempotent tooling. If the report no longer reflects the - stored combine, or the stored entry was edited by hand, combine stores the pipelines again, - rebuilds, validates and writes the result. -- **Different authored pipelines**: replaces the stored combine for that component only, rebuilds, - and writes the result (if structural validation passes). The `route_audit.json` records the replacement, - showing the old `fingerprint` in `replacements` and the new `fingerprint`. -- **Unique names**: an authored pipeline name may reuse a member of this component, but not another - name in the same list, a member of another component, or a pipeline another component's combine - authored; a clash is refused and nothing is written. -- `--pipelines-path` is a JSON **list** of pipeline IR dicts (the authored replacements), typically - carrying `AgenticComponentActivity` nodes (see below). Empty `pipelines` list is refused. -- Each authored pipeline **must** carry the source tag `"tags": {"source": "adf"}` (routing/agentic - conversion is ADF-only). Combine asserts this up front and **fails closed** (nothing written, with - a clear message) on a missing or non-`adf` tag, so a mis-tagged pipeline is caught here rather than - surviving to the package preflight. + `already_applied: true` with the message "already applied, unchanged"; nothing is written. If the + report no longer reflects the stored output, or the stored entry was edited by hand, it stores the + pipelines again, rebuilds, validates and writes the result. +- **Different authored pipelines** replace the stored output for that unit only and add + `{"from", "to"}` to the entry's `replaced` history (the first fill is not a replacement). The history + lives in `agentic_conversion.json`, so it survives switching the unit to deterministic and back. +- **Unique names**: an authored pipeline name may reuse a member of this unit, but not another name in + the same list, a member of another unit, a pipeline another unit's output authored, or a name that + shares a bundle folder with another pipeline once normalised (`Sales Load` and `Sales-Load` would + overwrite each other). +- **Fully converted**: an authored pipeline that still holds a `PlaceholderActivity` (top level or + inside a ForEach / If / Switch) is refused. +- Each authored pipeline **must** carry `"tags": {"source": "adf"}`; a missing or non-`adf` tag fails + closed. An empty `pipelines` list is refused. - The merged report is **always** validated with the structural bundle invariants (a real `prepare → write_bundle` pass) **before it is written** — no bypass — so a duplicate key, dangling - dependency, cycle, or dangling pipeline/run_job reference can never land on disk. Validation happens - first; if any violation is found, `ok` is `false`, `violations` lists them, and nothing is written - (all-or-nothing). + dependency, cycle, or dangling pipeline/run_job reference can never land on disk (all-or-nothing). ### Authoring an `AgenticComponentActivity` (the escape hatch) @@ -269,7 +264,7 @@ the package phase writes verbatim: ``` - `tags` — the authored pipeline **must** set `tags.source` to `"adf"` (routing/agentic is ADF-only); - combine fails closed otherwise (see the combine guarantees above). + fill-agentic fails closed otherwise. - `files` — files written below the bundle `src/`; each is `path` + either UTF-8 `content` or base64 `binary_content`. - `resources` — bundle resources in the `resource_key` + raw `definition` shape the bundle writer @@ -303,30 +298,38 @@ applies when authoring recommended patterns.) ## Step 4 — Continue to package -Once the routed-agentic groups are filled and the report validates, continue with `flowx-convert`'s -just-in-time configuration (`inspect`/`modify`) as usual, then `flowx-package`. The recorded -`metadata/conversion_plan.json` is kept alongside `inventory.json` as the decision of record. It is the -library's typed `ConversionPlan` (schema 2): one decision per component, bound to the inventory -fingerprint, the saved `source_graphs.json` and the saved `agentic_insights.json` it was decided on, -with a reserved, empty `assignments` list per component for per-node routing later. - -Package verifies the routing plan is still current against the inventory, checks the saved baseline -still has the hashes the routing record names ("re-run convert, then route" otherwise) and that no -stored combine was edited ("re-run fill-agentic combine"), then replays `rebuild` from the baseline, -the plan, and the stored combines. The live `.work/translation_report.json` must match the rebuild -exactly, record included; otherwise it refuses with "re-run route". Any other packaged report — -modify's configured `.work/translation_report.stamped.json`, or a `modify --out` copy — must carry -the live report's routing record; otherwise it refuses with "the configured report is out of date; -re-run modify". Route, combine and merge never rewrite -the configured report, so run `modify` again after any of them. - -`metadata/route_audit.json` records per component the `decision`, `outcome`, `fingerprint`, -`combine_sha256`, and `replacements` (changes to the fingerprint across re-routes), plus the hashes -of the plan, the baseline report and gaps, the inventory, and the packaged report. It survives the -prune of `.work/` and documents what was packaged and how. +Once every agentic unit is filled, continue with `flowx-convert`'s just-in-time configuration +(`inspect`/`modify`) as usual, then `flowx-package`. The recorded `metadata/conversion_plan.json` is the +library's typed `ConversionPlan` (schema 3), bound to the inventory fingerprint, the saved +`source_graphs.json` and the saved `agentic_insights.json` it was decided on, with a reserved, empty +`assignments` list per component for per-node routing later. + +Package refuses, writing nothing, when: + +- `agentic_insights.json` and `inventory.json` disagree (or the file fails its own hash), plan or no + plan — enrich may have stopped between its two writes; run enrich again with the same insights + (locally, delete `metadata/.enrich.lock` first if it is still there; on the hosted server, run + discover again, then enrich prepare and apply); +- the report carries a routing record but `conversion_plan.json` is missing; +- the plan is stale against the inventory, is not a complete decision, or has a decision pending; +- a routed-agentic unit has **no agent output yet** — an agentic decision is packaged agentically, + never as the deterministic stand-in; fill it, or decide it deterministic and route again; +- the saved baseline no longer has the hashes the record names ("re-run convert, then route"), or a + stored output or gap fill was edited ("re-run fill-agentic", or merge that gap again); +- the live `.work/translation_report.json` does not match a fresh rebuild ("re-run route"), or any + other packaged report — modify's configured `.work/translation_report.stamped.json`, or a + `modify --out` copy — does not carry the live report's routing record ("the configured report is + out of date; re-run modify"). Route, fill-agentic and merge never rewrite the configured report, so + run `modify` again after any of them. + +`metadata/route_audit.json` records per unit the `decision`, `outcome`, `fingerprint`, `output_sha256` +and `replaced` history (a grouping also lists `grouped_components`), the gap fills applied, the routing +`conversation`, and the hashes of the plan (the same canonical hash the routing record holds), the +baseline report and gaps, the inventory, and the packaged report. It survives the prune of `.work/` +and documents what was packaged and how. ## Reference -- `flowx-enrich` skill — the `insights` layer routing's agentic option consumes. +- `flowx-enrich` skill — the `insights` layer routing's agentic option and groupings consume. - `flowx-convert` — the deterministic engine, `merge_agentic` for convert's own gaps, and just-in-time config. - `flowx-package` — turns the (filled) report into the deployable DAB bundle. diff --git a/src/flowx/adapter/__main__.py b/src/flowx/adapter/__main__.py index 4e43ffc4..e99ab69f 100644 --- a/src/flowx/adapter/__main__.py +++ b/src/flowx/adapter/__main__.py @@ -168,27 +168,28 @@ def _run_enrich(args: argparse.Namespace) -> int: def _run_route(args: argparse.Namespace) -> int: - """Implements the reshaped ``route``: one command that recommends, decides, records, and edits. - - Groups pipelines by connected component and computes the per-component recommendation (reusing - :mod:`flowx.routing`). Then: - - * with **no decision** on a non-TTY it emits the recommendation (components + both options + - findings + a ready-to-record default plan) and exits -- the dry run an agent reads first; - * with a **decision** -- ``--plan-path FILE`` (or ``--plan-path -`` to read the plan from stdin), - or an interactive TTY prompt -- it validates and records the fingerprint-bound - ``metadata/conversion_plan.json`` and then rebuilds ``.work/translation_report.json`` + - ``gaps.json`` from the deterministic baseline, so every routed-agentic group's tasks become - placeholder gaps or its stored combine (deterministic groups keep the baseline; nothing routed - agentic leaves convert's report byte-identical). Re-routing under any decisions is allowed. + """Implements ``route``: record the plan in ``conversion_plan.json``, then apply it once decided. + + Groups pipelines by connected component and computes the per-component recommendation and the + suggested groupings (reusing :mod:`flowx.routing`). The decisions come from ``--plan-path FILE`` + (or ``-`` for stdin) when given; otherwise from the recorded ``metadata/conversion_plan.json`` as + the agent edited it, with every component it does not decide left pending, and -- on a TTY -- an + interactive prompt for the pending ones. Route validates and records the plan (library fields + recomputed) and writes ``metadata/routing_review.html``. While any decision is pending it stops + there and leaves the report alone. Once every component is decided it rebuilds + ``.work/translation_report.json`` + ``gaps.json`` from the deterministic baseline: routed-agentic + units become placeholder gaps or their stored agent output, deterministic ones keep the baseline, + and nothing routed agentic leaves convert's report byte-identical. Re-routing under any decisions + is allowed. Triggers the convert phase in-process when the report is missing and ``--source`` / - ``--source-path`` are supplied. Returns 1 on a missing report it cannot produce or on a plan that - fails validation (report + plan left untouched in each case). + ``--source-path`` are supplied. Returns 1 on insights files that disagree, a missing report it + cannot produce, or a plan that fails validation (report + plan left untouched in each case). """ from flowx import routing - from flowx.models.conversion_plan import ConversionPlan - from flowx.route_agentic import REPORT_FILENAME, WORK_DIRNAME, apply_plan + from flowx.models.conversion_plan import PLAN_FILENAME, ConversionPlan + from flowx.reporting.routing_review import write_routing_review + from flowx.route_agentic import REPORT_FILENAME, WORK_DIRNAME, apply_plan, prompt_for_decisions inventory_path = args.output_dir / "metadata" / "inventory.json" if not inventory_path.exists(): @@ -199,16 +200,43 @@ def _run_route(args: argparse.Namespace) -> int: except (OSError, json.JSONDecodeError) as error: print(f"Failed to read {inventory_path}: {error}", file=sys.stderr) return 1 - recommendation = routing.build_recommendation(inventory) + disagree = routing.insights_file_violations(args.output_dir, inventory) + if disagree: + _emit_json({"ok": False, "violations": disagree, "components": 0}, args.out) + return 1 + plan_path = args.output_dir / "metadata" / PLAN_FILENAME try: - plan = _route_decision(args, recommendation) + if args.plan_path is not None: + plan = json.loads(sys.stdin.read() if str(args.plan_path) == "-" else args.plan_path.read_text("utf-8")) + else: + previous = json.loads(plan_path.read_text(encoding="utf-8")) if plan_path.exists() else None + plan = routing.carried_forward_plan(previous, inventory) + if sys.stdin.isatty(): + plan = prompt_for_decisions(routing.build_recommendation(inventory), authored=plan) except (OSError, json.JSONDecodeError) as error: print(f"Failed to read the conversion plan: {error}", file=sys.stderr) return 1 - if plan is None: - # No decision on a non-TTY: emit the recommendation as a dry run and stop. - _emit_json(recommendation, args.out) + + violations = routing.validate_plan(plan, inventory) + if violations: + _emit_json({"ok": False, "violations": violations, "components": 0}, args.out) + return 1 + try: + result = routing.record_plan(args.output_dir, plan=plan) + except (FileNotFoundError, OSError, ValueError, json.JSONDecodeError) as error: + print(f"Failed to record conversion plan: {error}", file=sys.stderr) + return 1 + if not result.get("ok"): + _emit_json(result, args.out) + return 1 + locations = {"plan_path": str(plan_path), "review_path": str(write_routing_review(args.output_dir))} + if result["pending"]: + message = ( + f"decide {result['pending']} in {plan_path} (or pass a plan), then run route again to apply it; " + f"the review page is {locations['review_path']}" + ) + _emit_json({**result, **locations, "applied": False, "message": message}, args.out) return 0 report_path = args.output_dir / WORK_DIRNAME / REPORT_FILENAME @@ -223,20 +251,6 @@ def _run_route(args: argparse.Namespace) -> int: file=sys.stderr, ) return 1 - - violations = routing.validate_plan(plan, inventory) - if violations: - _emit_json({"ok": False, "violations": violations, "components": 0}, args.out) - return 1 - - try: - result = routing.record_plan(args.output_dir, plan=plan) - except (FileNotFoundError, OSError, ValueError, json.JSONDecodeError) as error: - print(f"Failed to record conversion plan: {error}", file=sys.stderr) - return 1 - if not result.get("ok"): - _emit_json(result, args.out) - return 1 try: recorded = ConversionPlan.load(args.output_dir) if recorded is None: @@ -245,28 +259,11 @@ def _run_route(args: argparse.Namespace) -> int: except (FileNotFoundError, OSError, ValueError, json.JSONDecodeError) as error: print(f"Failed to edit translation report: {error}", file=sys.stderr) return 1 - _emit_json({**result, "edit": edit}, args.out) + write_routing_review(args.output_dir) + _emit_json({**result, **locations, "applied": True, "edit": edit}, args.out) return 0 -def _route_decision(args: argparse.Namespace, recommendation: dict[str, Any]) -> dict[str, Any] | None: - """Resolve the routing decision as an authored plan, or ``None`` for a dry-run recommendation. - - Precedence: ``--plan-path FILE`` (a file), ``--plan-path -`` (stdin JSON), then -- when stdin is - a TTY -- an interactive per-component prompt. On a non-TTY with no ``--plan-path`` there is no - decision, so the caller emits the recommendation instead. - """ - from flowx.route_agentic import prompt_for_decisions - - if args.plan_path is not None: - if str(args.plan_path) == "-": - return json.loads(sys.stdin.read()) - return json.loads(Path(args.plan_path).read_text(encoding="utf-8")) - if sys.stdin.isatty(): - return prompt_for_decisions(recommendation) - return None - - def _trigger_convert(args: argparse.Namespace) -> int: """Run the convert phase in-process when ``--source`` / ``--source-path`` are supplied. @@ -284,23 +281,21 @@ def _trigger_convert(args: argparse.Namespace) -> int: def _run_fill_agentic(args: argparse.Namespace) -> int: - """Implements ``fill-agentic combine``: the cross-pipeline pipeline-grain fill. + """Implements ``fill-agentic``: fill a routed-agentic unit with the agent's pipelines. - Replaces a routed-agentic group's pipelines (``--members`` as a comma-separated list) with the + Replaces a routed-agentic unit's pipelines (``--members`` as a comma-separated list) with the agent-authored pipeline(s) read from ``--pipelines-path`` (a JSON list of pipeline IR dicts, each - typically carrying ``AgenticComponentActivity`` nodes). The membership must exactly match a - routed-agentic component in the recorded, fingerprint-bound ``metadata/conversion_plan.json``, and - the merged report is always validated structurally before it is written back -- there is no bypass. - It is the only fill for a routed-agentic group; ``convert --merge-agentic`` fills convert's own gaps. + typically carrying ``AgenticComponentActivity`` nodes), and stores them in + ``metadata/agentic_conversion.json``. The membership must exactly match a routed-agentic component + or accepted grouping in the recorded, fingerprint-bound ``metadata/conversion_plan.json``, and the + merged report is always validated structurally before it is written back -- there is no bypass. + It is the only fill for a routed-agentic unit; ``convert --merge-agentic`` fills convert's own gaps. """ - from flowx.route_agentic import apply_combine_fill + from flowx.route_agentic import apply_agentic_output - if args.action != "combine": - print(f"Unknown fill-agentic action {args.action!r}; expected 'combine'.", file=sys.stderr) - return 2 members = [member.strip() for member in args.members.split(",") if member.strip()] if not members: - print("fill-agentic combine requires a non-empty --members list.", file=sys.stderr) + print("fill-agentic requires a non-empty --members list.", file=sys.stderr) return 2 try: authored = json.loads(Path(args.pipelines_path).read_text(encoding="utf-8")) @@ -311,12 +306,12 @@ def _run_fill_agentic(args: argparse.Namespace) -> int: print("--pipelines-path must contain a JSON list of pipeline IR dicts.", file=sys.stderr) return 2 try: - result = apply_combine_fill(args.output_dir, members, authored) + result = apply_agentic_output(args.output_dir, members, authored) except FileNotFoundError as error: print(str(error), file=sys.stderr) return 1 except (OSError, ValueError, json.JSONDecodeError) as error: - print(f"Failed to apply combine fill: {error}", file=sys.stderr) + print(f"Failed to apply the agent's output: {error}", file=sys.stderr) return 1 _emit_json(result, args.out) return 0 if result.get("ok") else 1 @@ -683,9 +678,9 @@ def _build_parser() -> argparse.ArgumentParser: route = subparsers.add_parser( "route", help=( - "One command: recommend a per-connected-component route, take the decision (interactive on " - "a TTY, else --plan-path / stdin), record metadata/conversion_plan.json, and edit the " - "translation report so routed-agentic groups become placeholder gaps." + "Write the per-component recommendation into metadata/conversion_plan.json (keeping the " + "decisions already there) plus metadata/routing_review.html; once every component is decided, " + "apply the plan to the translation report." ), ) route.add_argument( @@ -694,7 +689,7 @@ def _build_parser() -> argparse.ArgumentParser: required=True, help=( "Migration output directory (reads metadata/inventory.json + .work/translation_report.json; " - "records metadata/conversion_plan.json and edits the report for routed-agentic groups)." + "records metadata/conversion_plan.json and edits the report for routed-agentic units)." ), ) route.add_argument( @@ -702,9 +697,9 @@ def _build_parser() -> argparse.ArgumentParser: type=Path, default=None, help=( - "Path to the agent-authored conversion plan JSON to validate, record, and apply. Use '-' to " - "read the plan from stdin. Omit on a TTY for an interactive prompt, or on a non-TTY to emit " - "the recommendation as a dry run." + "Path to an authored conversion plan JSON (the decisions, or an edited copy of " + "conversion_plan.json) to validate, record, and apply. Use '-' to read it from stdin. Omit to " + "use metadata/conversion_plan.json as edited (with a prompt for pending decisions on a TTY)." ), ) route.add_argument( @@ -722,18 +717,17 @@ def _build_parser() -> argparse.ArgumentParser: "--out", type=Path, default=None, - help="Optional output file for the recommendation / result JSON; defaults to stdout.", + help="Optional output file for the result JSON; defaults to stdout.", ) fill_agentic = subparsers.add_parser( "fill-agentic", help=( - "Cross-pipeline COMBINE fill: replace a routed group's pipelines with agent-authored " - "pipeline(s), validated structurally before writing. It is the only fill for a routed-agentic " - "group; convert --merge-agentic fills convert's own gaps." + "Fill a routed-agentic unit: replace its pipelines with agent-authored pipeline(s), stored in " + "metadata/agentic_conversion.json and validated structurally before writing. It is the only fill " + "for a routed-agentic unit; convert --merge-agentic fills convert's own gaps." ), ) - fill_agentic.add_argument("action", choices=("combine",), help="'combine' performs the pipeline-grain fill.") fill_agentic.add_argument( "--output-dir", type=Path, @@ -743,7 +737,7 @@ def _build_parser() -> argparse.ArgumentParser: fill_agentic.add_argument( "--members", required=True, - help="Comma-separated pipeline names of the routed group to replace.", + help="Comma-separated pipeline names of the routed-agentic component or accepted grouping to replace.", ) fill_agentic.add_argument( "--pipelines-path", diff --git a/src/flowx/bundler/dab_writer.py b/src/flowx/bundler/dab_writer.py index 2e21f1c3..81b52e52 100644 --- a/src/flowx/bundler/dab_writer.py +++ b/src/flowx/bundler/dab_writer.py @@ -388,51 +388,50 @@ def _default_report_path(output_dir: Path) -> Path: def _routing_plan_failures(output_dir: Path, report_path: Path) -> list[str]: """Check a recorded routing plan still matches what is about to be packaged. - A plan is bound to the inventory, the saved source graphs and the saved agentic insights it was - decided on. If discover or enrich ran again since route, package refuses rather than shipping a - report that no longer reflects the user's decision. It also refuses when the report's routing - record disagrees with the plan: a different plan hash, different components, members or - decisions, or no record although the plan routes a component agentic. - - The live ``.work/translation_report.json`` is the report route, combine and merge keep in step + Whenever enrich ran, ``agentic_insights.json`` must still agree with the inventory's ``insights`` + block, plan or no plan. A report that carries a routing record needs its plan: package refuses + one whose ``conversion_plan.json`` is missing. A plan is bound to the inventory, the saved source + graphs and the saved agentic insights it was decided on, and must still decide every component + once with nothing pending; if discover or enrich ran again since route, package refuses rather + than shipping a report that no longer reflects the user's decision. It also refuses when the + report's routing record disagrees with the plan, and while any agentic unit has no agent output + yet: an agentic decision is packaged agentically, never as the deterministic stand-in. + + The live ``.work/translation_report.json`` is the report route, the fill and merge keep in step with the saved deterministic baseline, so it is always the one replayed (see :func:`_live_report_failures`; its remedy is "re-run route"). Any other packaged report -- modify's configured copy, or one written with ``modify --out`` -- can't be replayed, because modify changes its pipelines, so its routing record must equal the live one; otherwise package refuses with "the configured report is out of date; re-run modify". - Returns one message per problem; an empty list means there is no recorded plan or it still matches. + Returns one message per problem; an empty list means there is no routing to check or it still matches. """ from flowx.models.conversion_plan import ConversionPlan from flowx.route_agentic import REPORT_FILENAME, WORK_DIRNAME, routing_record - from flowx.routing import plan_binding_violations + from flowx.routing import insights_file_violations, plan_binding_violations, recorded_plan_violations metadata_dir = Path(output_dir) / "metadata" + inventory_path = metadata_dir / "inventory.json" + inventory: Any = None + if inventory_path.is_file(): + try: + inventory = json.loads(inventory_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + return [f"metadata/inventory.json is unreadable: {error}"] + insights_failures = insights_file_violations(Path(output_dir), inventory if isinstance(inventory, dict) else None) + if insights_failures: + return insights_failures + try: plan = ConversionPlan.load(Path(output_dir)) except (OSError, ValueError) as error: return [f"metadata/conversion_plan.json is unreadable: {error}"] - if plan is None: - return [] - - inventory_path = metadata_dir / "inventory.json" - if not inventory_path.is_file(): - return ["metadata/conversion_plan.json was recorded but metadata/inventory.json is missing"] - try: - inventory = json.loads(inventory_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as error: - return [f"metadata/inventory.json is unreadable: {error}"] - if not isinstance(inventory, dict): - return ["metadata/inventory.json must contain a JSON object"] - - stale = plan_binding_violations(plan, inventory) - if stale: - return [f"metadata/{message}" if message.startswith("conversion_plan.json") else message for message in stale] + report: Any = None try: report = json.loads(Path(report_path).read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): - return [] + report = None live_path = Path(output_dir) / WORK_DIRNAME / REPORT_FILENAME configured = Path(report_path).resolve() != live_path.resolve() live_report = report @@ -441,7 +440,37 @@ def _routing_plan_failures(output_dir: Path, report_path: Path) -> list[str]: live_report = json.loads(live_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): live_report = {} - failures = _live_report_failures(Path(output_dir), plan.to_dict(), live_report) + + if plan is None: + if routing_record(report) is not None or routing_record(live_report) is not None: + return [ + f"{REPORT_FILENAME} was routed (it carries a routing record) but metadata/conversion_plan.json " + "is missing; restore the plan, or re-run convert to package without routing" + ] + return [] + + if inventory is None: + return ["metadata/conversion_plan.json was recorded but metadata/inventory.json is missing"] + if not isinstance(inventory, dict): + return ["metadata/inventory.json must contain a JSON object"] + + stale = plan_binding_violations(plan, inventory) + if stale: + return [f"metadata/{message}" if message.startswith("conversion_plan.json") else message for message in stale] + invalid = recorded_plan_violations(plan, inventory) + if invalid: + return [ + f"metadata/conversion_plan.json is not a valid decision: {violation}; re-run route" for violation in invalid + ] + pending = plan.pending_components() + if pending: + return [ + f"components {pending} have no decision yet; decide them in metadata/conversion_plan.json and re-run route" + ] + + if report is None: + return [] + failures = _live_report_failures(Path(output_dir), plan.to_dict(), live_report, inventory) if failures: return failures if configured and routing_record(report) != routing_record(live_report): @@ -449,22 +478,29 @@ def _routing_plan_failures(output_dir: Path, report_path: Path) -> list[str]: return [] -def _live_report_failures(output_dir: Path, plan_document: dict[str, Any], report: Any) -> list[str]: +def _live_report_failures( + output_dir: Path, plan_document: dict[str, Any], report: Any, inventory: dict[str, Any] +) -> list[str]: """Check the live translation report still equals a fresh replay of the rebuild. The report's routing record must match the plan, the saved deterministic baseline must still have - the hashes the record names, and no stored combine may have been edited. The rebuild from the - baseline, the plan and the stored combines must then reproduce the report exactly, record included. - A report without a record passes only when the plan routes nothing agentic. + the hashes the record names and the inventory's pipelines, and no stored agent output or gap fill + may have been edited. The rebuild from the baseline, the stored gap fills, the plan and the stored + outputs must then reproduce the report exactly, record included. Finally every agentic unit must + have its agent output applied. A report without a record passes only when the plan routes nothing + agentic. """ from flowx.discovery_serde import canonical_sha256 from flowx.models.conversion_plan import DECISION_AGENTIC from flowx.route_agentic import ( - COMBINES_FILENAME, + AGENTIC_OUTPUT_FILENAME, + OUTCOME_AGENTIC_NOT_VIABLE, REPORT_FILENAME, ROUTING_RECORD_KEY, + agentic_pipeline_names, + baseline_pipeline_mismatch, + load_agentic_output, load_baseline, - load_combines, rebuild, routing_record, routing_record_mismatches, @@ -494,28 +530,39 @@ def _live_report_failures(output_dir: Path, plan_document: dict[str, Any], repor actual_hashes = (hashlib.sha256(baseline_report_bytes).hexdigest(), hashlib.sha256(baseline_gaps_bytes).hexdigest()) if recorded_hashes != actual_hashes: return ["the deterministic baseline is missing or changed; re-run convert, then route"] + mismatch = baseline_pipeline_mismatch(baseline_report, inventory) + if mismatch and agentic_pipeline_names(plan_document): + return [mismatch] try: - stored, combines = load_combines(output_dir) + stored = load_agentic_output(output_dir) except (OSError, ValueError) as error: - return [f"metadata/{COMBINES_FILENAME} is unreadable: {error}"] - edited = sorted(set(stored) - set(combines)) - if edited: - return [f"the combine store has been edited ({', '.join(edited)}); re-run fill-agentic combine"] + return [f"metadata/{AGENTIC_OUTPUT_FILENAME} is unreadable: {error}"] + if stored.edited: + return [ + f"metadata/{AGENTIC_OUTPUT_FILENAME} has been edited ({', '.join(stored.edited)}); " + "re-run fill-agentic for that component, or merge that gap again" + ] rebuilt_report, _, rebuilt_record = rebuild( baseline_report, baseline_gaps, plan_document, - combines, - record, + stored.outputs, + stored.gap_fills, baseline_report_bytes, baseline_gaps_bytes, ) report_without_record = {key: value for key, value in report.items() if key != ROUTING_RECORD_KEY} if rebuilt_record != record or report_without_record != rebuilt_report: return [f"{REPORT_FILENAME} does not match a fresh rebuild; re-run route"] - return [] + + return [ + f"component {unit_id!r} ({', '.join(entry.get('members') or [])}) is routed agentic but has no agent " + "output yet; fill it with fill-agentic, or decide it deterministic and re-run route" + for unit_id, entry in record["components"].items() + if entry.get("outcome") == OUTCOME_AGENTIC_NOT_VIABLE + ] def _file_sha256(path: Path) -> str | None: @@ -527,27 +574,28 @@ def _file_sha256(path: Path) -> str | None: def _write_route_audit(output_dir: Path, report_path: Path | None = None) -> Path | None: - """Persist a compact routing/gaps audit to ``metadata/route_audit.json`` before the prune. + """Persist a compact routing audit to ``metadata/route_audit.json`` before the prune. Package prunes the transient ``.work/`` folder (translation report + ``gaps.json``) by default, which erases the "what did routing change?" trail. When a routing decision was recorded - (``metadata/conversion_plan.json`` exists), summarise the routed components -- their decisions, - and from the report's routing record their outcomes, fingerprints, applied combine hashes and - replacements -- and the gaps routing introduced in agentic-routed pipelines, into an additive - ``metadata/`` artifact that survives the prune. It also records hashes of the inventory, the saved - source graphs and agentic insights the plan was bound to, the plan, and the packaged report, plus - the baseline report and gaps route started from as the routing record names them, so the trail - can be checked against those files later. + (``metadata/conversion_plan.json`` exists), summarise each routing unit -- its decision, and from + the report's routing record its outcome, fingerprint and applied output hash, plus the history of + outputs it replaced from ``metadata/agentic_conversion.json`` -- and the gap fills applied, into an + additive ``metadata/`` artifact that survives the prune. It also records the routing conversation, + the hashes of the inventory, the saved source graphs and agentic insights the plan was bound to, + the plan (the same canonical hash the routing record holds), and the packaged report, plus the + baseline report and gaps route started from as the routing record names them, so the trail can be + checked against those files later. Returns the written path, or ``None`` when there is no recorded plan (no routing happened) -- so the no-route path writes nothing and stays byte-identical. """ from flowx.discovery_insights import inventory_fingerprint - from flowx.models.conversion_plan import DECISION_AGENTIC, DECISION_DETERMINISTIC, PLAN_FILENAME, ConversionPlan - from flowx.route_agentic import OUTCOME_DETERMINISTIC, routing_record + from flowx.discovery_serde import canonical_sha256 + from flowx.models.conversion_plan import DECISION_AGENTIC, DECISION_DETERMINISTIC, ConversionPlan + from flowx.route_agentic import OUTCOME_DETERMINISTIC, load_agentic_output, routing_record, routing_units metadata_dir = Path(output_dir) / "metadata" - plan_path = metadata_dir / PLAN_FILENAME try: plan = ConversionPlan.load(Path(output_dir)) except (OSError, ValueError): @@ -569,63 +617,59 @@ def _write_route_audit(output_dir: Path, report_path: Path | None = None) -> Pat record = routing_record(json.loads(Path(report_path).read_text(encoding="utf-8"))) except (OSError, json.JSONDecodeError): record = None - recorded_components = record.get("components") if record is not None else None - if not isinstance(recorded_components, dict): - recorded_components = {} + recorded_units = record.get("components") if record is not None else None + if not isinstance(recorded_units, dict): + recorded_units = {} + try: + stored = load_agentic_output(Path(output_dir)) + except ValueError: + stored = None + stored_entries = stored.document.get("components", {}) if stored is not None else {} - components: list[dict[str, Any]] = [] + plan_document = plan.to_dict() + units: list[dict[str, Any]] = [] agentic_pipelines: set[str] = set() - for component in plan.components: - recorded = recorded_components.get(component.component_id) - component_entry: dict[str, Any] = { - "component_id": component.component_id, - "members": list(component.members), - "decision": component.decision, - } + for unit_id, unit in routing_units(plan_document).items(): + recorded = recorded_units.get(unit_id) + unit_entry: dict[str, Any] = {"component_id": unit_id, "members": list(unit["members"])} + if unit.get("components"): + unit_entry["grouped_components"] = list(unit["components"]) + unit_entry["decision"] = unit["decision"] if isinstance(recorded, dict): - component_entry["outcome"] = recorded.get("outcome") - component_entry["combine_sha256"] = recorded.get("combine_sha256") - component_entry["fingerprint"] = recorded.get("fingerprint") - if recorded.get("replacements"): - component_entry["replacements"] = recorded.get("replacements") + unit_entry["outcome"] = recorded.get("outcome") + unit_entry["output_sha256"] = recorded.get("output_sha256") + unit_entry["fingerprint"] = recorded.get("fingerprint") else: - component_entry["outcome"] = OUTCOME_DETERMINISTIC if component.decision == DECISION_DETERMINISTIC else None - component_entry["combine_sha256"] = None - components.append(component_entry) - if component.decision == DECISION_AGENTIC: - agentic_pipelines.update(component.members) - - gaps_introduced: list[dict[str, Any]] = [] - gaps_path = Path(output_dir) / ".work" / "gaps.json" - if gaps_path.is_file(): - try: - gaps = json.loads(gaps_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - gaps = [] - for gap in gaps if isinstance(gaps, list) else []: - if isinstance(gap, dict) and gap.get("pipeline") in agentic_pipelines: - gaps_introduced.append( - { - "pipeline": gap.get("pipeline"), - "activity_name": gap.get("activity_name"), - "activity_type": gap.get("activity_type"), - } - ) + unit_entry["outcome"] = OUTCOME_DETERMINISTIC if unit["decision"] == DECISION_DETERMINISTIC else None + unit_entry["output_sha256"] = None + stored_entry = stored_entries.get(unit_id) + if isinstance(stored_entry, dict) and stored_entry.get("replaced"): + unit_entry["replaced"] = stored_entry["replaced"] + units.append(unit_entry) + if unit["decision"] == DECISION_AGENTIC: + agentic_pipelines.update(unit["members"]) + + baseline_report_sha256 = record.get("baseline_report_sha256") if record is not None else None + gap_fills = [ + {"pipeline": fill["pipeline"], "activity_name": fill["activity_name"], "task_sha256": fill["task_sha256"]} + for fill in (stored.gap_fills if stored is not None else []) + if baseline_report_sha256 is not None and fill["baseline_report_sha256"] == baseline_report_sha256 + ] audit = { - "schema": "flowx.route_audit/v1", + "schema": "flowx.route_audit/v2", "recorded_against_inventory_sha256": plan.inventory_sha256, "inventory_sha256": inventory_sha256, "source_graphs_sha256": plan.source_graphs_sha256, "agentic_insights_sha256": plan.agentic_insights_sha256, - "conversion_plan_sha256": _file_sha256(plan_path), - "baseline_report_sha256": record.get("baseline_report_sha256") if record is not None else None, + "conversion_plan_sha256": canonical_sha256(plan_document), + "baseline_report_sha256": baseline_report_sha256, "baseline_gaps_sha256": record.get("baseline_gaps_sha256") if record is not None else None, "translation_report_sha256": _file_sha256(report_path) if report_path is not None else None, - "components": components, + "components": units, "agentic_pipelines": sorted(agentic_pipelines), - "gaps_count": len(gaps_introduced), - "gaps_introduced": gaps_introduced, + "gap_fills": gap_fills, + "conversation": [entry.to_dict() for entry in plan.conversation], } metadata_dir.mkdir(parents=True, exist_ok=True) audit_path = metadata_dir / "route_audit.json" diff --git a/src/flowx/ir_serde.py b/src/flowx/ir_serde.py index a88efefb..89d61c3b 100644 --- a/src/flowx/ir_serde.py +++ b/src/flowx/ir_serde.py @@ -619,7 +619,7 @@ def _locate_task(tasks: list[dict[str, Any]], activity_name: str) -> tuple[list[ return None -def _find_and_replace_task(tasks: list[dict[str, Any]], activity_name: str, replacement: dict[str, Any]) -> bool: +def replace_task_by_name(tasks: list[dict[str, Any]], activity_name: str, replacement: dict[str, Any]) -> bool: """Replace the task named *activity_name* with *replacement*, recursing into containers. Preserves the placeholder's ``task_key`` and ``depends_on`` when the replacement omits them so @@ -667,16 +667,16 @@ def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Pat dependencies, timeout, and retries whatever the agent wrote. This fills convert's own agentic gaps. A pipeline the routing record routes agentic is filled - only by ``fill-agentic combine``, so a result that names one, or an untargeted result that would + only by ``fill-agentic``, so a result that names one, or an untargeted result that would land in one, is refused and nothing is written. Results apply one by one in file-name order, and each is checked against the pipeline it actually lands in. When the report has a routing record and the merge is written in place into the live - ``/.work/translation_report.json``, every merge is applied to the same pipeline of - route's saved deterministic baseline too, so the next rebuild keeps it, and the record's - ``baseline_report_sha256`` is refreshed. A merge written anywhere else (``output_path``, or into - a copy such as modify's configured report) leaves the baseline alone. Nothing is written until - every result has been applied. + ``/.work/translation_report.json``, every merge is also stored as a gap fill in + ``metadata/agentic_conversion.json``, so every later rebuild applies it on top of route's saved + deterministic baseline, which is never changed. A merge written anywhere else (``output_path``, or + into a copy such as modify's configured report) stores nothing. Nothing is written until every + result has been applied. Args: report_path: ``translation_report.json`` produced by the translate phase. @@ -691,15 +691,12 @@ def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Pat ValueError: A result would land in a pipeline the routing record routes agentic, or the routing baseline is missing or lacks the task a merge replaced. """ - import hashlib - from flowx.route_agentic import ( - BASELINE_DIRNAME, - BASELINE_REPORT_FILENAME, REPORT_FILENAME, ROUTED_AGENTIC_MERGE_REFUSED, WORK_DIRNAME, load_baseline, + record_gap_fills, routed_agentic_pipelines, routing_record, ) @@ -714,8 +711,8 @@ def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Pat updates_baseline = ( record is not None and destination.resolve() == report_path.resolve() == live_report_path.resolve() ) - baseline_report: Any = None baseline_by_name: dict[Any, dict[str, Any]] = {} + gap_fills: list[dict[str, Any]] = [] if updates_baseline: baseline_report = load_baseline(output_dir, fresh=False)[0] baseline_pipelines = baseline_report["pipelines"] if "pipelines" in baseline_report else [baseline_report] @@ -748,23 +745,22 @@ def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Pat logger.warning("No placeholder named '%s' found for %s", activity_name, result_file.name) unmatched += 1 continue - _find_and_replace_task(target["tasks"], activity_name, dict(task)) + replace_task_by_name(target["tasks"], activity_name, dict(task)) if updates_baseline: baseline_pipeline = baseline_by_name.get(target.get("name")) - if baseline_pipeline is None or not _find_and_replace_task( + if baseline_pipeline is None or not replace_task_by_name( baseline_pipeline.get("tasks", []), activity_name, dict(task) ): raise ValueError( f"the routing baseline has no task {activity_name!r} in pipeline {target.get('name')!r}; " "re-run route, then merge again" ) + gap_fills.append({"pipeline": str(target.get("name")), "activity_name": activity_name, "task": task}) merged += 1 logger.info("Merged agentic result for '%s' from %s", activity_name, result_file.name) - if record is not None and updates_baseline and merged: - baseline_bytes = json.dumps(baseline_report, indent=2, default=str).encode("utf-8") - (output_dir / WORK_DIRNAME / BASELINE_DIRNAME / BASELINE_REPORT_FILENAME).write_bytes(baseline_bytes) - record["baseline_report_sha256"] = hashlib.sha256(baseline_bytes).hexdigest() + if record is not None and gap_fills: + record_gap_fills(output_dir, gap_fills, str(record.get("baseline_report_sha256"))) destination.write_text(json.dumps(report, indent=2, default=str), encoding="utf-8") logger.info("Wrote merged report to %s (%d merged, %d unmatched)", destination, merged, unmatched) return merged, unmatched diff --git a/src/flowx/mcp/server.py b/src/flowx/mcp/server.py index ffe95764..5499f9ac 100644 --- a/src/flowx/mcp/server.py +++ b/src/flowx/mcp/server.py @@ -55,9 +55,9 @@ def _transport_security() -> TransportSecuritySettings: flowx("enrich", {"output_dir": "...", "action": "prepare"}) # optional: read inventory.json to author insights flowx("enrich", {"output_dir": "...", "insights": {...}}) # optional: record agent-authored insights flowx("convert", {"source": "adf", "output_dir": "..."}) - flowx("route", {"output_dir": "..."}) # optional: recommend, then re-call with a plan - flowx("route", {"output_dir": "...", "plan": {...}}) # record the decision + edit the report - flowx("fill_agentic", {"output_dir": "...", "members": [...], "pipelines": [...]}) # fill a routed-agentic group + flowx("route", {"output_dir": "..."}) # optional: write the plan + review page + flowx("route", {"output_dir": "...", "plan": {...}}) # record the decisions, apply once decided + flowx("fill_agentic", {"output_dir": "...", "members": [...], "pipelines": [...]}) # fill an agentic unit flowx("inspect", {"report_path": "/.work/translation_report.json"}) flowx("apply_answers", {"report_path": "...", "answers": ["id=value"], "output_dir": "..."}) flowx("package", {"output_dir": "...", "catalog": "main", "schema": "default"}) @@ -350,14 +350,17 @@ def _run(path: str) -> dict[str, Any]: def _cmd_route(p: dict[str, Any]) -> dict[str, Any]: - """One command: recommend a per-component route, or record + apply the user's decision. - - With **no** ``plan`` / ``plan_path`` this emits the recommendation (components, both conversion - options per component, findings, and a ready-to-record default plan) -- the dry run an agent reads - first. With a decision (``plan`` inline or ``plan_path`` -- at most one) it validates and records - the fingerprint-bound ``metadata/conversion_plan.json`` AND edits ``.work/translation_report.json`` - + ``gaps.json`` so every routed-agentic group's tasks become placeholder gaps (deterministic groups - untouched). The returned ``ok`` reflects the CLI's success; ``result`` carries its JSON. + """Record the conversion plan, and apply it once every component is decided. + + With **no** ``plan`` / ``plan_path``, route writes its recommendation straight into + ``metadata/conversion_plan.json`` (keeping the decisions already recorded there, every other one + pending) plus ``metadata/routing_review.html``, and applies the plan once nothing is pending. With + a plan (``plan`` inline or ``plan_path`` -- at most one; the decisions, or an edited copy of the + recorded plan) it records those decisions instead. Either way the recorded plan and the review page + come back like ``enrich prepare`` returns the inventory: inline under ``bundle``, or uploaded to + ``output_volume_path`` / ``output_workspace_path``, because a hosted agent cannot read the server's + ``output_dir``; edit the plan and call route again with it as ``plan``. The returned ``ok`` + reflects the CLI's success; ``result`` carries its JSON. """ output_dir = p.get("output_dir", "./flowx_output") plan = p.get("plan") @@ -365,30 +368,41 @@ def _cmd_route(p: dict[str, Any]) -> dict[str, Any]: if plan is not None and plan_path is not None: return {"ok": False, "error": "provide at most one of 'plan' (inline object) or 'plan_path'."} + def _run(extra: list[str]) -> dict[str, Any]: + result = runner.run_adapter(["route", "--output-dir", output_dir, *extra, *source_args]) + payload = runner.parse_stdout_json(result) + ok = bool(isinstance(payload, dict) and payload.get("ok")) + return {"ok": ok, "result": payload, "process": result.as_dict(), **_route_files(p, Path(output_dir))} + # Forward source + resolved source-path so the MCP path mirrors the CLI: route can then trigger # convert when the report is missing. `source` is optional here (unlike discover/convert). source_args, cleanup = _route_source_args(p) try: if plan is None and plan_path is None: - result = runner.run_adapter(["route", "--output-dir", output_dir, *source_args]) - return {"ok": result.ok, "result": runner.parse_stdout_json(result), "process": result.as_dict()} - - def _run(path: str) -> dict[str, Any]: - result = runner.run_adapter(["route", "--output-dir", output_dir, "--plan-path", path, *source_args]) - payload = runner.parse_stdout_json(result) - ok = bool(isinstance(payload, dict) and payload.get("ok")) - return {"ok": ok, "result": payload, "process": result.as_dict()} - + return _run([]) if plan_path is not None: - return _run(str(plan_path)) + return _run(["--plan-path", str(plan_path)]) with tempfile.TemporaryDirectory(prefix="flowx-plan-") as temporary: inline_path = Path(temporary) / "conversion_plan.json" inline_path.write_text(json.dumps(plan, indent=2), encoding="utf-8") - return _run(str(inline_path)) + return _run(["--plan-path", str(inline_path)]) finally: cleanup() +def _route_files(p: dict[str, Any], output_dir: Path) -> dict[str, Any]: + """Deliver the recorded plan and the routing review page, when route wrote them.""" + metadata = output_dir / "metadata" + files = [metadata / name for name in ("conversion_plan.json", "routing_review.html")] + if not any(path.is_file() for path in files): + return {} + with tempfile.TemporaryDirectory(prefix="flowx-route-") as temporary: + for path in files: + if path.is_file(): + shutil.copy(path, temporary) + return _bundle_output(p, Path(temporary)) + + def _route_source_args(p: dict[str, Any]) -> tuple[list[str], Callable[[], None]]: """Resolve optional ``source`` / source-path into ``route`` CLI flags (empty when no source given). @@ -407,13 +421,14 @@ def _route_source_args(p: dict[str, Any]) -> tuple[list[str], Callable[[], None] def _cmd_fill_agentic(p: dict[str, Any]) -> dict[str, Any]: - """Cross-pipeline COMBINE fill: replace a routed group's pipelines with agent-authored pipeline(s). - - ``members`` (a list of pipeline names, or a comma-separated string) names the routed group to - replace; the authored pipeline IR dicts come inline as ``pipelines`` (a JSON list) or via - ``pipelines_path`` (exactly one, staged to a temp file so the same CLI contract runs on both). The - merged report is validated structurally before it is written back; ``ok`` is False (nothing - written) when a structural invariant is violated. It is the only fill for a routed-agentic group; + """Fill a routed-agentic unit: replace its pipelines with agent-authored pipeline(s). + + ``members`` (a list of pipeline names, or a comma-separated string) names the routed-agentic + component or accepted grouping to replace; the authored pipeline IR dicts come inline as + ``pipelines`` (a JSON list) or via ``pipelines_path`` (exactly one, staged to a temp file so the + same CLI contract runs on both). They are stored in ``metadata/agentic_conversion.json``. The merged + report is validated structurally before it is written back; ``ok`` is False (nothing written) when + a structural invariant is violated. It is the only fill for a routed-agentic unit; ``merge_agentic`` fills convert's own gaps only. """ output_dir = p.get("output_dir", "./flowx_output") @@ -429,7 +444,7 @@ def _cmd_fill_agentic(p: dict[str, Any]) -> dict[str, Any]: return {"ok": False, "error": "provide exactly one of 'pipelines' (inline list) or 'pipelines_path'."} def _run(path: str) -> dict[str, Any]: - args: list[Any] = ["fill-agentic", "combine", "--output-dir", output_dir, "--members", members] + args: list[Any] = ["fill-agentic", "--output-dir", output_dir, "--members", members] args += ["--pipelines-path", path] result = runner.run_adapter(args) payload = runner.parse_stdout_json(result) @@ -700,9 +715,9 @@ def flowx(command: str, parameters: dict[str, Any] | None = None) -> dict[str, A merge ADF agent results for convert's own agentic gaps. Refuses (writing nothing) any result landing in a pipeline the routing record routes agentic — fill those with fill_agentic. After routing, a merge written in place into the live - .work/translation_report.json also updates the stored deterministic baseline (so the next - rebuild keeps it) and the routing record's baseline hash; a merge written to output_path - leaves the baseline alone. Airflow's legacy name-based merge is disabled; use resolve_agentic. + .work/translation_report.json is also stored as a gap fill in metadata/agentic_conversion.json, + so every rebuild re-applies it on top of the unchanged deterministic baseline; a merge written + to output_path stores nothing. Airflow's legacy name-based merge is disabled; use resolve_agentic. - "resolve_agentic": source(req: "airflow"), action(req: prepare | stage | apply), output_dir, airflow_source_path, report_path, gap_id, candidates, replace, accept_gap | accept_gaps, accept_all, review_complete, review_manifest, reset — @@ -722,28 +737,37 @@ def flowx(command: str, parameters: dict[str, Any] | None = None) -> dict[str, A otherwise (see the flowx-enrich skill's insights.md). - "route": output_dir(req), at most one of plan(inline object) | plan_path, plus optional source + a source path (forwarded so route can trigger convert if the report is absent, like - the CLI) — one command that groups pipelines into connected components over control lineage - and routes each. With NO plan it emits the components with BOTH conversion options - (deterministic capability + motif/coverage evidence, and the agentic recommended patterns - with any simplification pattern surfaced), plus a ready-to-record default plan — the dry run - to read first. With a plan it validates + records metadata/conversion_plan.json AND rebuilds - the translation report from an immutable baseline, the plan, and stored combines: deterministic - components stay as-is, agentic components with a stored combine get it applied, agentic without - a combine become placeholders (with no agentic decision the report stays byte-identical to the - baseline). The plan is freely editable — change any component's route without restriction. - Route stamps a routing record onto the report (per component_id: members, decision, outcome, - combine_sha256, fingerprint, replacements) that fill_agentic carries forward; every re-route - rebuilds deterministically and idempotently from the baseline and current plan. + the CLI), output_volume_path, output_workspace_path — groups pipelines into connected + components over control lineage and writes the recommendation straight into + metadata/conversion_plan.json: per component BOTH conversion options (deterministic capability + + motif/coverage evidence, and the agentic recommended patterns with any simplification + pattern surfaced) and a `decision`, plus the `suggested_groupings` (whole components the + insights link by an inferred relationship or a shared simplification pattern, each with its + basis and `accepted`) and an optional `conversation` (the questions asked and the user's + answers). With NO plan it keeps the decisions already recorded and leaves every other + component pending (decision null); with a plan (the decisions, or an edited copy of the + recorded plan) it records those. It also writes metadata/routing_review.html, the standard + review page. The plan and the page come back under `bundle` (or are uploaded). While a + decision is pending nothing is applied; once every component is decided route rebuilds the + translation report from the unchanged baseline, the stored gap fills, the plan, and the + agent's outputs in metadata/agentic_conversion.json: deterministic units stay as-is, agentic + units with a stored output get it applied, agentic without one become placeholders (with no + agentic decision the report stays byte-identical to the baseline). An accepted grouping is one + agentic unit. The plan is freely editable and can be re-applied as often as needed. Route + stamps a routing record onto the report (per unit: members, decision, outcome, output_sha256, + fingerprint); every re-route rebuilds deterministically and idempotently. - "fill_agentic": output_dir(req), members(req: list of pipeline names or comma-separated - string), one of pipelines(inline list of pipeline IR dicts) | pipelines_path — cross-pipeline - COMBINE fill: apply agent-authored pipeline(s) to a routed-agentic component (typically - AgenticComponentActivity nodes). `members` must exactly match a routed-agentic component in - the recorded, fingerprint-bound conversion_plan.json and the report's routing record. Same - authored pipelines as already stored returns already_combined: true (idempotent, no writes). - Different pipelines replace the stored combine and rebuild. The merged report is always - validated structurally before writing (all-or-nothing: any violation and nothing is written). - It is the only fill for a routed-agentic group (to keep a pipeline 1:1, author one - same-named pipeline). + string), one of pipelines(inline list of pipeline IR dicts) | pipelines_path — fill a + routed-agentic unit with agent-authored pipeline(s) (typically AgenticComponentActivity nodes), + stored in metadata/agentic_conversion.json. `members` must exactly match a routed-agentic + component or accepted grouping in the recorded, fingerprint-bound conversion_plan.json and the + report's routing record. Same authored pipelines as already stored returns already_applied: + true (idempotent, no writes). Different pipelines replace the stored output (recorded in its + `replaced` history) and rebuild. Authored pipelines must hold no placeholder task and no name + that shares a bundle folder with another pipeline. The merged report is always validated + structurally before writing (all-or-nothing). It is the only fill for a routed-agentic unit + (to keep a pipeline 1:1, author one same-named pipeline); package refuses until every agentic + unit is filled. - "inspect": report_path(req) — return the full translation-option schema (every option with a `show_when` condition) for the agent to walk locally. See "Collecting options" below. - "apply_answers": report_path(req), answers(req, list of "ID=VALUE"), output_dir, lookup_csv. diff --git a/src/flowx/models/conversion_plan.py b/src/flowx/models/conversion_plan.py index da3013b7..9141291f 100644 --- a/src/flowx/models/conversion_plan.py +++ b/src/flowx/models/conversion_plan.py @@ -7,18 +7,21 @@ writes it, and every reader (route, the fill, package) goes through it. Validation of the authored decision against the inventory lives in :mod:`flowx.routing`. -The agent authors **only** :attr:`ComponentPlan.decision` (and an optional -:attr:`ComponentPlan.rationale`). Everything else -- ``component_id``, ``members``, ``recommended``, -and both conversion options -- is recomputed by the library on record so the recorded facts -can never drift from the inventory or be faked. The library also owns :attr:`ConversionPlan.schema_version` -and the hashes that bind the plan to what it was decided on: :attr:`ConversionPlan.inventory_sha256` -(the deterministic inventory), :attr:`ConversionPlan.source_graphs_sha256` (the saved -``source_graphs.json``) and :attr:`ConversionPlan.agentic_insights_sha256` (the saved -``agentic_insights.json``, when enrich ran). - -Phase 1 records one decision per component and applies it to the IR after convert. Each component -also carries :attr:`ComponentPlan.assignments`, reserved for per-node or per-subgraph routing -(deterministic, agentic with a named pattern, or mixed at a boundary); Phase 1 requires it empty. +Route writes its recommendation straight into this file, with every decision pending (``None``), +and the agent edits it with the user. The agent authors only :attr:`ComponentPlan.decision` (and an +optional :attr:`ComponentPlan.rationale`), :attr:`SuggestedGrouping.accepted`, and the optional +:attr:`ConversionPlan.conversation`. Everything else -- ``component_id``, ``members``, +``recommended``, both conversion options, the suggested groupings and their basis -- is recomputed by +the library on every route, so the recorded facts can never drift from the inventory or be faked. The +library also owns :attr:`ConversionPlan.schema_version` and the hashes that bind the plan to what it +was decided on: :attr:`ConversionPlan.inventory_sha256` (the deterministic inventory), +:attr:`ConversionPlan.source_graphs_sha256` (the saved ``source_graphs.json``) and +:attr:`ConversionPlan.agentic_insights_sha256` (the saved ``agentic_insights.json``, when enrich ran). + +Phase 1 records one decision per component and applies it to the IR after convert. An accepted +grouping joins whole components into one agentic unit. Each component also carries +:attr:`ComponentPlan.assignments`, reserved for per-node or per-subgraph routing (deterministic, +agentic with a named pattern, or mixed at a boundary); Phase 1 requires it empty. """ from __future__ import annotations @@ -30,9 +33,9 @@ from typing import Any # The plan schema version stamped onto the recorded artifact. Bump on any backwards-incompatible -# change to the recorded shape. Version 2 adds the source graph and agentic insights hashes and the -# reserved per-node assignments. -SCHEMA_VERSION = "2" +# change to the recorded shape. Version 3 allows a pending (null) decision, and adds the suggested +# groupings, the routing conversation and the agentic_insights_sha256 binding. +SCHEMA_VERSION = "3" # Where the recorded plan lives, beside inventory.json under the output's metadata/ folder. PLAN_FILENAME = "conversion_plan.json" @@ -53,6 +56,11 @@ # or distinguish it). This is factual labelling, never a warning or an alarm. DISCLOSED_RELEASE_STATES: tuple[str, ...] = ("public_preview", "private_preview", "beta") +# The two kinds of link a grouping suggestion can rest on: an inferred relationship from the insights, +# or a simplification pattern the insights recommend for pipelines in different components. +BASIS_INFERRED_RELATIONSHIP = "inferred_relationship" +BASIS_SHARED_PATTERN = "shared_pattern" + @dataclass(slots=True, kw_only=True) class NodeAssignment: @@ -104,7 +112,8 @@ class ComponentPlan: members: The component's pipeline names, sorted (library-computed). recommended: The library's starting suggestion -- ``"deterministic"`` when the component is engine-capable, else ``"agentic"``. - decision: The user's authored per-component choice (may override :attr:`recommended`). + decision: The user's authored per-component choice (may override :attr:`recommended`), or + ``None`` while it is still pending. Route applies a plan only once nothing is pending. options: Both conversion options with their evidence (library-computed), in the recorded JSON shape :mod:`flowx.routing` builds. Optional so the authored input -- which carries only the decision -- can round-trip through this model. @@ -115,7 +124,7 @@ class ComponentPlan: component_id: str members: list[str] = field(default_factory=list) recommended: str | None = None - decision: str + decision: str | None = None options: dict[str, Any] | None = None rationale: str | None = None assignments: list[NodeAssignment] = field(default_factory=list) @@ -136,24 +145,91 @@ def to_dict(self) -> dict[str, Any]: @classmethod def from_dict(cls, raw: dict[str, Any]) -> ComponentPlan: - """Rehydrate from the recorded JSON shape.""" + """Rehydrate from the recorded JSON shape (already checked by :meth:`ConversionPlan.from_dict`).""" return cls( - component_id=str(raw.get("component_id", "")), - members=[str(member) for member in raw.get("members") or []], + component_id=str(raw["component_id"]), + members=[str(member) for member in raw["members"]], recommended=raw.get("recommended"), - decision=str(raw.get("decision", "")), + decision=raw.get("decision"), options=raw.get("options"), rationale=raw.get("rationale"), assignments=[NodeAssignment.from_dict(item) for item in raw.get("assignments") or []], ) +@dataclass(slots=True, kw_only=True) +class SuggestedGrouping: + """Whole components the insights suggest converting together, agentically, as one unit. + + Route computes these from the agentic insights (see :func:`flowx.routing.suggest_groupings`); the + user accepts one by setting :attr:`accepted`. An accepted grouping becomes one routing unit with + the grouping's id: one agent output replaces all of its pipelines. It never splits a component. + + Attributes: + grouping_id: Stable id assigned by the library (``"grouping-"``). + components: The ids of the components it joins (two or more, library-computed). + members: Every pipeline of those components, sorted (library-computed). + basis: Why it is suggested (library-computed): each entry is an inferred relationship from the + insights with its evidence, or a simplification pattern the insights recommend for + pipelines in more than one of the components. + accepted: The user's choice (authored); ``False`` until the user accepts it. + """ + + grouping_id: str + components: list[str] = field(default_factory=list) + members: list[str] = field(default_factory=list) + basis: list[dict[str, Any]] = field(default_factory=list) + accepted: bool = False + + def to_dict(self) -> dict[str, Any]: + """Serialise to the recorded JSON shape.""" + return { + "grouping_id": self.grouping_id, + "components": list(self.components), + "members": list(self.members), + "basis": list(self.basis), + "accepted": self.accepted, + } + + @classmethod + def from_dict(cls, raw: dict[str, Any]) -> SuggestedGrouping: + """Rehydrate from the recorded JSON shape (already checked by :meth:`ConversionPlan.from_dict`).""" + return cls( + grouping_id=str(raw["grouping_id"]), + components=[str(component) for component in raw.get("components") or []], + members=[str(member) for member in raw.get("members") or []], + basis=[item for item in raw.get("basis") or [] if isinstance(item, dict)], + accepted=raw.get("accepted") is True, + ) + + +@dataclass(slots=True, kw_only=True) +class ConversationEntry: + """One open question the agent asked the user while routing, and the user's answer. + + Attributes: + question: What the agent asked (for example what the user wants the conversion to achieve). + answer: What the user said. + """ + + question: str + answer: str + + def to_dict(self) -> dict[str, str]: + """Serialise to the recorded JSON shape.""" + return {"question": self.question, "answer": self.answer} + + @dataclass(slots=True, kw_only=True) class ConversionPlan: """The recorded conversion-plan artifact. Attributes: components: One :class:`ComponentPlan` per connected component. + suggested_groupings: The groupings route suggests from the insights, each with the user's + acceptance. + conversation: The routing conversation with the user, in the order it happened (authored, + optional). Package copies it into ``route_audit.json``. findings: Human-readable notes about unresolved/dangling control edges retained during component computation (never silently severed). schema_version: Library-owned plan schema version. @@ -166,12 +242,18 @@ class ConversionPlan: """ components: list[ComponentPlan] = field(default_factory=list) + suggested_groupings: list[SuggestedGrouping] = field(default_factory=list) + conversation: list[ConversationEntry] = field(default_factory=list) findings: list[str] = field(default_factory=list) schema_version: str = SCHEMA_VERSION inventory_sha256: str | None = None source_graphs_sha256: str | None = None agentic_insights_sha256: str | None = None + def pending_components(self) -> list[str]: + """The ids of the components whose decision is still pending, in plan order.""" + return [component.component_id for component in self.components if component.decision is None] + def to_dict(self) -> dict[str, Any]: """Serialise to the recorded ``conversion_plan.json`` shape.""" return { @@ -180,16 +262,19 @@ def to_dict(self) -> dict[str, Any]: "source_graphs_sha256": self.source_graphs_sha256, "agentic_insights_sha256": self.agentic_insights_sha256, "components": [component.to_dict() for component in self.components], + "suggested_groupings": [grouping.to_dict() for grouping in self.suggested_groupings], + "conversation": [entry.to_dict() for entry in self.conversation], "findings": list(self.findings), } @classmethod def from_dict(cls, raw: Any) -> ConversionPlan: - """Rehydrate a recorded plan. + """Rehydrate a recorded plan, refusing one that is not complete and well formed. Raises: - ValueError: The document is not an object, or was recorded under another schema version - (re-run ``route`` to record it again). + ValueError: The document is not an object, was recorded under another schema version, or + its components, groupings or conversation are missing or malformed (re-run ``route`` + to record it again). """ if not isinstance(raw, dict): raise ValueError(f"{PLAN_FILENAME} must contain a JSON object, got {type(raw).__name__}") @@ -198,13 +283,20 @@ def from_dict(cls, raw: Any) -> ConversionPlan: f"{PLAN_FILENAME} has schema_version {raw.get('schema_version')!r}; expected {SCHEMA_VERSION!r}. " "Re-run route to record the decision again." ) - components = [item for item in raw.get("components") or [] if isinstance(item, dict)] + problems = _recorded_shape_problems(raw) + if problems: + raise ValueError(f"{PLAN_FILENAME} is malformed ({'; '.join(problems)}); re-run route to record it again") return cls( schema_version=SCHEMA_VERSION, inventory_sha256=raw.get("inventory_sha256"), source_graphs_sha256=raw.get("source_graphs_sha256"), agentic_insights_sha256=raw.get("agentic_insights_sha256"), - components=[ComponentPlan.from_dict(item) for item in components], + components=[ComponentPlan.from_dict(item) for item in raw["components"]], + suggested_groupings=[SuggestedGrouping.from_dict(item) for item in raw.get("suggested_groupings") or []], + conversation=[ + ConversationEntry(question=item["question"], answer=item["answer"]) + for item in raw.get("conversation") or [] + ], findings=[str(finding) for finding in raw.get("findings") or []], ) @@ -213,7 +305,7 @@ def load(cls, output_dir: Path) -> ConversionPlan | None: """Load the recorded plan from ``/metadata``, or ``None`` when none was recorded. Raises: - ValueError: The file is not valid JSON or not a version this library reads. + ValueError: The file is not valid JSON, not a version this library reads, or malformed. """ path = Path(output_dir) / "metadata" / PLAN_FILENAME if not path.is_file(): @@ -232,3 +324,35 @@ def write(self, output_dir: Path) -> Path: temporary.write_text(json.dumps(self.to_dict(), indent=2), encoding="utf-8") os.replace(temporary, path) return path + + +def _recorded_shape_problems(raw: dict[str, Any]) -> list[str]: + """What keeps a recorded plan document from loading: a missing or malformed list or entry.""" + problems: list[str] = [] + components = raw.get("components") + if not isinstance(components, list): + problems.append("'components' must be a list") + else: + for index, component in enumerate(components): + if not isinstance(component, dict): + problems.append(f"components[{index}] must be an object") + continue + if not isinstance(component.get("component_id"), str) or not component["component_id"]: + problems.append(f"components[{index}] needs a 'component_id'") + members = component.get("members") + if not isinstance(members, list) or not members or not all(isinstance(item, str) for item in members): + problems.append(f"components[{index}] needs 'members', a list of pipeline names") + if component.get("decision") is not None and component.get("decision") not in DECISIONS: + problems.append(f"components[{index}] has decision {component.get('decision')!r}") + groupings = raw.get("suggested_groupings", []) + if not isinstance(groupings, list) or not all( + isinstance(item, dict) and isinstance(item.get("grouping_id"), str) for item in groupings + ): + problems.append("'suggested_groupings' must be a list of objects with a 'grouping_id'") + conversation = raw.get("conversation", []) + if not isinstance(conversation, list) or not all( + isinstance(item, dict) and isinstance(item.get("question"), str) and isinstance(item.get("answer"), str) + for item in conversation + ): + problems.append("'conversation' must be a list of {question, answer} objects") + return problems diff --git a/src/flowx/reporting/routing_review.py b/src/flowx/reporting/routing_review.py new file mode 100644 index 00000000..90fbb828 --- /dev/null +++ b/src/flowx/reporting/routing_review.py @@ -0,0 +1,302 @@ +"""The routing review page: one standard HTML view of how a factory will be converted. + +Route writes ``metadata/routing_review.html`` from the recorded ``conversion_plan.json``, the +inventory's insights and the report's routing record, so every run presents the same eight sections in +the same order: a summary, what the source does, the components, the suggested groupings, how each +unit will be converted, the routing conversation, the findings, and how to steer. The page is filled +from :data:`routing_review_template.html <_TEMPLATE_PATH>` with the standard library only (no script +or external asset, so it opens offline and in the workspace file viewer), carries no timestamp, and +every value from the plan or insights is HTML-escaped. The same inputs always give the same bytes. + +The page is generated and never read back: the user steers by editing ``conversion_plan.json`` (or +passing a plan) and running route again. +""" + +from __future__ import annotations + +import json +from html import escape +from pathlib import Path +from string import Template +from typing import Any + +from flowx.models.conversion_plan import DECISION_AGENTIC, DECISION_DETERMINISTIC, ConversionPlan + +_TEMPLATE_PATH = Path(__file__).with_name("routing_review_template.html") + +# Where the page is written, beside the plan it presents. +REVIEW_FILENAME = "routing_review.html" + + +def render_routing_review(plan: ConversionPlan, inventory: dict[str, Any] | None, record: dict[str, Any] | None) -> str: + """Render the routing review page for a recorded plan. + + Args: + plan: The recorded conversion plan. + inventory: The discover ``inventory.json`` (its ``insights`` block describes the source), or + ``None`` when it cannot be read. + record: The live report's routing record (each unit's outcome), or ``None`` before route has + applied anything. + + Returns: + The complete HTML document. + """ + from flowx.route_agentic import routing_units + + insights = (inventory or {}).get("insights") + insights = insights if isinstance(insights, dict) else None + units = routing_units(plan.to_dict()) + outcomes = record.get("components") if isinstance(record, dict) else None + outcomes = outcomes if isinstance(outcomes, dict) else {} + template = Template(_TEMPLATE_PATH.read_text(encoding="utf-8")) + return template.substitute( + summary=_summary(plan, inventory, units), + source=_source(insights), + components=_components(plan, insights), + groupings=_groupings(plan), + conversion=_conversion(units, outcomes), + conversation=_conversation(plan), + findings=_list(plan.findings, empty="No findings."), + steer=_steer(), + ) + + +def write_routing_review(output_dir: Path) -> Path | None: + """Write ``metadata/routing_review.html`` for the recorded plan; ``None`` when there is no plan. + + Reads the plan, the inventory and the live report's routing record from ``output_dir``. A report + or inventory that cannot be read is shown as not yet routed or without insights. + """ + from flowx.route_agentic import REPORT_FILENAME, WORK_DIRNAME, routing_record + + plan = ConversionPlan.load(Path(output_dir)) + if plan is None: + return None + metadata_dir = Path(output_dir) / "metadata" + inventory = _read_json(metadata_dir / "inventory.json") + record = routing_record(_read_json(Path(output_dir) / WORK_DIRNAME / REPORT_FILENAME)) + path = metadata_dir / REVIEW_FILENAME + path.write_text(render_routing_review(plan, inventory if isinstance(inventory, dict) else None, record), "utf-8") + return path + + +def _read_json(path: Path) -> Any: + """The parsed JSON at ``path``, or ``None`` when it is missing or unreadable.""" + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + + +def _text(value: Any) -> str: + """Escape any value for HTML; ``None`` shows as a dash.""" + return "—" if value is None else escape(str(value)) + + +def _tag(label: str, css_class: str) -> str: + """A small coloured label.""" + return f'{escape(label)}' + + +def _decision_tag(decision: str | None) -> str: + """The label for a decision, pending included.""" + if decision is None: + return _tag("pending", "pending") + return _tag(decision, "agentic" if decision == DECISION_AGENTIC else "deterministic") + + +def _rows(rows: list[tuple[str, str]]) -> str: + """A two-column table of already-escaped cells.""" + body = "".join(f"{escape(label)}{cell}" for label, cell in rows) + return f"{body}
" + + +def _list(items: list[Any], *, empty: str) -> str: + """An escaped bullet list, or a muted note when there is nothing to list.""" + if not items: + return f'

{escape(empty)}

' + return "
    " + "".join(f"
  • {_text(item)}
    • " for item in items) + "
" + + +def _summary(plan: ConversionPlan, inventory: dict[str, Any] | None, units: dict[str, dict[str, Any]]) -> str: + """Section 1: counts and the hashes the plan is bound to.""" + pipelines = (inventory or {}).get("pipelines") + decisions = [component.decision for component in plan.components] + accepted = [grouping.grouping_id for grouping in plan.suggested_groupings if grouping.accepted] + return _rows( + [ + ("Source", _text((inventory or {}).get("source", "adf"))), + ("Pipelines", _text(len(pipelines) if isinstance(pipelines, list) else None)), + ("Components", _text(len(plan.components))), + ("Suggested groupings", _text(f"{len(plan.suggested_groupings)} ({len(accepted)} accepted)")), + ("Routing units", _text(len(units))), + ( + "Decisions", + _text( + f"{decisions.count(DECISION_DETERMINISTIC)} deterministic, {decisions.count(DECISION_AGENTIC)} " + f"agentic, {decisions.count(None)} pending" + ), + ), + ("Inventory", f"{_text(plan.inventory_sha256)}"), + ("Source graphs", f"{_text(plan.source_graphs_sha256)}"), + ("Agentic insights", f"{_text(plan.agentic_insights_sha256)}"), + ] + ) + + +def _patterns(patterns: Any) -> str: + """A list of recommended patterns with their fit and simplification flag.""" + items = [] + for pattern in patterns if isinstance(patterns, list) else []: + if not isinstance(pattern, dict): + continue + flag = " " + _tag("simplification", "agentic") if pattern.get("simplification_pattern") else "" + where = f' ({_text(pattern["pipeline"])})' if pattern.get("pipeline") else "" + items.append( + f"
  • {_text(pattern.get('pattern'))}{flag}{where}: {_text(pattern.get('fit'))}
    • " + ) + return "
      " + "".join(items) + "
    " if items else 'none' + + +def _source(insights: dict[str, Any] | None) -> str: + """Section 2: the insights overview and the factory-wide recommendation.""" + if insights is None: + return '

    Enrich has not run, so there are no insights about what the source does.

    ' + system = insights.get("system_recommendation") + system = system if isinstance(system, dict) else {} + cascade = system.get("cascade") + return _rows( + [ + ("Overview", _text(insights.get("overview"))), + ("Headline", _text(system.get("headline"))), + ("Recommended architecture", _patterns(system.get("recommended_patterns"))), + ("What it collapses", _list(cascade if isinstance(cascade, list) else [], empty="Nothing stated.")), + ("Deciding question", _text(system.get("decision_driver"))), + ] + ) + + +def _components(plan: ConversionPlan, insights: dict[str, Any] | None) -> str: + """Section 3: one card per component, always with the same fields.""" + intents: dict[str, Any] = {} + for entry in (insights or {}).get("pipeline_insights") or []: + if isinstance(entry, dict) and entry.get("pipeline") is not None: + intents[str(entry["pipeline"])] = entry.get("intent") + cards = [] + for component in plan.components: + options = component.options or {} + deterministic = options.get("deterministic") or {} + agentic = options.get("agentic") or {} + members = ( + "
      " + + "".join( + f"
    • {_text(member)}: {_text(intents.get(member))}
      • " for member in component.members + ) + + "
    " + ) + uncovered = [ + f"{entry.get('pipeline')} / {entry.get('activity')} ({entry.get('type')})" + for entry in deterministic.get("uncovered") or [] + if isinstance(entry, dict) + ] + counts = deterministic.get("activity_counts") or {} + disclosures = [ + entry.get("message") for entry in agentic.get("release_disclosures") or [] if isinstance(entry, dict) + ] + rows = [ + ("Members and intent", members), + ("Recommended", _decision_tag(component.recommended)), + ("Decision", _decision_tag(component.decision)), + ("Deterministic: fully capable", _text("yes" if deterministic.get("capable") else "no")), + ( + "Deterministic: activities", + _text(", ".join(f"{count} {bucket}" for bucket, count in counts.items()) or None), + ), + ("Deterministic: motifs", _list(list(deterministic.get("motifs") or []), empty="none")), + ("Deterministic: not covered", _list(uncovered, empty="none")), + ("Agentic: recommended patterns", _patterns(agentic.get("recommended_patterns"))), + ("Agentic: release disclosures", _list(disclosures, empty="none")), + ("Rationale", _text(component.rationale)), + ] + cards.append(f'

    {_text(component.component_id)}

    {_rows(rows)}
    ') + return "".join(cards) or '

    No components.

    ' + + +def _describe_basis(basis: dict[str, Any]) -> str: + """One escaped line on why a grouping is suggested.""" + if basis.get("kind") == "shared_pattern": + pipelines = ", ".join(str(pipeline) for pipeline in basis.get("pipelines") or []) + pattern = _text(basis.get("pattern")) + return f"Shared simplification pattern {pattern}, recommended for {_text(pipelines)}" + summary = f" — {_text(basis['relationship_summary'])}" if basis.get("relationship_summary") else "" + return ( + f"Inferred relationship {_text(basis.get('from_pipeline'))} → " + f"{_text(basis.get('to_pipeline'))} via {_text(basis.get('edge_identity'))} " + f"({_text(basis.get('confidence'))} confidence; evidence: {_text(basis.get('evidence'))}){summary}" + ) + + +def _groupings(plan: ConversionPlan) -> str: + """Section 4: every suggested grouping, its basis, and whether it is accepted.""" + cards = [] + for grouping in plan.suggested_groupings: + status = _tag("accepted", "agentic") if grouping.accepted else _tag("not accepted", "pending") + basis = "
      " + "".join(f"
    • {_describe_basis(entry)}
      • " for entry in grouping.basis) + "
    " + rows = [ + ("Status", status), + ("Components", _text(", ".join(grouping.components))), + ("Members", _text(", ".join(grouping.members))), + ("Why it is suggested", basis), + ] + cards.append(f'

    {_text(grouping.grouping_id)}

    {_rows(rows)}
    ') + return "".join(cards) or '

    No groupings are suggested.

    ' + + +def _conversion(units: dict[str, dict[str, Any]], outcomes: dict[str, Any]) -> str: + """Section 5: what package will do with each routing unit.""" + from flowx.route_agentic import OUTCOME_AGENTIC_APPLIED + + rows = [] + for unit_id, unit in units.items(): + recorded = outcomes.get(unit_id) + outcome: dict[str, Any] = recorded if isinstance(recorded, dict) else {} + if unit["decision"] is None: + what = _tag("pending", "pending") + " Decision pending; route applies nothing until it is decided." + elif unit["decision"] == DECISION_DETERMINISTIC: + what = _tag("deterministic", "deterministic") + " Converted 1:1 by the deterministic engine." + elif outcome.get("outcome") == OUTCOME_AGENTIC_APPLIED: + what = _tag("agentic", "agentic") + ( + f" The agent's output {_text(str(outcome.get('output_sha256'))[:12])} replaces these " + "pipelines; package ships it." + ) + else: + what = _tag("waiting", "refused") + ( + " Waiting for the agent's output (fill-agentic); package refuses until it is filled." + ) + label = f"{unit_id} ({', '.join(unit.get('components') or [])})" if unit.get("components") else unit_id + rows.append((label, f'{what}
    {_text(", ".join(unit["members"]))}')) + return _rows(rows) if rows else '

    Nothing to convert.

    ' + + +def _conversation(plan: ConversionPlan) -> str: + """Section 6: the questions asked while routing and the user's answers.""" + if not plan.conversation: + return '

    No routing conversation is recorded.

    ' + return _rows([(entry.question, _text(entry.answer)) for entry in plan.conversation]) + + +def _steer() -> str: + """Section 8: the exact edits and commands that change the conversion.""" + steps = [ + 'Set each component\'s decision to "deterministic" or "agentic" ' + "in metadata/conversion_plan.json (or pass the edited plan as plan).", + "To convert a suggested grouping as one unit, set its accepted to true and " + 'decide each of its components "agentic".', + "Optionally record the questions you asked and the answers under conversation.", + "Run route again: python -m flowx.adapter route --output-dir <dir>, or " + 'flowx(command="route", parameters={"output_dir": ...}). Route applies the plan once ' + "nothing is pending; the plan can be changed and re-applied as often as needed.", + "Fill each agentic unit with fill-agentic --members <members> --pipelines-path <file> " + "(MCP: fill_agentic); run it again with new pipelines to change the approach.", + ] + return "
      " + "".join(f"
    1. {step}
      2. " for step in steps) + "
    " diff --git a/src/flowx/reporting/routing_review_template.html b/src/flowx/reporting/routing_review_template.html new file mode 100644 index 00000000..d7f067ba --- /dev/null +++ b/src/flowx/reporting/routing_review_template.html @@ -0,0 +1,83 @@ + + + + + +flowx routing review + + + +
    +

    flowx routing review

    +

    How your source pipelines are grouped and how each group will be converted. Generated by + route from metadata/conversion_plan.json; edit that file, not this page.

    +
    + +
    +

    1. Summary

    + $summary +
    + +
    +

    2. What the source does

    + $source +
    + +
    +

    3. Components

    +

    Pipelines joined by proven control calls (ExecutePipeline) form one component; a component is + always routed as a whole.

    + $components +
    + +
    +

    4. Suggested groupings

    +

    Whole components the insights suggest converting together, agentically, as one unit.

    + $groupings +
    + +
    +

    5. How it will be converted

    + $conversion +
    + +
    +

    6. Routing conversation

    + $conversation +
    + +
    +

    7. Findings

    + $findings +
    + +
    +

    8. How to steer

    + $steer +
    + + diff --git a/src/flowx/route_agentic.py b/src/flowx/route_agentic.py index 144bc6cf..7bd9bb2a 100644 --- a/src/flowx/route_agentic.py +++ b/src/flowx/route_agentic.py @@ -8,34 +8,29 @@ pattern similar to :mod:`flowx.agentic`, always keeping the work in-engine so the package phase, structural validation, and provenance all still apply: -* A report without a routing record is a fresh convert. When route first routes a component agentic - from one, it saves an exact copy of ``.work/translation_report.json`` and ``.work/gaps.json`` to - ``.work/route_baseline/`` and records their hashes; later routes rebuild from that copy (only a - merge of convert's own gaps changes it). Combines are stored in ``metadata/agentic_combines.json`` - with a canonical hash of their pipelines and are idempotent: the same combine is a no-op, a - different combine replaces the stored entry. Switching every component back to deterministic writes - the baseline back byte for byte. Route and combine never write modify's configured copy. -* :func:`rebuild` is a pure function that takes the immutable baseline, the routing plan, and stored - combines, and produces edited report + gaps + routing record. It implements both re-routing and - re-applying combines deterministically: every rebuild call with the same inputs produces identical - bytes. The routing record tracks each component's outcome (deterministic, agentic-applied, or - agentic-not-viable) and fingerprint changes. -* The agent authors the fill. A routed-agentic component is filled **only** by the cross-pipeline - COMBINE (N pipelines -> M, e.g. one Lakeflow Connect pipeline, or one same-named pipeline to keep - it 1:1): stored in ``metadata/agentic_combines.json`` and applied via rebuild. The name-matched - :func:`flowx.ir_serde.merge_agentic_results` stays for convert's own gaps (after routing, a merge - written in place into the live report also updates the stored baseline so the next rebuild keeps - it) and refuses routed-agentic pipelines. +* A report without a routing record is a fresh convert. When route first routes from one, it saves an + exact copy of ``.work/translation_report.json`` and ``.work/gaps.json`` to ``.work/route_baseline/`` + and records their hashes; that copy is never written again, and later routes rebuild from it. +* The agent's conversion output lives in ``metadata/agentic_conversion.json``: one entry per agentic + routing unit (the authored pipelines, their canonical hash and the history of earlier outputs it + replaced), plus the fills of convert's own gaps merged after routing. Filling a unit again with the + same pipelines is a no-op; different pipelines replace the stored entry and add to its history. + Switching every unit back to deterministic writes the baseline back byte for byte (unless fills of + convert's own gaps are stored). Route and the fill never write modify's configured copy. +* :func:`rebuild` is a pure function: baseline, then the stored gap fills, then the plan's agentic + units and their stored outputs, giving the edited report + gaps + routing record. The same inputs + always give identical bytes. The routing record holds each unit's outcome (deterministic, + agentic-applied, or agentic-not-viable until filled) and a fingerprint. +* A routing unit is a component, or an accepted grouping of whole components the plan joins into one + agentic unit. The agent authors the fill: a routed-agentic unit is filled **only** by + ``fill-agentic`` (N pipelines -> M, e.g. one Lakeflow Connect pipeline, or one same-named pipeline + to keep it 1:1). The name-matched :func:`flowx.ir_serde.merge_agentic_results` stays for convert's + own gaps and refuses routed-agentic pipelines. * :func:`validate_report_structurally` packages the merged report through the same ``prepare -> write_bundle`` path the package phase uses and runs the existing :func:`flowx.validate.bundle_invariants.check_bundle_dir` over the output, so a fill can never introduce duplicate keys, dangling dependencies, a cycle, or a dangling pipeline/run_job reference. -Route stamps one routing record onto the report (:data:`ROUTING_RECORD_KEY`), keyed by the plan's -component ids, holding each component's members, decision, outcome, and a fingerprint. The routing -record also tracks the baseline hashes and combine hashes. Every rebuild carries it forward with -updated outcomes and fingerprints. - There is no LLM here: the tool edits the report and validates the authored fill; the fill itself is supplied by the agent/harness, following the same ask -> author -> continue pattern as the existing agentic flow. @@ -50,46 +45,49 @@ import sys import tempfile from collections.abc import Callable, Iterable +from dataclasses import dataclass, field from pathlib import Path from typing import Any from flowx.discovery_serde import canonical_sha256 -from flowx.models.conversion_plan import DECISION_AGENTIC, ConversionPlan +from flowx.models.conversion_plan import DECISION_AGENTIC, DECISION_DETERMINISTIC, ConversionPlan # The report + gaps live under the shared output dir's transient .work/ folder, beside the pipeline IR. WORK_DIRNAME = ".work" REPORT_FILENAME = "translation_report.json" GAPS_FILENAME = "gaps.json" -# The immutable baseline copies, saved on first route with agentic components. +# The immutable baseline copies, saved on the first route from a fresh convert. BASELINE_DIRNAME = "route_baseline" BASELINE_REPORT_FILENAME = "translation_report.json" BASELINE_GAPS_FILENAME = "gaps.json" -# The recorded plan + inventory the combine fill binds against live under metadata/. +# The recorded plan + inventory the fill binds against live under metadata/, beside the agent's output. METADATA_DIRNAME = "metadata" INVENTORY_FILENAME = "inventory.json" -COMBINES_FILENAME = "agentic_combines.json" +AGENTIC_OUTPUT_FILENAME = "agentic_conversion.json" # Top-level report key holding the routing record. It lives in the report, so a fresh convert (which # rewrites the report) clears it; the package phase and ir_serde read only the pipelines beside it. ROUTING_RECORD_KEY = "_routing_record" -# What became of each component in the record: a deterministic component is left as convert wrote it; -# an agentic one stays "not viable" until combine fills it. +# What became of each routing unit in the record: a deterministic unit is left as convert wrote it; +# an agentic one stays "not viable" until fill-agentic fills it, and package refuses until then. OUTCOME_DETERMINISTIC = "deterministic" OUTCOME_AGENTIC_APPLIED = "agentic-applied" OUTCOME_AGENTIC_NOT_VIABLE = "agentic-not-viable" ROUTED_AGENTIC_MERGE_REFUSED = ( - "this pipeline is routed agentic; fill it with fill-agentic combine " - "(to change the approach, change the plan, re-run convert and route, then combine again)" + "this pipeline is routed agentic; fill its component with fill-agentic " + "(run fill-agentic again with new pipelines to change the approach)" ) # Guidance stamped onto every placeholder the alteration produces. _PLACEHOLDER_COMMENT = ( - "Routed agentic by the conversion plan; replace the whole group with agent-authored pipeline(s) " - "using fill-agentic combine." + "Routed agentic by the conversion plan; replace the whole unit with agent-authored pipeline(s) using fill-agentic." ) +# Where a task can hold nested tasks in the report IR (IfCondition / ForEach / Switch containers). +_NESTED_TASK_KEYS = ("inner_activities", "if_true_activities", "if_false_activities", "default_activities") + # --------------------------------------------------------------------------- # # Reading the recorded / authored plan. @@ -109,23 +107,46 @@ def agentic_pipeline_names(plan: dict[str, Any]) -> set[str]: return names +def routing_units(plan: dict[str, Any]) -> dict[str, dict[str, Any]]: + """The plan's routing units, keyed by unit id, in plan order. + + A unit is one component (its sorted members and decision), or an accepted suggested grouping, + which joins its components into one agentic unit keyed by the grouping id that also lists the + ``components`` it joins. The record, the fill, package and the audit all work per unit. + """ + accepted = [ + grouping + for grouping in plan.get("suggested_groupings") or [] + if isinstance(grouping, dict) and grouping.get("accepted") is True + ] + grouping_of = { + str(component_id): grouping for grouping in accepted for component_id in grouping.get("components") or [] + } + units: dict[str, dict[str, Any]] = {} + for component in plan.get("components", []): + if not isinstance(component, dict): + continue + component_id = str(component.get("component_id")) + grouping = grouping_of.get(component_id) + if grouping is None: + units[component_id] = { + "members": sorted(str(member) for member in component.get("members", [])), + "decision": component.get("decision"), + } + elif str(grouping.get("grouping_id")) not in units: + units[str(grouping.get("grouping_id"))] = { + "members": sorted(str(member) for member in grouping.get("members") or []), + "decision": DECISION_AGENTIC, + "components": [str(component_id) for component_id in grouping.get("components") or []], + } + return units + + # --------------------------------------------------------------------------- # # The routing record stamped onto the report. # --------------------------------------------------------------------------- # -def _planned_components(plan: dict[str, Any]) -> dict[str, dict[str, Any]]: - """Each plan component's sorted members and decision, keyed by component id.""" - return { - str(component.get("component_id")): { - "members": sorted(str(member) for member in component.get("members", [])), - "decision": component.get("decision"), - } - for component in plan.get("components", []) - if isinstance(component, dict) - } - - def routing_record(report: Any) -> dict[str, Any] | None: """The routing record a report carries, or ``None`` when route has not edited it.""" record = report.get(ROUTING_RECORD_KEY) if isinstance(report, dict) else None @@ -133,40 +154,40 @@ def routing_record(report: Any) -> dict[str, Any] | None: def routing_record_mismatches(record: dict[str, Any], plan: dict[str, Any]) -> list[str]: - """Where a report's routing record disagrees with a plan's components, members or decisions. + """Where a report's routing record disagrees with a plan's units, members or decisions. - Returns one message per component that differs; an empty list means the record was routed under - these same decisions. + Returns one message per unit that differs; an empty list means the record was routed under these + same decisions. """ recorded = record.get("components") if not isinstance(recorded, dict): recorded = {} - planned = _planned_components(plan) + planned = routing_units(plan) mismatches: list[str] = [] - for component_id in sorted(set(recorded) | set(planned)): - entry = recorded.get(component_id) - decided = planned.get(component_id) + for unit_id in sorted(set(recorded) | set(planned)): + entry = recorded.get(unit_id) + decided = planned.get(unit_id) if not isinstance(entry, dict): - mismatches.append(f"component {component_id!r} is in the plan but not in the routing record") + mismatches.append(f"component {unit_id!r} is in the plan but not in the routing record") elif decided is None: - mismatches.append(f"component {component_id!r} is in the routing record but not in the plan") + mismatches.append(f"component {unit_id!r} is in the routing record but not in the plan") elif sorted(entry.get("members") or []) != decided["members"]: - mismatches.append(f"component {component_id!r} was routed with different members than the plan lists") + mismatches.append(f"component {unit_id!r} was routed with different members than the plan lists") elif entry.get("decision") != decided["decision"]: mismatches.append( - f"component {component_id!r} was routed {entry.get('decision')!r} but the plan decides " + f"component {unit_id!r} was routed {entry.get('decision')!r} but the plan decides " f"{decided['decision']!r}" ) elif decided["decision"] != DECISION_AGENTIC and entry.get("outcome") != OUTCOME_DETERMINISTIC: - mismatches.append(f"a fill was applied to component {component_id!r}, which the plan routes deterministic") + mismatches.append(f"a fill was applied to component {unit_id!r}, which the plan routes deterministic") return mismatches def routed_agentic_pipelines(report: Any) -> set[str]: - """The pipelines only combine may fill: routed-agentic members and the pipelines combine authored. + """The pipelines only fill-agentic may fill: routed-agentic members and the pipelines it authored. - Every pipeline convert wrote is a member of some recorded component, so a pipeline in the report - that belongs to none was authored by a combine. Empty when the report has no routing record. + Every pipeline convert wrote is a member of some recorded unit, so a pipeline in the report that + belongs to none was authored by fill-agentic. Empty when the report has no routing record. """ record = routing_record(report) components = record.get("components") if record is not None else None @@ -185,7 +206,7 @@ def routed_agentic_pipelines(report: Any) -> set[str]: # --------------------------------------------------------------------------- # -# Baseline management and combines store. +# The deterministic baseline and the agent's conversion output. # --------------------------------------------------------------------------- # @@ -222,14 +243,30 @@ def _save_baseline_bytes(output_dir: Path, report_bytes: bytes, gaps_bytes: byte """Save an immutable copy of the baseline files as exact bytes (byte-for-byte).""" baseline_dir = Path(output_dir) / WORK_DIRNAME / BASELINE_DIRNAME baseline_dir.mkdir(parents=True, exist_ok=True) - baseline_report_path = baseline_dir / BASELINE_REPORT_FILENAME - baseline_gaps_path = baseline_dir / BASELINE_GAPS_FILENAME - baseline_report_path.write_bytes(report_bytes) - baseline_gaps_path.write_bytes(gaps_bytes) + (baseline_dir / BASELINE_REPORT_FILENAME).write_bytes(report_bytes) + (baseline_dir / BASELINE_GAPS_FILENAME).write_bytes(gaps_bytes) + + +@dataclass(slots=True, kw_only=True) +class AgenticOutput: + """The agent's conversion output, as read from ``metadata/agentic_conversion.json``. + + Attributes: + document: The file's content as stored: ``{"components": {: entry}, "gap_fills": [...]}``. + outputs: The unit entries that check out (sorted members, a hash matching their pipelines), + keyed by unit id. Only these are ever applied. + gap_fills: The fills of convert's own gaps that check out, in stored order. + edited: A label for every entry that no longer checks out because it was edited by hand. + """ + + document: dict[str, Any] = field(default_factory=dict) + outputs: dict[str, dict[str, Any]] = field(default_factory=dict) + gap_fills: list[dict[str, Any]] = field(default_factory=list) + edited: list[str] = field(default_factory=list) -def _combine_checks_out(entry: Any) -> bool: - """Whether a stored combine is as combine wrote it: sorted members, and a hash matching its pipelines.""" +def _output_checks_out(entry: Any) -> bool: + """Whether a stored unit output is as fill-agentic wrote it: sorted members, a hash matching its pipelines.""" if not isinstance(entry, dict): return False members = entry.get("members") @@ -240,34 +277,90 @@ def _combine_checks_out(entry: Any) -> bool: and members == sorted(set(members)) and isinstance(pipelines, list) and bool(pipelines) - and entry.get("combine_sha256") == canonical_sha256(pipelines) + and entry.get("output_sha256") == canonical_sha256(pipelines) + and isinstance(entry.get("replaced", []), list) + ) + + +def _gap_fill_checks_out(entry: Any) -> bool: + """Whether a stored gap fill is as the merge wrote it: a task whose hash matches, bound to a baseline.""" + return ( + isinstance(entry, dict) + and isinstance(entry.get("pipeline"), str) + and isinstance(entry.get("activity_name"), str) + and bool(entry["activity_name"]) + and isinstance(entry.get("task"), dict) + and entry.get("task_sha256") == canonical_sha256(entry["task"]) + and isinstance(entry.get("baseline_report_sha256"), str) ) -def load_combines(output_dir: Path) -> tuple[dict[str, Any], dict[str, dict[str, Any]]]: - """Read ``metadata/agentic_combines.json``: every entry as stored, and the entries that check out. +def load_agentic_output(output_dir: Path) -> AgenticOutput: + """Read ``metadata/agentic_conversion.json``; empty when the agent has written nothing yet. - Route, combine and package all read the store here. A rebuild applies only the entries that check - out, so an edited entry is never applied: package refuses while one exists, and re-running combine - for that component replaces it. Both are empty when there is no store yet. + Route, fill-agentic, the merge and package all read the store here. A rebuild applies only the + entries that check out, so an edited entry is never applied: package refuses while one exists, + and filling (or merging) that entry again replaces it. Raises: - ValueError: The store is not valid JSON or not a JSON object. + ValueError: The file is not valid JSON or not shaped as ``{"components": {...}, "gap_fills": [...]}``. """ - combines_path = Path(output_dir) / METADATA_DIRNAME / COMBINES_FILENAME - if not combines_path.exists(): - return {}, {} - stored = json.loads(combines_path.read_text(encoding="utf-8")) - if not isinstance(stored, dict): - raise ValueError(f"{COMBINES_FILENAME} must contain a JSON object") - return stored, {component_id: entry for component_id, entry in stored.items() if _combine_checks_out(entry)} + path = Path(output_dir) / METADATA_DIRNAME / AGENTIC_OUTPUT_FILENAME + if not path.exists(): + return AgenticOutput() + try: + document = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as error: + raise ValueError(f"{AGENTIC_OUTPUT_FILENAME} is not valid JSON: {error}") from error + components = document.get("components", {}) if isinstance(document, dict) else None + gap_fills = document.get("gap_fills", []) if isinstance(document, dict) else None + if not isinstance(components, dict) or not isinstance(gap_fills, list): + raise ValueError(f"{AGENTIC_OUTPUT_FILENAME} must hold a 'components' object and a 'gap_fills' list") + outputs = {unit_id: entry for unit_id, entry in components.items() if _output_checks_out(entry)} + edited = sorted(set(components) - set(outputs)) + checked_fills: list[dict[str, Any]] = [] + for index, fill in enumerate(gap_fills): + if _gap_fill_checks_out(fill): + checked_fills.append(fill) + else: + label = fill.get("activity_name") if isinstance(fill, dict) else None + edited.append(f"gap fill {label!r}" if label else f"gap_fills[{index}]") + return AgenticOutput( + document={"components": components, "gap_fills": gap_fills}, + outputs=outputs, + gap_fills=checked_fills, + edited=edited, + ) + + +def save_agentic_output(output_dir: Path, document: dict[str, Any]) -> None: + """Write ``metadata/agentic_conversion.json`` atomically.""" + _write_json_atomic(Path(output_dir) / METADATA_DIRNAME / AGENTIC_OUTPUT_FILENAME, document) -def _save_combines(output_dir: Path, combines: dict[str, dict[str, Any]]) -> None: - """Save the agentic combines store.""" - combines_path = Path(output_dir) / METADATA_DIRNAME / COMBINES_FILENAME - combines_path.parent.mkdir(parents=True, exist_ok=True) - _write_json_atomic(combines_path, combines) +def record_gap_fills(output_dir: Path, fills: list[dict[str, Any]], baseline_report_sha256: str) -> None: + """Store fills of convert's own gaps, merged after routing, so every rebuild applies them. + + Each fill is ``{"pipeline", "activity_name", "task"}``; it is stored with its task's hash and the + baseline it was merged against, replacing an earlier fill of the same pipeline and activity. The + baseline itself is never changed. + """ + stored = load_agentic_output(output_dir).document or {"components": {}, "gap_fills": []} + by_key = { + (fill.get("pipeline"), fill.get("activity_name")): fill + for fill in stored.get("gap_fills", []) + if isinstance(fill, dict) + } + for fill in fills: + task = copy.deepcopy(fill["task"]) + by_key[(fill["pipeline"], fill["activity_name"])] = { + "pipeline": fill["pipeline"], + "activity_name": fill["activity_name"], + "task": task, + "task_sha256": canonical_sha256(task), + "baseline_report_sha256": baseline_report_sha256, + } + save_agentic_output(output_dir, {"components": stored.get("components", {}), "gap_fills": list(by_key.values())}) # --------------------------------------------------------------------------- # @@ -283,16 +376,20 @@ def _stderr(line: str) -> None: def prompt_for_decisions( recommendation: dict[str, Any], *, + authored: dict[str, Any] | None = None, input_fn: Callable[[str], str] | None = None, output_fn: Callable[[str], None] | None = None, ) -> dict[str, Any]: - """Ask the user, per connected component, whether to convert deterministically or agentically. + """Ask the user for every pending decision: suggested groupings first, then each component. - Shows each component's members, the library's recommendation, and -- when the agentic option - carries a re-architecture (e.g. a multi-pipeline -> Lakeflow Connect simplification) -- flags it - prominently. An empty answer accepts the recommendation; ``d``/``a`` (or the full words) choose - a route. Returns an authored plan dict (``{"components": [{component_id, members, decision}]}``) - ready to hand to :func:`flowx.routing.record_plan`. + ``authored`` is the plan so far (see :func:`flowx.routing.carried_forward_plan`); a component it + already decides is not asked again, and without it every component is pending. A grouping whose + components are all pending is offered first, with its basis; accepting it decides those components + agentic. Each remaining pending component shows its members, the library's recommendation and -- + when the agentic option carries a re-architecture (e.g. a multi-pipeline -> Lakeflow Connect + simplification) -- a prominent flag. An empty answer accepts the recommendation; ``d``/``a`` (or + the full words) choose a route. Returns the authored plan, ready for + :func:`flowx.routing.record_plan`. ``input_fn`` and ``output_fn`` are injectable so the prompt is exercised without a real TTY; both are resolved at call time (defaulting to the builtin ``input`` and a stderr sink) so a test that @@ -300,27 +397,75 @@ def prompt_for_decisions( """ ask = input_fn if input_fn is not None else input say = output_fn if output_fn is not None else _stderr + decided = { + str(component.get("component_id")): component + for component in (authored or {}).get("components", []) + if isinstance(component, dict) and component.get("decision") is not None + } + accepted = { + str(grouping.get("grouping_id")) + for grouping in (authored or {}).get("suggested_groupings", []) + if isinstance(grouping, dict) and grouping.get("accepted") is True + } + decisions: dict[str, str] = {} + for grouping in recommendation.get("suggested_groupings", []): + grouping_id = str(grouping.get("grouping_id")) + components = [str(component_id) for component_id in grouping.get("components", [])] + if grouping_id in accepted or any(component_id in decided for component_id in components): + continue + say(f"\nSuggested grouping {grouping_id}: {', '.join(components)} ({', '.join(grouping.get('members', []))})") + for basis in grouping.get("basis", []): + say(f" because: {_describe_basis(basis)}") + answer = ask(" convert these together agentically as one unit? [y/N]: ").strip().lower() + if answer in ("y", "yes"): + accepted.add(grouping_id) + decisions.update({component_id: DECISION_AGENTIC for component_id in components}) + components_out: list[dict[str, Any]] = [] for component in recommendation.get("components", []): component_id = str(component.get("component_id")) members = list(component.get("members", [])) - recommended = str(component.get("recommended", "deterministic")) - options = component.get("options") or {} - has_simplification = bool((options.get("agentic") or {}).get("has_simplification")) - - say(f"\nComponent {component_id}: {', '.join(members) or '(none)'}") - say(f" recommended: {recommended}") - if has_simplification: - say(" agentic option includes a simplification re-architecture (e.g. Lakeflow Connect)") - answer = ask(f" route [d]eterministic / [a]gentic (default={recommended}): ").strip().lower() - if answer in ("a", "agentic"): - decision = DECISION_AGENTIC - elif answer in ("d", "deterministic"): - decision = "deterministic" - else: - decision = recommended - components_out.append({"component_id": component_id, "members": members, "decision": decision}) - return {"components": components_out} + if component_id in decided: + components_out.append(dict(decided[component_id])) + continue + if component_id not in decisions: + decisions[component_id] = _ask_component(component, ask, say) + components_out.append({"component_id": component_id, "members": members, "decision": decisions[component_id]}) + result: dict[str, Any] = {"components": components_out} + if recommendation.get("suggested_groupings"): + result["suggested_groupings"] = [ + {"grouping_id": grouping["grouping_id"], "accepted": grouping["grouping_id"] in accepted} + for grouping in recommendation["suggested_groupings"] + ] + if authored and authored.get("conversation"): + result["conversation"] = authored["conversation"] + return result + + +def _ask_component(component: dict[str, Any], ask: Callable[[str], str], say: Callable[[str], None]) -> str: + """Ask for one component's route; an empty or unknown answer takes the recommendation.""" + recommended = str(component.get("recommended", DECISION_DETERMINISTIC)) + options = component.get("options") or {} + say(f"\nComponent {component.get('component_id')}: {', '.join(component.get('members', [])) or '(none)'}") + say(f" recommended: {recommended}") + if (options.get("agentic") or {}).get("has_simplification"): + say(" agentic option includes a simplification re-architecture (e.g. Lakeflow Connect)") + answer = ask(f" route [d]eterministic / [a]gentic (default={recommended}): ").strip().lower() + if answer in ("a", "agentic"): + return DECISION_AGENTIC + if answer in ("d", "deterministic"): + return DECISION_DETERMINISTIC + return recommended + + +def _describe_basis(basis: dict[str, Any]) -> str: + """One line on why a grouping is suggested.""" + if basis.get("kind") == "shared_pattern": + return f"{basis.get('pattern')!r} is recommended for {', '.join(basis.get('pipelines') or [])}" + return ( + f"{basis.get('from_pipeline')} -> {basis.get('to_pipeline')} via {basis.get('edge_identity')!r} " + f"({basis.get('confidence')} confidence: {basis.get('evidence')})" + ) # --------------------------------------------------------------------------- # @@ -332,95 +477,83 @@ def rebuild( baseline_report: dict[str, Any], baseline_gaps: list[dict[str, Any]], plan: dict[str, Any], - combines: dict[str, dict[str, Any]] | None, - previous_record: dict[str, Any] | None, + outputs: dict[str, dict[str, Any]] | None, + gap_fills: list[dict[str, Any]] | None, baseline_report_bytes: bytes, baseline_gaps_bytes: bytes, ) -> tuple[dict[str, Any], list[dict[str, Any]], dict[str, Any]]: - """Pure rebuild: derive the routed report, gaps, and record from baseline and plan. + """Pure rebuild: derive the routed report, gaps, and record from the baseline, fills and plan. - Returns ``(report, gaps, record)``, all deterministically derived from inputs. For each component: - * **deterministic:** the baseline pipelines stay; outcome is ``deterministic``. - * **agentic with a stored combine whose members match:** replace members with combine's pipelines, - reuse the combine_sha256. Outcome is ``agentic-applied``. - * **agentic without a matching combine:** placeholders and tagged gaps as ``alter_report`` does. - Outcome is ``agentic-not-viable``. + Returns ``(report, gaps, record)``, all deterministically derived from the inputs. It starts from + a copy of the baseline, applies the stored fills of convert's own gaps that were merged against this + baseline, then turns each agentic unit's tasks into placeholders and gaps, and finally, per unit: - Only a combine that is applied puts its ``combine_sha256`` in the record and the fingerprint; every - other component records ``None``, whatever the store holds for it. + * **deterministic:** the baseline pipelines stay; outcome is ``deterministic``. + * **agentic with a stored output whose members match:** the output's pipelines replace the members + and the members' routed gaps are dropped. Outcome is ``agentic-applied``. + * **agentic without a matching output:** placeholders and tagged gaps as ``alter_report`` does. + Outcome is ``agentic-not-viable``, which package refuses. - The record holds per-component members, decision, outcome, combine_sha256, and fingerprint. When - fingerprint differs from previous_record's, a replacement entry is appended to track the change. - An all-deterministic plan returns no record (byte-identical to baseline). + Each unit's record entry holds its members, decision, outcome, ``output_sha256`` (only while its + stored output is applied, otherwise ``None``) and ``fingerprint``. A plan with no agentic unit and + no applicable gap fill returns the baseline itself and no record (byte-identical to convert). Args: baseline_report: Parsed baseline report JSON. baseline_gaps: Parsed baseline gaps JSON. plan: Routing plan dict. - combines: The stored combines that check out (see :func:`load_combines`). - previous_record: Prior routing record for change tracking. + outputs: The stored unit outputs that check out (see :func:`load_agentic_output`). + gap_fills: The stored gap fills that check out; those merged against another baseline are skipped. baseline_report_bytes: Raw bytes of the baseline report file, hashed into the record. baseline_gaps_bytes: Raw bytes of the baseline gaps file, hashed into the record. """ - combines_store = combines or {} - planned = _planned_components(plan) + from flowx.ir_serde import replace_task_by_name + + baseline_report_sha256 = hashlib.sha256(baseline_report_bytes).hexdigest() + applicable_fills = [ + fill for fill in gap_fills or [] if fill.get("baseline_report_sha256") == baseline_report_sha256 + ] agentic_names = agentic_pipeline_names(plan) - if not agentic_names and previous_record is None: + if not agentic_names and not applicable_fills: return baseline_report, baseline_gaps, {} + report = copy.deepcopy(baseline_report) - gaps = copy.deepcopy(baseline_gaps) - new_report, new_gaps = alter_report(report, gaps, agentic_names) + pipelines_by_name = {pipeline.get("name"): pipeline for pipeline in _report_pipelines(report)} + for fill in applicable_fills: + pipeline = pipelines_by_name.get(fill["pipeline"]) + if pipeline is not None: + replace_task_by_name(pipeline.get("tasks", []), fill["activity_name"], copy.deepcopy(fill["task"])) + new_report, new_gaps = alter_report(report, copy.deepcopy(baseline_gaps), agentic_names) if not isinstance(new_report.get("pipelines"), list): new_report = {"pipelines": [new_report]} - components: dict[str, dict[str, Any]] = {} - for component_id, planned_entry in planned.items(): - decision = planned_entry["decision"] - members_set = set(planned_entry["members"]) - stored_combine = combines_store.get(component_id) - combine_data = ( - stored_combine - if decision == DECISION_AGENTIC and stored_combine and set(stored_combine.get("members", [])) == members_set + + units: dict[str, dict[str, Any]] = {} + for unit_id, unit in routing_units(plan).items(): + members = set(unit["members"]) + stored = (outputs or {}).get(unit_id) + output = ( + stored + if unit["decision"] == DECISION_AGENTIC and stored and set(stored.get("members", [])) == members else None ) - combine_sha256 = combine_data.get("combine_sha256") if combine_data else None - fingerprint_input = { - "members": planned_entry["members"], - "decision": decision, - "combine_sha256": combine_sha256, - } - fingerprint = canonical_sha256(fingerprint_input) - if combine_data: + output_sha256 = output.get("output_sha256") if output else None + if output: outcome = OUTCOME_AGENTIC_APPLIED - new_report = combine_group_fill(new_report, members_set, combine_data.get("pipelines", [])) - elif decision == DECISION_AGENTIC: + new_report = replace_unit_pipelines(new_report, members, output.get("pipelines", [])) + new_gaps = [gap for gap in new_gaps if not (isinstance(gap, dict) and gap.get("pipeline") in members)] + elif unit["decision"] == DECISION_AGENTIC: outcome = OUTCOME_AGENTIC_NOT_VIABLE else: outcome = OUTCOME_DETERMINISTIC - entry: dict[str, Any] = { - **planned_entry, - "outcome": outcome, - "combine_sha256": combine_sha256, - "fingerprint": fingerprint, - } - if previous_record and component_id in previous_record.get("components", {}): - old_fingerprint = previous_record["components"][component_id].get("fingerprint") - if old_fingerprint and old_fingerprint != fingerprint: - old_record = previous_record["components"][component_id] - entry["replacements"] = (old_record.get("replacements") or []) + [ - {"from": old_fingerprint, "to": fingerprint} - ] - else: - entry["replacements"] = previous_record["components"][component_id].get("replacements") or [] - else: - entry["replacements"] = [] - components[component_id] = entry - if all(entry["decision"] != DECISION_AGENTIC for entry in components.values()): - return baseline_report, baseline_gaps, {} + fingerprint = canonical_sha256( + {"members": unit["members"], "decision": unit["decision"], "output_sha256": output_sha256} + ) + units[unit_id] = {**unit, "outcome": outcome, "output_sha256": output_sha256, "fingerprint": fingerprint} record = { "conversion_plan_sha256": canonical_sha256(plan), - "baseline_report_sha256": hashlib.sha256(baseline_report_bytes).hexdigest(), + "baseline_report_sha256": baseline_report_sha256, "baseline_gaps_sha256": hashlib.sha256(baseline_gaps_bytes).hexdigest(), - "components": components, + "components": units, } return new_report, new_gaps, record @@ -551,22 +684,51 @@ def _write_json_atomic(path: Path, document: Any) -> None: _write_atomic(path, json.dumps(document, indent=2, default=str).encode("utf-8")) -def apply_plan_to_report(output_dir: Path, plan: dict[str, Any]) -> dict[str, Any]: +def baseline_pipeline_mismatch(baseline_report: dict[str, Any], inventory: dict[str, Any]) -> str | None: + """Say how a baseline report's pipelines differ from the inventory's, or ``None`` when they match. + + The baseline is the convert output a rebuild starts from; when discover ran again after convert + its pipelines no longer match, and routing it would ship removed pipelines or miss new ones. + """ + inventory_names = { + str(pipeline.get("name")) + for pipeline in inventory.get("pipelines", []) + if isinstance(pipeline, dict) and pipeline.get("name") is not None + } + report_names = {str(pipeline.get("name")) for pipeline in _report_pipelines(baseline_report)} + if report_names == inventory_names: + return None + missing, extra = sorted(inventory_names - report_names), sorted(report_names - inventory_names) + return ( + f"the translation report was converted from a different discover (missing {missing}, extra {extra}); " + "re-run convert, then route" + ) + + +def apply_plan_to_report( + output_dir: Path, plan: dict[str, Any], *, inventory: dict[str, Any] | None = None +) -> dict[str, Any]: """Apply a routing decision to the report on disk by rebuilding it from the deterministic baseline. A report without a routing record is a fresh convert, so it becomes the baseline: the first time - the plan routes a component agentic, route saves it and its gaps to ``.work/route_baseline/``. A - routed report is rebuilt from that saved copy. The rebuild applies the plan and the stored combines - from ``metadata/agentic_combines.json``, and the report + gaps are written atomically. When no - component is routed agentic the baseline bytes are written back unchanged, so switching back to - deterministic restores convert's output exactly and a never-routed report is left untouched. Route - never writes modify's configured copy; package asks for ``modify`` again when it is out of date. + the rebuild changes anything, route saves it and its gaps to ``.work/route_baseline/``; gap fills + merged against an earlier baseline are then dropped (merge them again). A routed report is rebuilt + from the saved copy. The rebuild applies the stored gap fills, the plan and the stored unit outputs + from ``metadata/agentic_conversion.json``, and the report + gaps are written atomically. When the + rebuild changes nothing the baseline bytes are written back, so switching back to deterministic + restores convert's output exactly and a never-routed report is left untouched. Route never writes + modify's configured copy; package asks for ``modify`` again when it is out of date. + + With ``inventory`` given and an agentic unit in the plan, the baseline's pipelines must be the + inventory's (see :func:`baseline_pipeline_mismatch`). - Returns a summary dict with the altered pipeline names and the resulting gap count. + Returns a summary dict with the altered pipeline names, the resulting gap count, each unit's + outcome and how many stale gap fills were dropped. Raises: FileNotFoundError: when the translation report is missing (run convert first). - ValueError: when a routed report's saved baseline or the combine store cannot be read. + ValueError: when a routed report's saved baseline or the agent's output cannot be read, or the + baseline no longer matches the inventory. """ work = Path(output_dir) / WORK_DIRNAME report_path = work / REPORT_FILENAME @@ -579,22 +741,35 @@ def apply_plan_to_report(output_dir: Path, plan: dict[str, Any]) -> dict[str, An baseline_report, baseline_gaps, baseline_report_bytes, baseline_gaps_bytes = load_baseline( output_dir, fresh=record is None ) - _, combines = load_combines(output_dir) agentic = agentic_pipeline_names(plan) + if inventory is not None and agentic: + mismatch = baseline_pipeline_mismatch(baseline_report, inventory) + if mismatch: + raise ValueError(mismatch) + stored = load_agentic_output(output_dir) + dropped = 0 + if record is None and stored.document.get("gap_fills"): + baseline_report_sha256 = hashlib.sha256(baseline_report_bytes).hexdigest() + kept = [fill for fill in stored.gap_fills if fill["baseline_report_sha256"] == baseline_report_sha256] + dropped = len(stored.document["gap_fills"]) - len(kept) + if dropped: + save_agentic_output(output_dir, {"components": stored.document["components"], "gap_fills": kept}) + stored = load_agentic_output(output_dir) new_report, new_gaps, new_record = rebuild( baseline_report, baseline_gaps, plan, - combines, - record, + stored.outputs, + stored.gap_fills, baseline_report_bytes, baseline_gaps_bytes, ) + summary: dict[str, Any] = {"agentic_pipelines": sorted(agentic), "dropped_gap_fills": dropped} if not new_record: if record is not None: _write_atomic(report_path, baseline_report_bytes) _write_atomic(gaps_path, baseline_gaps_bytes) - return {"agentic_pipelines": sorted(agentic), "gaps": len(baseline_gaps), "altered": record is not None} + return {**summary, "gaps": len(baseline_gaps), "altered": record is not None, "outcomes": {}} if record is None: _save_baseline_bytes(output_dir, baseline_report_bytes, baseline_gaps_bytes) old_gaps_bytes = gaps_path.read_bytes() if gaps_path.exists() else b"[]" @@ -604,73 +779,88 @@ def apply_plan_to_report(output_dir: Path, plan: dict[str, Any]) -> dict[str, An new_report[ROUTING_RECORD_KEY] = new_record _write_json_atomic(report_path, new_report) _write_atomic(gaps_path, new_gaps_bytes) - return {"agentic_pipelines": sorted(agentic), "gaps": len(new_gaps), "altered": altered} + outcomes = {unit_id: entry["outcome"] for unit_id, entry in new_record["components"].items()} + return {**summary, "gaps": len(new_gaps), "altered": altered, "outcomes": outcomes} def apply_plan(output_dir: Path, plan: ConversionPlan) -> dict[str, Any]: """Apply a recorded, typed plan to the IR after convert: the library's one routing entry point. Checks the plan still matches the inventory, source graphs and agentic insights it was decided - on, then rebuilds the translation report from the deterministic baseline (see - :func:`apply_plan_to_report`). Each routed-agentic component is then filled only by - ``fill-agentic combine``; ``convert --merge-agentic`` stays for convert's own gaps. Phase 1 decides - whole components, so the reserved per-node assignments are not read here. + on, that ``agentic_insights.json`` agrees with the inventory, and that no decision is pending, + then rebuilds the translation report from the deterministic baseline (see + :func:`apply_plan_to_report`). Each routed-agentic unit is then filled only by ``fill-agentic``; + ``convert --merge-agentic`` stays for convert's own gaps. Phase 1 decides whole components (or + accepted groupings of them), so the reserved per-node assignments are not read here. Raises: FileNotFoundError: The inventory or translation report is missing. - ValueError: The plan is stale against the current discovery outputs. + ValueError: The plan is stale against the current discovery outputs, the insights files + disagree, a decision is pending, or the baseline was converted from another discover. """ - from flowx.routing import plan_binding_violations + from flowx.routing import insights_file_violations, plan_binding_violations inventory_path = Path(output_dir) / METADATA_DIRNAME / INVENTORY_FILENAME if not inventory_path.exists(): raise FileNotFoundError(f"No {INVENTORY_FILENAME} under {inventory_path.parent}; run the discover phase first.") - stale = plan_binding_violations(plan, json.loads(inventory_path.read_text(encoding="utf-8"))) - if stale: - raise ValueError("; ".join(stale)) - return apply_plan_to_report(output_dir, plan.to_dict()) + inventory = json.loads(inventory_path.read_text(encoding="utf-8")) + problems = plan_binding_violations(plan, inventory) + insights_file_violations(Path(output_dir), inventory) + if problems: + raise ValueError("; ".join(problems)) + pending = plan.pending_components() + if pending: + raise ValueError( + f"components {pending} are still pending; decide them in metadata/conversion_plan.json and re-run route" + ) + return apply_plan_to_report(output_dir, plan.to_dict(), inventory=inventory) # --------------------------------------------------------------------------- # -# Cross-pipeline COMBINE: the pipeline-grain fill. +# fill-agentic: replace a routed-agentic unit with the agent's pipelines. # --------------------------------------------------------------------------- # -def combine_group_fill( +def replace_unit_pipelines( report: dict[str, Any], - group_members: Iterable[str], + unit_members: Iterable[str], authored_pipelines: list[dict[str, Any]], ) -> dict[str, Any]: - """Replace a routed group's pipelines with the agent-authored pipeline(s). + """Replace a routed unit's pipelines with the agent-authored pipeline(s). - Drops every pipeline whose name is in ``group_members`` and appends the ``authored_pipelines`` + Drops every pipeline whose name is in ``unit_members`` and appends the ``authored_pipelines`` (each a pipeline IR dict, typically carrying ``AgenticComponentActivity`` nodes). Pipelines - outside the group are preserved in order. Returns a ``{"pipelines": [...]}`` report; the input is + outside the unit are preserved in order. Returns a ``{"pipelines": [...]}`` report; the input is not mutated. """ - members = {name for name in group_members} + members = {name for name in unit_members} kept = [copy.deepcopy(pipeline) for pipeline in _report_pipelines(report) if pipeline.get("name") not in members] kept.extend(copy.deepcopy(pipeline) for pipeline in authored_pipelines) return {"pipelines": kept} -def _resolve_agentic_component( +def _resolve_agentic_unit( output_dir: Path, members: set[str] ) -> tuple[str | None, dict[str, Any] | None, str | None, str | None]: - """Bind ``members`` to a routed-**agentic** component in the recorded, fingerprint-bound plan. + """Bind ``members`` to a routed-**agentic** unit in the recorded, fingerprint-bound plan. Reads ``metadata/conversion_plan.json`` and ``metadata/inventory.json`` and returns - ``(component_id, plan, source, error)``, where ``plan`` is the recorded plan document and - ``source`` the inventory's source, which every authored pipeline must carry. The error is set - (and the other three are ``None``) when: the inventory's source cannot be routed agentic - (:data:`flowx.routing.AGENTIC_ROUTING_SOURCES`); the plan or inventory is missing; the plan's - ``inventory_sha256`` no longer matches the current inventory (a stale plan); ``members`` do not - exactly equal one component's members (a partial, superset, or mistyped group); or the - exactly-matching component is routed deterministic rather than agentic. Requiring an exact match - to a routed-agentic component stops a caller from swapping deterministic pipelines or a partial - group. The plan is returned so the combine can check the report's routing record against it. + ``(unit_id, plan, source, error)``, where ``plan`` is the recorded plan document and ``source`` + the inventory's source, which every authored pipeline must carry. The error is set (and the other + three are ``None``) when: the inventory's source cannot be routed agentic + (:data:`flowx.routing.AGENTIC_ROUTING_SOURCES`); the plan or inventory is missing; the plan no + longer matches the current inventory, source graphs or insights, or the insights files disagree; + a decision is still pending; ``members`` do not exactly equal one unit's members (a partial, + superset, or mistyped group); or the exactly-matching unit is routed deterministic rather than + agentic. Requiring an exact match to a routed-agentic unit stops a caller from swapping + deterministic pipelines or a partial group. The plan is returned so the fill can check the + report's routing record against it. """ - from flowx.routing import agentic_routing_supported, inventory_source, plan_binding_violations + from flowx.routing import ( + agentic_routing_supported, + insights_file_violations, + inventory_source, + plan_binding_violations, + ) metadata = Path(output_dir) / METADATA_DIRNAME inventory_path = metadata / INVENTORY_FILENAME @@ -689,23 +879,31 @@ def _resolve_agentic_component( stale = plan_binding_violations(recorded, inventory) if stale: return None, None, None, f"conversion_plan.json is stale: {stale[0]}; re-run `route` before filling." + disagree = insights_file_violations(Path(output_dir), inventory) + if disagree: + return None, None, None, disagree[0] + pending = recorded.pending_components() + if pending: + return ( + None, + None, + None, + f"components {pending} are still pending; decide them and re-run `route` before filling.", + ) plan = recorded.to_dict() - for component in plan.get("components", []): - if not isinstance(component, dict): - continue - component_members = {str(member) for member in component.get("members", [])} - if component_members != members: + for unit_id, unit in routing_units(plan).items(): + if set(unit["members"]) != members: continue - if component.get("decision") == DECISION_AGENTIC: - return str(component.get("component_id")), plan, inventory_source(inventory), None + if unit["decision"] == DECISION_AGENTIC: + return unit_id, plan, inventory_source(inventory), None return ( None, None, None, ( - f"members {sorted(members)} match component {component.get('component_id')!r}, which is " - f"routed {component.get('decision')!r}, not agentic; only routed-agentic components can be combined." + f"members {sorted(members)} match component {unit_id!r}, which is routed {unit['decision']!r}, " + "not agentic; only routed-agentic components can be filled." ), ) return ( @@ -713,19 +911,19 @@ def _resolve_agentic_component( None, None, ( - f"members {sorted(members)} do not exactly match any component in the recorded plan " - "(partial, superset, or mistyped); pass the exact member set of one routed-agentic component." + f"members {sorted(members)} do not exactly match any component or accepted grouping in the recorded " + "plan (partial, superset, or mistyped); pass the exact member set of one routed-agentic unit." ), ) def _authored_source_tag_violations(authored_pipelines: list[dict[str, Any]], source: str) -> list[str]: - """Reports each authored combine pipeline that does not carry ``tags.source`` equal to ``source``. + """Reports each authored pipeline that does not carry ``tags.source`` equal to ``source``. The authored pipelines replace pipelines of the routed inventory, so one that omits or mis-sets - the source tag is an authoring error. Catching it here fails the combine closed (nothing - written) with a clear message, rather than letting the mis-tagged pipeline reach the package - preflight where it is only rejected much later. + the source tag is an authoring error. Catching it here fails the fill closed (nothing written) + with a clear message, rather than letting the mis-tagged pipeline reach the package preflight + where it is only rejected much later. """ violations: list[str] = [] for index, pipeline in enumerate(authored_pipelines): @@ -734,48 +932,111 @@ def _authored_source_tag_violations(authored_pipelines: list[dict[str, Any]], so authored_source = tags.get("source") if isinstance(tags, dict) else None if authored_source != source: violations.append( - f"{label}: authored combine pipeline must carry tags.source == {source!r}, got {authored_source!r}" + f"{label}: authored pipeline must carry tags.source == {source!r}, got {authored_source!r}" ) return violations -def apply_combine_fill( +def _placeholder_task_names(tasks: Any) -> list[str]: + """The names of every ``PlaceholderActivity`` among ``tasks``, looking inside containers too.""" + names: list[str] = [] + for task in tasks if isinstance(tasks, list) else []: + if not isinstance(task, dict): + continue + if task.get("type") == "PlaceholderActivity": + names.append(str(task.get("name"))) + for key in _NESTED_TASK_KEYS: + names.extend(_placeholder_task_names(task.get(key))) + for case in task.get("cases") or []: + if isinstance(case, dict): + names.extend(_placeholder_task_names(case.get("activities"))) + return names + + +def _authored_placeholder_violations(authored_pipelines: list[dict[str, Any]]) -> list[str]: + """Reports each authored pipeline that still holds a placeholder task, which would ship as a stub.""" + violations: list[str] = [] + for index, pipeline in enumerate(authored_pipelines): + if not isinstance(pipeline, dict): + continue + placeholders = _placeholder_task_names(pipeline.get("tasks")) + if placeholders: + label = pipeline.get("name") or f"pipeline[{index}]" + violations.append( + f"{label}: still has placeholder tasks {placeholders}; convert every task before filling the unit" + ) + return violations + + +def _bundle_folder_clashes(report: dict[str, Any], authored_names: list[Any]) -> list[str]: + """Authored names that are empty, or share a bundle folder with another pipeline once normalised. + + Package writes each pipeline of a multi-pipeline report to the folder ``normalize_task_key(name)``, + so ``Sales Load`` and ``Sales-Load`` would overwrite each other's bundle. + """ + from flowx.utils import normalize_task_key + + clashes: list[str] = [] + folders: dict[str, list[str]] = {} + for pipeline in _report_pipelines(report): + name = pipeline.get("name") + if isinstance(name, str) and name.strip(): + folders.setdefault(normalize_task_key(name), []).append(name) + for name in authored_names: + if not isinstance(name, str) or not normalize_task_key(name): + clashes.append(f"authored pipeline name {name!r} is empty; give every pipeline a name") + continue + sharing = folders.get(normalize_task_key(name), []) + if len(sharing) > 1: + others = sorted(set(sharing) - {name}) + clashes.append( + f"authored pipeline {name!r} shares the bundle folder {normalize_task_key(name)!r} with {others}; " + "give each a distinct name" + ) + return clashes + + +def apply_agentic_output( output_dir: Path, - group_members: Iterable[str], + unit_members: Iterable[str], authored_pipelines: list[dict[str, Any]], ) -> dict[str, Any]: - """Combine a routed-agentic group into agent-authored pipeline(s) on disk. + """Fill a routed-agentic unit with agent-authored pipeline(s) on disk. - Validates the group's membership against the recorded plan, stores the authored pipelines in - ``metadata/agentic_combines.json``, and rebuilds (applying it deterministically). The group's - membership must exactly match a routed-agentic component in the plan, and every authored pipeline - must carry the correct source tag and a name of its own: no repeat within the list, no member of - another component, and no pipeline another component's combine authored. A name may reuse a member - of this component, since the combine replaces it. + Validates the unit's membership against the recorded plan, stores the authored pipelines in + ``metadata/agentic_conversion.json``, and rebuilds (applying them deterministically). The + membership must exactly match a routed-agentic unit (a component, or an accepted grouping) in the + plan, and every authored pipeline must carry the correct source tag, hold no placeholder task, and + have a name of its own: no repeat within the list, no member of another unit, no pipeline another + unit's output authored, and no bundle folder shared with another pipeline once normalised. A name + may reuse a member of this unit, since the fill replaces it. - Returns ``{"ok", "violations", "error", "component_id", "pipelines", "already_combined", "message"}``. - ``ok`` is ``False`` (and nothing written) on a plan/membership error, source tag violation, name - clash, structural validation failure, or any other error. Empty ``authored_pipelines`` is refused. + Returns ``{"ok", "violations", "error", "component_id", "pipelines", "already_applied", "message"}``. + ``ok`` is ``False`` (and nothing written) on a plan/membership error, source tag violation, a + placeholder left in, a name clash, structural validation failure, or any other error. Empty + ``authored_pipelines`` is refused. The stored entry carries the canonical hash of the complete authored pipelines. When that hash matches the stored one and the live report already holds those pipelines, this returns - ``ok: true, already_combined: true, message: 'already applied, unchanged'`` and writes nothing. - Otherwise (different pipelines, an edited stored entry, or a report that no longer reflects the - combine) it replaces that component's entry only and rebuilds. Nothing is written until - structural validation passes (all-or-nothing). Modify's configured copy is never written. + ``ok: true, already_applied: true, message: 'already applied, unchanged'`` and writes nothing. + Otherwise it replaces that unit's entry only -- adding ``{"from", "to"}`` to the entry's + ``replaced`` history when a different output was stored before -- and rebuilds. Nothing is written + until structural validation passes (all-or-nothing). Modify's configured copy is never written; the + routing review page is rewritten. """ - members = {str(member) for member in group_members} + members = {str(member) for member in unit_members} if not authored_pipelines: return {"ok": False, "error": "pipelines list cannot be empty", "violations": [], "pipelines": 0} - component_id, plan, source, error = _resolve_agentic_component(output_dir, members) + unit_id, plan, source, error = _resolve_agentic_unit(output_dir, members) if error is not None: return {"ok": False, "error": error, "violations": [], "pipelines": 0} - assert component_id is not None and plan is not None and source is not None + assert unit_id is not None and plan is not None and source is not None - tag_violations = _authored_source_tag_violations(authored_pipelines, source) - if tag_violations: - return {"ok": False, "error": None, "violations": tag_violations, "pipelines": 0} + authoring_violations = _authored_source_tag_violations(authored_pipelines, source) + authoring_violations += _authored_placeholder_violations(authored_pipelines) + if authoring_violations: + return {"ok": False, "error": None, "violations": authoring_violations, "pipelines": 0} work = Path(output_dir) / WORK_DIRNAME report_path = work / REPORT_FILENAME @@ -788,21 +1049,18 @@ def apply_combine_fill( error = f"{REPORT_FILENAME} carries no routing record; run `route` to apply the plan before filling." return {"ok": False, "error": error, "violations": [], "pipelines": 0} if routing_record_mismatches(record, plan): - error = "the report was routed under a different plan; re-run route, then combine again" + error = "the report was routed under a different plan; re-run route, then fill-agentic again" return {"ok": False, "error": error, "violations": [], "pipelines": 0} - combine_sha256 = canonical_sha256(authored_pipelines) - stored, combines = load_combines(output_dir) + output_sha256 = canonical_sha256(authored_pipelines) + stored = load_agentic_output(output_dir) authored_names = [pipeline.get("name") for pipeline in authored_pipelines] taken_names = { - member - for component in plan.get("components", []) - if isinstance(component, dict) and component.get("component_id") != component_id - for member in component.get("members", []) + member for other_id, unit in routing_units(plan).items() if other_id != unit_id for member in unit["members"] } | { pipeline.get("name") - for other_id, entry in combines.items() - if other_id != component_id + for other_id, entry in stored.outputs.items() + if other_id != unit_id for pipeline in entry["pipelines"] if isinstance(pipeline, dict) } @@ -814,32 +1072,43 @@ def apply_combine_fill( ) return {"ok": False, "error": error, "violations": [], "pipelines": 0} in_report = {pipeline.get("name"): pipeline for pipeline in _report_pipelines(report)} - if (combines.get(component_id) or {}).get("combine_sha256") == combine_sha256 and all( + previous = stored.outputs.get(unit_id) + if (previous or {}).get("output_sha256") == output_sha256 and all( in_report.get(pipeline.get("name")) == pipeline for pipeline in authored_pipelines ): return { "ok": True, "error": None, "violations": [], - "component_id": component_id, + "component_id": unit_id, "pipelines": len(_report_pipelines(report)), - "already_combined": True, + "already_applied": True, "message": "already applied, unchanged", } - entry = {"members": sorted(members), "pipelines": authored_pipelines, "combine_sha256": combine_sha256} - stored[component_id] = entry - combines[component_id] = entry + raw_previous = stored.document.get("components", {}).get(unit_id) + replaced = list(raw_previous.get("replaced") or []) if isinstance(raw_previous, dict) else [] + if previous is not None and previous["output_sha256"] != output_sha256: + replaced.append({"from": previous["output_sha256"], "to": output_sha256}) + entry = { + "members": sorted(members), + "pipelines": authored_pipelines, + "output_sha256": output_sha256, + "replaced": replaced, + } baseline_report, baseline_gaps, baseline_report_bytes, baseline_gaps_bytes = load_baseline(output_dir, fresh=False) new_report, new_gaps, new_record = rebuild( baseline_report, baseline_gaps, plan, - combines, - record, + {**stored.outputs, unit_id: entry}, + stored.gap_fills, baseline_report_bytes, baseline_gaps_bytes, ) + folder_clashes = _bundle_folder_clashes(new_report, authored_names) + if folder_clashes: + return {"ok": False, "error": None, "violations": folder_clashes, "pipelines": 0} new_report[ROUTING_RECORD_KEY] = new_record result = validate_report_structurally(new_report) @@ -847,17 +1116,24 @@ def apply_combine_fill( violations = [f"[{finding.code}] {finding.location}: {finding.message}" for finding in result.violations] return {"ok": False, "error": None, "violations": violations, "pipelines": 0} - _save_combines(output_dir, stored) + document = { + "components": {**stored.document.get("components", {}), unit_id: entry}, + "gap_fills": stored.document.get("gap_fills", []), + } + save_agentic_output(output_dir, document) _write_json_atomic(report_path, new_report) _write_json_atomic(work / GAPS_FILENAME, new_gaps) + from flowx.reporting.routing_review import write_routing_review + + write_routing_review(Path(output_dir)) return { "ok": True, "error": None, "violations": [], - "component_id": component_id, + "component_id": unit_id, "pipelines": len(new_report["pipelines"]), - "already_combined": False, + "already_applied": False, } diff --git a/src/flowx/routing.py b/src/flowx/routing.py index 2a1bc644..148d06c7 100644 --- a/src/flowx/routing.py +++ b/src/flowx/routing.py @@ -15,13 +15,18 @@ ``simplification_pattern`` such as a multi-pipeline -> Lakeflow Connect re-architecture surfaced prominently) -- so the user can choose per group; * ``recommended`` is a library-computed starting suggestion (deterministic when the whole component - is engine-capable), and ``decision`` is the user's authored per-component choice. + is engine-capable), and ``decision`` is the user's authored per-component choice, ``None`` while + it is still pending; +* **suggested groupings** join whole components the insights link -- by an inferred relationship, + or by a simplification pattern recommended for pipelines in more than one component -- so the user + can accept converting them together agentically as one unit (:func:`suggest_groupings`). Like :mod:`flowx.discovery_insights`, there is **no LLM here**: the tool computes the recommendation -deterministically and only *validates and records* the agent-authored decision. The agent authors -**only** the decision (and an optional rationale); the library recomputes ``members``, -``recommended``, and both options' evidence on record so they can never drift from the inventory or -be faked. +deterministically and only *validates and records* the agent-authored decision. Route writes the +recommendation straight into ``conversion_plan.json`` with every decision pending; the agent edits +the decisions, the groupings' ``accepted`` flags and the optional routing ``conversation`` there (or +passes them as a plan), and the library recomputes ``members``, ``recommended``, both options' +evidence and the groupings on every record so they can never drift from the inventory or be faked. The recorded plan is bound to the inventory via :func:`flowx.discovery_insights.inventory_fingerprint` -- a SHA-256 over the deterministic inventory base (the ``insights`` block excluded). That base is @@ -45,8 +50,26 @@ from pathlib import Path from typing import Any -from flowx.discovery_insights import inventory_fingerprint -from flowx.models.conversion_plan import DECISIONS, PLAN_FILENAME, SCHEMA_VERSION, ComponentPlan, ConversionPlan +from flowx.discovery_insights import ( + AGENTIC_INSIGHTS_FILENAME, + ENRICH_LOCK_FILENAME, + INSIGHTS_KEY, + agentic_insights_hash_violations, + inventory_fingerprint, +) +from flowx.models.conversion_plan import ( + BASIS_INFERRED_RELATIONSHIP, + BASIS_SHARED_PATTERN, + DECISION_AGENTIC, + DECISION_DETERMINISTIC, + DECISIONS, + PLAN_FILENAME, + SCHEMA_VERSION, + ComponentPlan, + ConversationEntry, + ConversionPlan, + SuggestedGrouping, +) # The strategy value that marks an activity as individually engine-capable (a 1:1 deterministic # translation). Any other value (``"agentic"`` / ``"unsupported"`` / missing) is a gap unless the @@ -59,18 +82,29 @@ AGENTIC_ROUTING_SOURCES = frozenset({"adf"}) _LEGACY_INVENTORY_SOURCE = "adf" -# Authored top-level keys (everything else the library owns and rejects on input). -_PLAN_TOP_KEYS = {"components"} -_LIBRARY_TOP_KEYS = { +# Each level of the plan holds authored keys and library-owned keys. The library recomputes its own +# keys on every record, so an edited copy of the recorded plan can be handed straight back; any key +# that is neither is refused. +_PLAN_AUTHORED_KEYS = {"components", "suggested_groupings", "conversation"} +_PLAN_LIBRARY_KEYS = { "schema_version", "inventory_sha256", "source_graphs_sha256", "agentic_insights_sha256", "findings", } -# Authored per-component keys vs the fields the library recomputes and rejects on input. _COMPONENT_AUTHORED_KEYS = {"component_id", "members", "decision", "rationale", "assignments"} _COMPONENT_LIBRARY_KEYS = {"recommended", "options"} +_GROUPING_AUTHORED_KEYS = {"grouping_id", "accepted"} +_GROUPING_LIBRARY_KEYS = {"components", "members", "basis"} +_CONVERSATION_KEYS = {"question", "answer"} + +# How to bring agentic_insights.json and inventory.json back in step, mirroring enrich's own advice. +_INSIGHTS_RECOVERY = ( + "enrich may have stopped between its two writes, or one of the files was edited. To recover, run enrich " + f"again with the same insights (locally, first delete metadata/{ENRICH_LOCK_FILENAME} if it is still there; " + "on the hosted MCP server, run discover again, then enrich prepare and apply)" +) def inventory_source(inventory: dict[str, Any]) -> str: @@ -356,14 +390,106 @@ def recommend_component(members: list[str], inventory: dict[str, Any]) -> tuple[ return recommended, {"deterministic": deterministic, "agentic": agentic} +def suggest_groupings(inventory: dict[str, Any]) -> list[dict[str, Any]]: + """Suggest whole components to convert together, agentically, from the inventory's insights. + + Two kinds of link join components, both read from the insights enrich recorded: + + * an ``inferred`` pipeline relationship (a coupling the parser could not see, with the agent's + evidence and confidence) between pipelines of different components; + * one simplification pattern (a ``recommended_patterns`` entry with ``simplification_pattern`` + true, by the same pattern name) recommended for pipelines in more than one component -- the + independent extractors one managed pattern could replace, which have no coupling at all. + + Each set of two or more components joined by these links, directly or through each other, + becomes one suggestion, so suggestions never overlap. A suggestion lists its ``basis``: every + relationship or shared pattern behind it. Nothing is suggested without insights, or for an + inventory whose source cannot be routed agentic (:data:`AGENTIC_ROUTING_SOURCES`). + + Returns ``[{"grouping_id", "components", "members", "basis", "accepted": False}]`` with ids + ``grouping-`` in component order, deterministic for a given inventory. + """ + insights = inventory.get(INSIGHTS_KEY) + if not isinstance(insights, dict) or not agentic_routing_supported(inventory): + return [] + components_by_id = _components_by_id(inventory) + component_of = {member: component_id for component_id, members in components_by_id.items() for member in members} + + links: list[tuple[set[str], dict[str, Any]]] = [] + for relationship in insights.get("pipeline_relationships") or []: + edge = relationship.get("lineage_edge") if isinstance(relationship, dict) else None + if not isinstance(edge, dict) or edge.get("edge_type") != "inferred": + continue + source, target = relationship.get("from_pipeline"), relationship.get("to_pipeline") + if source not in component_of or target not in component_of: + continue + basis: dict[str, Any] = { + "kind": BASIS_INFERRED_RELATIONSHIP, + "from_pipeline": source, + "to_pipeline": target, + "edge_identity": edge.get("edge_identity"), + "evidence": edge.get("evidence"), + "confidence": edge.get("confidence"), + } + if relationship.get("relationship_summary"): + basis["relationship_summary"] = relationship["relationship_summary"] + links.append(({component_of[source], component_of[target]}, basis)) + + pipelines_by_pattern: dict[str, set[str]] = {} + for insight in insights.get("pipeline_insights") or []: + pipeline = insight.get("pipeline") if isinstance(insight, dict) else None + if pipeline not in component_of: + continue + for pattern in insight.get("recommended_patterns") or []: + if isinstance(pattern, dict) and pattern.get("simplification_pattern") is True: + if isinstance(pattern.get("pattern"), str) and pattern["pattern"]: + pipelines_by_pattern.setdefault(pattern["pattern"], set()).add(pipeline) + for pattern_name in sorted(pipelines_by_pattern): + pipelines = pipelines_by_pattern[pattern_name] + basis = {"kind": BASIS_SHARED_PATTERN, "pattern": pattern_name, "pipelines": sorted(pipelines)} + links.append(({component_of[pipeline] for pipeline in pipelines}, basis)) + + parent = {component_id: component_id for component_id in components_by_id} + + def find(component_id: str) -> str: + while parent[component_id] != component_id: + component_id = parent[component_id] + return component_id + + order = {component_id: index for index, component_id in enumerate(components_by_id)} + for linked, _ in links: + roots = sorted({find(component_id) for component_id in linked}, key=order.__getitem__) + for root in roots[1:]: + parent[root] = roots[0] + + joined: dict[str, list[str]] = {} + for component_id in components_by_id: + joined.setdefault(find(component_id), []).append(component_id) + groupings: list[dict[str, Any]] = [] + for component_ids in joined.values(): + if len(component_ids) < 2: + continue + members = sorted(member for component_id in component_ids for member in components_by_id[component_id]) + groupings.append( + { + "grouping_id": f"grouping-{len(groupings) + 1}", + "components": component_ids, + "members": members, + "basis": [basis for linked, basis in links if linked <= set(component_ids) and len(linked) > 1], + "accepted": False, + } + ) + return groupings + + def build_recommendation(inventory: dict[str, Any]) -> dict[str, Any]: """Compute the full routing recommendation over every component in the inventory. Returns a dict with ``components`` (each carrying ``component_id``, sorted ``members``, - ``recommended`` and both ``options``), the ``findings`` from component computation, and a - ``default_plan`` that proposes ``decision == recommended`` for every component -- ready to hand - straight to :func:`record_plan` when the user accepts the recommendations wholesale, or to edit - per component for overrides. + ``recommended`` and both ``options``), the ``suggested_groupings`` (see :func:`suggest_groupings`), + the ``findings`` from component computation, and a ``default_plan`` that proposes + ``decision == recommended`` for every component -- ready to hand straight to :func:`record_plan` + when the user accepts the recommendations wholesale, or to edit per component for overrides. """ components, findings = build_components(inventory) if not agentic_routing_supported(inventory): @@ -379,6 +505,7 @@ def build_recommendation(inventory: dict[str, Any]) -> dict[str, Any]: default_plan_components.append({"component_id": component_id, "members": members, "decision": recommended}) return { "components": component_entries, + "suggested_groupings": suggest_groupings(inventory), "findings": findings, "default_plan": {"components": default_plan_components}, } @@ -400,25 +527,27 @@ def validate_plan(raw: Any, inventory: dict[str, Any]) -> list[str]: """Validate an authored conversion plan against the inventory's connected components. Returns a list of human-readable violation strings; an empty list means the plan is valid. Never - raises on a malformed payload. The rules: + raises on a malformed payload. The plan may be the authored fields alone or an edited copy of the + recorded ``conversion_plan.json``: library-owned keys are accepted and ignored (they are recomputed + on record), and any other unknown key is refused. The rules: - * only the authored top-level key ``components`` is allowed (library-owned keys are rejected with - a hint), and each component entry may carry only the authored fields (``recommended`` / - ``options`` are library-computed and rejected on input); - * ``decision`` must be one of the known routes and ``rationale`` (when present) a non-empty string; + * ``decision`` is one of the known routes, or ``None`` / absent while still pending, and + ``rationale`` (when present) is a non-empty string; * every member must be a real inventory pipeline, and a component's ``members`` must exactly match one computed connected component -- so a decision can never split a component or span two; - * the plan is a **bijection** over components: every component is decided exactly once (no + * the plan is a **bijection** over components: every component is listed exactly once (no partial plan, no duplicate/conflicting decisions); - * an ``agentic`` decision is only accepted for an ADF inventory (:func:`agentic_routing_supported`). + * an ``agentic`` decision is only accepted for an ADF inventory (:func:`agentic_routing_supported`); + * ``suggested_groupings`` entries name a grouping route suggests for this inventory, ``accepted`` + is a boolean, and an accepted grouping has none of its components decided ``deterministic``; + * ``conversation`` is a list of ``{"question", "answer"}`` objects with non-empty strings. """ if not isinstance(raw, dict): return [f"conversion plan must be a JSON object, got {type(raw).__name__}"] violations: list[str] = [] - for key in sorted(set(raw) - _PLAN_TOP_KEYS): - hint = " (set by the library, not the author)" if key in _LIBRARY_TOP_KEYS else "" - violations.append(f"unknown top-level key: {key!r}{hint}") + for key in sorted(set(raw) - _PLAN_AUTHORED_KEYS - _PLAN_LIBRARY_KEYS): + violations.append(f"unknown top-level key: {key!r}") components = raw.get("components") if not isinstance(components, list): @@ -430,15 +559,17 @@ def validate_plan(raw: Any, inventory: dict[str, Any]) -> list[str]: computed_by_members = {frozenset(members): component_id for component_id, members in computed_by_id.items()} decided_ids: list[str] = [] + decision_by_id: dict[str, Any] = {} for index, component in enumerate(components): loc = f"components[{index}]" matched_id = _validate_component_entry(component, loc, names, computed_by_members, violations) if matched_id is not None: decided_ids.append(matched_id) + decision_by_id[matched_id] = component.get("decision") if not agentic_routing_supported(inventory): for index, component in enumerate(components): - if isinstance(component, dict) and component.get("decision") == "agentic": + if isinstance(component, dict) and component.get("decision") == DECISION_AGENTIC: violations.append(f"components[{index}]: {_agentic_routing_unsupported_note(inventory)}") for component_id, count in Counter(decided_ids).items(): @@ -450,6 +581,9 @@ def validate_plan(raw: Any, inventory: dict[str, Any]) -> list[str]: for component_id, members in computed_by_id.items(): if component_id not in decided: violations.append(f"component {component_id!r} ({members}) has no decision; every component must be routed") + + violations.extend(_grouping_violations(raw.get("suggested_groupings", []), inventory, decision_by_id)) + violations.extend(_conversation_violations(raw.get("conversation", []))) return violations @@ -469,14 +603,13 @@ def _validate_component_entry( violations.append(f"{loc} must be an object") return None - for key in sorted(set(component) - _COMPONENT_AUTHORED_KEYS): - hint = " (set by the library, not the author)" if key in _COMPONENT_LIBRARY_KEYS else "" - violations.append(f"{loc}: unknown field {key!r}{hint}") + for key in sorted(set(component) - _COMPONENT_AUTHORED_KEYS - _COMPONENT_LIBRARY_KEYS): + violations.append(f"{loc}: unknown field {key!r}") decision = component.get("decision") - if decision not in DECISIONS: + if decision is not None and decision not in DECISIONS: allowed = ", ".join(repr(value) for value in DECISIONS) - violations.append(f"{loc}: 'decision' must be one of {{{allowed}}}, got {decision!r}") + violations.append(f"{loc}: 'decision' must be one of {{{allowed}}}, or null while pending, got {decision!r}") rationale = component.get("rationale") if rationale is not None and (not isinstance(rationale, str) or not rationale.strip()): @@ -512,7 +645,8 @@ def _validate_component_entry( if matched_id is None: violations.append( f"{loc}: members {sorted(members)} do not form a connected component " - f"(they split or span computed components); route each component as a whole" + f"(they split or span computed components); route each component as a whole, " + "or accept a suggested grouping to convert several together" ) return None if isinstance(component_id, str) and component_id and component_id != matched_id: @@ -523,6 +657,68 @@ def _validate_component_entry( return matched_id +def _grouping_violations(groupings: Any, inventory: dict[str, Any], decision_by_id: dict[str, Any]) -> list[str]: + """Check the authored ``suggested_groupings`` entries against the groupings route suggests.""" + if not isinstance(groupings, list): + return ["'suggested_groupings' must be a list"] + suggestions = {grouping["grouping_id"]: grouping for grouping in suggest_groupings(inventory)} + violations: list[str] = [] + seen: set[str] = set() + for index, entry in enumerate(groupings): + loc = f"suggested_groupings[{index}]" + if not isinstance(entry, dict): + violations.append(f"{loc} must be an object") + continue + for key in sorted(set(entry) - _GROUPING_AUTHORED_KEYS - _GROUPING_LIBRARY_KEYS): + violations.append(f"{loc}: unknown field {key!r}") + grouping_id = entry.get("grouping_id") + suggestion = suggestions.get(grouping_id) if isinstance(grouping_id, str) else None + if suggestion is None: + violations.append( + f"{loc}: {grouping_id!r} is not a grouping route suggests for this inventory; only a suggested " + "grouping can be accepted (to group other components, record an inferred relationship through enrich)" + ) + continue + if grouping_id in seen: + violations.append(f"{loc}: grouping {grouping_id!r} is listed more than once") + seen.add(str(grouping_id)) + accepted = entry.get("accepted", False) + if not isinstance(accepted, bool): + violations.append(f"{loc}: 'accepted' must be true or false, got {accepted!r}") + elif accepted: + deterministic = [ + component_id + for component_id in suggestion["components"] + if decision_by_id.get(component_id) == DECISION_DETERMINISTIC + ] + if deterministic: + violations.append( + f"{loc}: grouping {grouping_id!r} converts {suggestion['components']} together agentically, but " + f"{deterministic} {'is' if len(deterministic) == 1 else 'are'} decided 'deterministic'; decide " + "each of its components 'agentic', or set accepted to false" + ) + return violations + + +def _conversation_violations(conversation: Any) -> list[str]: + """Check the authored routing ``conversation``: a list of non-empty ``{question, answer}`` pairs.""" + if not isinstance(conversation, list): + return ["'conversation' must be a list of {question, answer} objects"] + violations: list[str] = [] + for index, entry in enumerate(conversation): + loc = f"conversation[{index}]" + if not isinstance(entry, dict): + violations.append(f"{loc} must be an object") + continue + for key in sorted(set(entry) - _CONVERSATION_KEYS): + violations.append(f"{loc}: unknown field {key!r}") + for key in sorted(_CONVERSATION_KEYS): + value = entry.get(key) + if not isinstance(value, str) or not value.strip(): + violations.append(f"{loc}: {key!r} must be a non-empty string") + return violations + + # --------------------------------------------------------------------------- # # Loading, recording, and the atomic idempotent write. # --------------------------------------------------------------------------- # @@ -542,14 +738,69 @@ def load_plan(*, plan: dict[str, Any] | None = None, plan_path: Path | None = No return json.loads(Path(plan_path).read_text(encoding="utf-8")) +def carried_forward_plan(previous: Any, inventory: dict[str, Any]) -> dict[str, Any]: + """Build the authored plan route records when no plan is supplied: the recommendation, kept decisions. + + This is how route writes its recommendation straight into ``conversion_plan.json`` and then picks + up the agent's edits to it. Every current component starts pending (``decision: None``); a + component the *previous* recorded plan lists with the same members keeps its ``decision``, + ``rationale`` and ``assignments`` as written there, a suggested grouping with the same id and + members keeps its ``accepted`` flag, and the ``conversation`` is kept. Entries whose components no + longer exist are dropped, so a re-discover that regroups pipelines leaves those components + pending. Values are carried as written; :func:`validate_plan` then checks them. + + Args: + previous: The parsed ``conversion_plan.json`` (edited or not), or ``None`` when there is none. + inventory: The current discover ``inventory.json`` document. + """ + recommendation = build_recommendation(inventory) + previous = previous if isinstance(previous, dict) else {} + previous_components = previous.get("components") if isinstance(previous.get("components"), list) else [] + by_members: dict[frozenset[str], dict[str, Any]] = {} + for entry in previous_components: + members = entry.get("members") if isinstance(entry, dict) else None + if isinstance(members, list) and all(isinstance(member, str) for member in members): + by_members[frozenset(members)] = entry + + components: list[dict[str, Any]] = [] + for component in recommendation["components"]: + authored: dict[str, Any] = { + "component_id": component["component_id"], + "members": component["members"], + "decision": None, + } + kept = by_members.get(frozenset(component["members"])) + if kept is not None: + for key in sorted(_COMPONENT_AUTHORED_KEYS - {"component_id", "members"}): + if key in kept: + authored[key] = kept[key] + components.append(authored) + + previous_groupings = previous.get("suggested_groupings") + accepted_before: set[tuple[Any, tuple[str, ...]]] = set() + for entry in previous_groupings if isinstance(previous_groupings, list) else []: + if isinstance(entry, dict) and entry.get("accepted") is True and isinstance(entry.get("members"), list): + accepted_before.add((entry.get("grouping_id"), tuple(sorted(str(member) for member in entry["members"])))) + groupings = [ + { + "grouping_id": grouping["grouping_id"], + "accepted": (grouping["grouping_id"], tuple(grouping["members"])) in accepted_before, + } + for grouping in recommendation["suggested_groupings"] + ] + conversation = previous.get("conversation", []) + return {"components": components, "suggested_groupings": groupings, "conversation": conversation} + + def build_plan(inventory: dict[str, Any], raw: dict[str, Any]) -> ConversionPlan: """Build the typed plan from a validated authored plan. - The library recomputes ``members`` / ``recommended`` / both ``options`` and overlays only the - authored ``decision`` (and optional ``rationale``) per component, so the recorded facts cannot - drift from the inventory. It stamps the schema version and the hashes that bind the plan to what - it was decided on: the inventory fingerprint, the saved source graphs the inventory records, and - the saved agentic insights when enrich ran. Does not mutate the inputs and performs no I/O. + The library recomputes ``members`` / ``recommended`` / both ``options`` and the suggested + groupings, and overlays only the authored ``decision`` (and optional ``rationale``) per component, + each grouping's ``accepted`` flag and the ``conversation``, so the recorded facts cannot drift from + the inventory. It stamps the schema version and the hashes that bind the plan to what it was + decided on: the inventory fingerprint, the saved source graphs the inventory records, and the saved + agentic insights when enrich ran. Does not mutate the inputs and performs no I/O. """ recommendation = build_recommendation(inventory) computed_by_id = _components_by_id(inventory) @@ -565,19 +816,39 @@ def build_plan(inventory: dict[str, Any], raw: dict[str, Any]) -> ConversionPlan component_id=entry["component_id"], members=entry["members"], recommended=entry["recommended"], - decision=authored_by_id[entry["component_id"]]["decision"], + decision=authored_by_id[entry["component_id"]].get("decision"), options=entry["options"], rationale=authored_by_id[entry["component_id"]].get("rationale"), ) for entry in recommendation["components"] ] - insights = inventory.get("insights") + accepted_ids = { + entry.get("grouping_id") + for entry in raw.get("suggested_groupings") or [] + if isinstance(entry, dict) and entry.get("accepted") is True + } + groupings = [ + SuggestedGrouping( + grouping_id=grouping["grouping_id"], + components=grouping["components"], + members=grouping["members"], + basis=grouping["basis"], + accepted=grouping["grouping_id"] in accepted_ids, + ) + for grouping in recommendation["suggested_groupings"] + ] + insights = inventory.get(INSIGHTS_KEY) return ConversionPlan( schema_version=SCHEMA_VERSION, inventory_sha256=inventory_fingerprint(inventory), source_graphs_sha256=inventory.get("source_graphs_sha256"), agentic_insights_sha256=insights.get("agentic_insights_sha256") if isinstance(insights, dict) else None, components=components, + suggested_groupings=groupings, + conversation=[ + ConversationEntry(question=entry["question"], answer=entry["answer"]) + for entry in raw.get("conversation") or [] + ], findings=recommendation["findings"], ) @@ -597,13 +868,58 @@ def plan_binding_violations(plan: ConversionPlan, inventory: dict[str, Any]) -> ) if plan.source_graphs_sha256 != inventory.get("source_graphs_sha256"): violations.append(f"{PLAN_FILENAME} was recorded against different source graphs; re-run route") - insights = inventory.get("insights") + insights = inventory.get(INSIGHTS_KEY) current_insights = insights.get("agentic_insights_sha256") if isinstance(insights, dict) else None if plan.agentic_insights_sha256 != current_insights: violations.append(f"{PLAN_FILENAME} was recorded against different agentic insights; re-run route") return violations +def recorded_plan_violations(plan: ConversionPlan, inventory: dict[str, Any]) -> list[str]: + """Check a recorded plan is still a complete, valid decision for the inventory's components. + + Runs the same rules :func:`validate_plan` applies on record, so a recorded file that lost a + component, gained one, or holds an invalid grouping is refused rather than packaged as if every + component had been decided. + """ + return validate_plan(plan.to_dict(), inventory) + + +def insights_file_violations(output_dir: Path, inventory: dict[str, Any] | None) -> list[str]: + """Check ``agentic_insights.json`` and the inventory's ``insights`` block still agree. + + Enrich writes ``metadata/agentic_insights.json`` and then ``inventory.json``; a run stopped between + the two, or a hand edit, leaves them out of step. This refuses when the inventory carries insights + but the file is missing, the file exists but the inventory's block differs from it (or there is no + inventory to compare with), or the file no longer matches its own ``agentic_insights_sha256``. + Returns one message saying how to recover; an empty list when they agree or enrich never ran. + """ + path = Path(output_dir) / "metadata" / AGENTIC_INSIGHTS_FILENAME + inventory_insights = inventory.get(INSIGHTS_KEY) if isinstance(inventory, dict) else None + if not path.exists() and inventory_insights is None: + return [] + problem: str | None = None + if not path.exists(): + problem = f"inventory.json carries insights but metadata/{AGENTIC_INSIGHTS_FILENAME} is missing" + else: + try: + document = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + document = None + problem = f"metadata/{AGENTIC_INSIGHTS_FILENAME} is unreadable ({error})" + if problem is None and not isinstance(document, dict): + problem = f"metadata/{AGENTIC_INSIGHTS_FILENAME} must contain a JSON object" + elif problem is None and isinstance(document, dict) and agentic_insights_hash_violations(document): + problem = f"metadata/{AGENTIC_INSIGHTS_FILENAME} does not match its recorded agentic_insights_sha256" + elif problem is None and inventory is None: + problem = f"metadata/{AGENTIC_INSIGHTS_FILENAME} is present but metadata/inventory.json is missing" + elif problem is None and inventory_insights != document: + problem = f"metadata/{AGENTIC_INSIGHTS_FILENAME} and metadata/inventory.json disagree" + if problem is None: + return [] + return [f"{problem}; {_INSIGHTS_RECOVERY}"] + + def record_plan( output_dir: Path, *, @@ -615,10 +931,11 @@ def record_plan( Reads ``/metadata/inventory.json``, validates the authored plan, and -- only when there are no violations -- writes ``/metadata/conversion_plan.json`` atomically. The inventory file is never touched. Provide the authored plan via exactly one of ``plan`` (inline - dict) or ``plan_path`` (a JSON file). + dict) or ``plan_path`` (a JSON file). A plan may leave decisions pending; it is recorded, and + route applies it once nothing is pending. - Returns ``{"ok", "violations", "inventory_sha256", "components", "findings"}``. ``ok`` is - ``False`` (and no plan written) when there are violations. + Returns ``{"ok", "violations", "inventory_sha256", "components", "pending", "findings"}``. ``ok`` + is ``False`` (and no plan written) when there are violations. Raises: FileNotFoundError: when ``inventory.json`` does not exist (run discover first). @@ -646,5 +963,7 @@ def record_plan( "source_graphs_sha256": recorded.source_graphs_sha256, "agentic_insights_sha256": recorded.agentic_insights_sha256, "components": len(recorded.components), + "pending": recorded.pending_components(), + "accepted_groupings": [grouping.grouping_id for grouping in recorded.suggested_groupings if grouping.accepted], "findings": len(recorded.findings), } diff --git a/tests/unit/test_cli_route_agentic.py b/tests/unit/test_cli_route_agentic.py index 2d907dd3..fa527e71 100644 --- a/tests/unit/test_cli_route_agentic.py +++ b/tests/unit/test_cli_route_agentic.py @@ -1,10 +1,10 @@ -"""Tests for the reshaped ``route`` CLI and the ``fill-agentic`` combine surface. +"""Tests for the ``route`` CLI and the ``fill-agentic`` surface. -``route`` is now one command: with no decision on a non-TTY it emits the recommendation (a dry run -agents read first); with a decision (``--plan-path``, ``--plan-path -`` for stdin, or an interactive -TTY prompt) it records the fingerprint-bound plan AND edits the report for the routed-agentic groups. -``fill-agentic combine`` performs the cross-pipeline pipeline-grain fill, validating structurally -before writing. +``route`` writes its recommendation straight into ``metadata/conversion_plan.json`` (keeping the +decisions already recorded there, every other one pending) and ``metadata/routing_review.html``; the +decisions can also come from ``--plan-path``, ``--plan-path -`` (stdin) or an interactive TTY prompt. +Once nothing is pending it edits the report for the routed-agentic units. ``fill-agentic`` fills a +routed-agentic unit, validating structurally before writing. """ from __future__ import annotations @@ -86,19 +86,94 @@ def _agentic_plan() -> dict[str, Any]: } -def test_route_without_a_decision_on_a_non_tty_emits_the_recommendation( +def test_route_without_a_decision_writes_a_pending_plan_and_review_page_and_leaves_the_report( tmp_path: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch ) -> None: _setup(tmp_path) + report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME + report_before = report_path.read_bytes() monkeypatch.setattr("sys.stdin", io.StringIO("")) # not a TTY + code = adapter_cli_main(["route", "--output-dir", str(tmp_path)]) + assert code == 0 payload = json.loads(capsys.readouterr().out) - assert [c["component_id"] for c in payload["components"]] == ["component-1", "component-2"] - assert "default_plan" in payload - # No decision => report untouched. + assert payload["applied"] is False + assert payload["pending"] == ["component-1", "component-2"] + plan = json.loads((tmp_path / "metadata" / "conversion_plan.json").read_text(encoding="utf-8")) + assert [component["decision"] for component in plan["components"]] == [None, None] + assert [component["recommended"] for component in plan["components"]] == ["deterministic", "deterministic"] + assert (tmp_path / "metadata" / "routing_review.html").is_file() + assert report_path.read_bytes() == report_before + assert not (tmp_path / WORK_DIRNAME / "route_baseline").exists() + + +def _edit_recorded_plan(tmp_path: Path, decisions: dict[str, str]) -> None: + """Edit the recorded conversion_plan.json in place, the way an agent does.""" + plan_path = tmp_path / "metadata" / "conversion_plan.json" + plan = json.loads(plan_path.read_text(encoding="utf-8")) + for component in plan["components"]: + component["decision"] = decisions[component["component_id"]] + plan_path.write_text(json.dumps(plan, indent=2), encoding="utf-8") + + +def test_editing_the_recorded_plan_and_re_running_route_applies_it( + tmp_path: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch +) -> None: + _setup(tmp_path) + monkeypatch.setattr("sys.stdin", io.StringIO("")) + assert adapter_cli_main(["route", "--output-dir", str(tmp_path)]) == 0 + _edit_recorded_plan(tmp_path, {"component-1": "agentic", "component-2": "deterministic"}) + capsys.readouterr() + + assert adapter_cli_main(["route", "--output-dir", str(tmp_path)]) == 0 + + payload = json.loads(capsys.readouterr().out) + assert payload["applied"] is True and payload["edit"]["agentic_pipelines"] == ["child", "parent"] + plan = json.loads((tmp_path / "metadata" / "conversion_plan.json").read_text(encoding="utf-8")) + assert [component["decision"] for component in plan["components"]] == ["agentic", "deterministic"] report = json.loads((tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_text(encoding="utf-8")) - assert all(pipeline["tasks"][0]["type"] != "PlaceholderActivity" for pipeline in report["pipelines"]) + by_name = {pipeline["name"]: pipeline for pipeline in report["pipelines"]} + assert by_name["parent"]["tasks"][0]["type"] == "PlaceholderActivity" + + +def test_route_recomputes_library_fields_an_agent_edited( + tmp_path: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch +) -> None: + _setup(tmp_path) + monkeypatch.setattr("sys.stdin", io.StringIO("")) + assert adapter_cli_main(["route", "--output-dir", str(tmp_path)]) == 0 + plan_path = tmp_path / "metadata" / "conversion_plan.json" + plan = json.loads(plan_path.read_text(encoding="utf-8")) + recorded_hash = plan["inventory_sha256"] + plan["inventory_sha256"] = "forged" + plan["components"][0]["recommended"] = "agentic" + plan["components"][0]["options"] = {"deterministic": {"capable": False}} + plan_path.write_text(json.dumps(plan, indent=2), encoding="utf-8") + + assert adapter_cli_main(["route", "--output-dir", str(tmp_path)]) == 0 + + rewritten = json.loads(plan_path.read_text(encoding="utf-8")) + assert rewritten["inventory_sha256"] == recorded_hash + assert rewritten["components"][0]["recommended"] == "deterministic" + assert rewritten["components"][0]["options"]["deterministic"]["capable"] is True + + +def test_route_refuses_an_invalid_edited_decision_and_keeps_the_file( + tmp_path: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch +) -> None: + _setup(tmp_path) + monkeypatch.setattr("sys.stdin", io.StringIO("")) + assert adapter_cli_main(["route", "--output-dir", str(tmp_path)]) == 0 + _edit_recorded_plan(tmp_path, {"component-1": "Agentic", "component-2": "deterministic"}) + edited = (tmp_path / "metadata" / "conversion_plan.json").read_bytes() + capsys.readouterr() + + assert adapter_cli_main(["route", "--output-dir", str(tmp_path)]) == 1 + + payload = json.loads(capsys.readouterr().out) + assert any("'decision' must be one of" in violation for violation in payload["violations"]) + assert (tmp_path / "metadata" / "conversion_plan.json").read_bytes() == edited def test_route_with_plan_path_records_and_edits_the_report(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: @@ -188,7 +263,7 @@ def test_route_errors_when_report_missing_and_no_source_to_trigger_convert( # --------------------------------------------------------------------------- # -# fill-agentic combine. +# fill-agentic. # --------------------------------------------------------------------------- # @@ -211,19 +286,18 @@ def _lfc_pipeline() -> dict[str, Any]: def _record_agentic_plan(tmp_path: Path) -> None: - """Route the parent<->child component agentic and record the plan so combine can bind to it.""" + """Route the parent<->child component agentic and record the plan so fill-agentic can bind to it.""" plan_path = tmp_path / "route_plan.json" plan_path.write_text(json.dumps(_agentic_plan()), encoding="utf-8") assert adapter_cli_main(["route", "--output-dir", str(tmp_path), "--plan-path", str(plan_path)]) == 0 -def _run_combine(tmp_path: Path, members: str, authored: list[dict[str, Any]]) -> int: +def _run_fill(tmp_path: Path, members: str, authored: list[dict[str, Any]]) -> int: pipelines_path = tmp_path / "authored.json" pipelines_path.write_text(json.dumps(authored), encoding="utf-8") return adapter_cli_main( [ "fill-agentic", - "combine", "--output-dir", str(tmp_path), "--members", @@ -234,11 +308,11 @@ def _run_combine(tmp_path: Path, members: str, authored: list[dict[str, Any]]) - ) -def test_fill_agentic_combine_writes_merged_report(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: +def test_fill_agentic_writes_merged_report(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: _setup(tmp_path) _record_agentic_plan(tmp_path) - capsys.readouterr() # drop the route-record output so only the combine JSON remains - code = _run_combine(tmp_path, "child,parent", [_lfc_pipeline()]) + capsys.readouterr() # drop the route output so only the fill JSON remains + code = _run_fill(tmp_path, "child,parent", [_lfc_pipeline()]) assert code == 0 payload = json.loads(capsys.readouterr().out) assert payload["ok"] is True @@ -247,27 +321,25 @@ def test_fill_agentic_combine_writes_merged_report(tmp_path: Path, capsys: pytes assert names == ["orders_lfc", "solo"] -def test_fill_agentic_combine_rejects_dangling_reference(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: +def test_fill_agentic_rejects_dangling_reference(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: _setup(tmp_path) _record_agentic_plan(tmp_path) - capsys.readouterr() # drop the route-record output so only the combine JSON remains + capsys.readouterr() # drop the route output so only the fill JSON remains report_before = (tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_bytes() dangling = _lfc_pipeline() dangling["tasks"][0]["task"] = {"pipeline_task": {"pipeline_id": "${resources.pipelines.ghost.id}"}} - code = _run_combine(tmp_path, "child,parent", [dangling]) + code = _run_fill(tmp_path, "child,parent", [dangling]) assert code == 1 payload = json.loads(capsys.readouterr().out) assert payload["ok"] is False and payload["violations"] assert (tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_bytes() == report_before -def test_fill_agentic_combine_rejects_a_deterministic_member_set( - tmp_path: Path, capsys: pytest.CaptureFixture[str] -) -> None: +def test_fill_agentic_rejects_a_deterministic_member_set(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: _setup(tmp_path) _record_agentic_plan(tmp_path) # component-1 (child, parent) agentic; solo is deterministic - capsys.readouterr() # drop the route-record output so only the combine JSON remains - code = _run_combine(tmp_path, "solo", [_lfc_pipeline()]) + capsys.readouterr() # drop the route output so only the fill JSON remains + code = _run_fill(tmp_path, "solo", [_lfc_pipeline()]) assert code == 1 payload = json.loads(capsys.readouterr().out) assert payload["ok"] is False and payload["error"] @@ -279,7 +351,6 @@ def test_fill_agentic_has_no_no_validate_flag(tmp_path: Path) -> None: adapter_cli_main( [ "fill-agentic", - "combine", "--output-dir", str(tmp_path), "--members", @@ -330,3 +401,55 @@ def fake_run_phase(phase: str, forward: list[str]) -> int: assert triggered and triggered[0][0] == "convert" payload = json.loads(capsys.readouterr().out) assert payload["ok"] is True and payload["edit"]["agentic_pipelines"] == ["child", "parent"] + + +def test_the_old_combine_action_is_gone(tmp_path: Path) -> None: + with pytest.raises(SystemExit) as excinfo: + adapter_cli_main( + ["fill-agentic", "combine", "--output-dir", str(tmp_path), "--members", "solo", "--pipelines-path", "x"] + ) + assert excinfo.value.code == 2 + + +def test_recorded_decisions_survive_a_re_enrich_and_route_re_applies_them( + tmp_path: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch +) -> None: + from flowx.discovery_insights import enrich_inventory + + _setup(tmp_path) + monkeypatch.setattr("sys.stdin", io.StringIO("")) + assert adapter_cli_main(["route", "--output-dir", str(tmp_path)]) == 0 + _edit_recorded_plan(tmp_path, {"component-1": "agentic", "component-2": "deterministic"}) + assert adapter_cli_main(["route", "--output-dir", str(tmp_path)]) == 0 + assert enrich_inventory(tmp_path, insights={"overview": "Orders flow from parent to child."})["ok"] is True + capsys.readouterr() + + assert adapter_cli_main(["route", "--output-dir", str(tmp_path)]) == 0 + + payload = json.loads(capsys.readouterr().out) + assert payload["applied"] is True and payload["pending"] == [] + plan = json.loads((tmp_path / "metadata" / "conversion_plan.json").read_text(encoding="utf-8")) + assert [component["decision"] for component in plan["components"]] == ["agentic", "deterministic"] + assert plan["agentic_insights_sha256"] is not None + + +def test_route_records_the_routing_conversation_and_refuses_a_malformed_one( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + _setup(tmp_path) + plan_path = tmp_path / "plan.json" + conversation = [{"question": "Which databases do the extractors read?", "answer": "SQL Server 2019"}] + plan_path.write_text(json.dumps({**_agentic_plan(), "conversation": conversation}), encoding="utf-8") + assert adapter_cli_main(["route", "--output-dir", str(tmp_path), "--plan-path", str(plan_path)]) == 0 + recorded = json.loads((tmp_path / "metadata" / "conversion_plan.json").read_text(encoding="utf-8")) + assert recorded["conversation"] == conversation + capsys.readouterr() + + plan_path.write_text(json.dumps({**_agentic_plan(), "conversation": [{"question": " "}]}), encoding="utf-8") + + assert adapter_cli_main(["route", "--output-dir", str(tmp_path), "--plan-path", str(plan_path)]) == 1 + violations = json.loads(capsys.readouterr().out)["violations"] + assert violations == [ + "conversation[0]: 'answer' must be a non-empty string", + "conversation[0]: 'question' must be a non-empty string", + ] diff --git a/tests/unit/test_conversion_plan_v2.py b/tests/unit/test_conversion_plan_v2.py index 5ab58e63..c5c8a9a1 100644 --- a/tests/unit/test_conversion_plan_v2.py +++ b/tests/unit/test_conversion_plan_v2.py @@ -1,8 +1,9 @@ -"""The typed conversion plan (schema 2): the library's load, validate and record path. +"""The typed conversion plan (schema 3): the library's load, validate and record path. The plan is bound to what it was decided on: the deterministic inventory fingerprint, the saved source graphs hash the inventory records (H0) and the saved agentic insights hash (H1). Phase 1 -decides whole components; per-node assignments are reserved and must stay empty. +decides whole components; per-node assignments are reserved and must stay empty. A recorded plan +must be complete and well formed to load. """ from __future__ import annotations @@ -16,6 +17,7 @@ from flowx import routing from flowx.bundler.dab_writer import main as package_main from flowx.discovery_inventory import STRATEGY_PROPERTY, build_source_inventory +from flowx.discovery_serde import canonical_sha256 from flowx.ir_serde import pipeline_to_dict from flowx.models.conversion_plan import SCHEMA_VERSION, ComponentPlan, ConversionPlan, NodeAssignment from flowx.models.discovery import CONCEPT_NOTEBOOK, SourceGraph, SourceNode @@ -23,7 +25,13 @@ from flowx.route_agentic import REPORT_FILENAME, WORK_DIRNAME, apply_plan -def _inventory(insights_hash: str | None = "insights-1") -> dict[str, Any]: +def _insights(label: str) -> dict[str, Any]: + """A self-hashed agentic insights document; a different ``label`` gives a different hash.""" + content = {"schema_version": "1", "overview": label} + return {**content, "agentic_insights_sha256": canonical_sha256(content)} + + +def _inventory(insights_label: str | None = "insights-1") -> dict[str, Any]: node = SourceNode( source_id="load", task_key="load", @@ -39,15 +47,22 @@ def _inventory(insights_hash: str | None = "insights-1") -> dict[str, Any]: source_dir="/src", source_graphs_sha256="graphs-1", ) - if insights_hash is not None: - inventory["insights"] = {"schema_version": "1", "agentic_insights_sha256": insights_hash} + if insights_label is not None: + inventory["insights"] = _insights(insights_label) return inventory -def _setup(output_dir: Path, inventory: dict[str, Any]) -> None: +def _write_inventory(output_dir: Path, inventory: dict[str, Any]) -> None: + """Write the inventory and, when it is enriched, the agentic_insights.json it was built with.""" metadata = output_dir / "metadata" metadata.mkdir(parents=True, exist_ok=True) (metadata / "inventory.json").write_text(json.dumps(inventory, indent=2), encoding="utf-8") + if "insights" in inventory: + (metadata / "agentic_insights.json").write_text(json.dumps(inventory["insights"], indent=2), encoding="utf-8") + + +def _setup(output_dir: Path, inventory: dict[str, Any]) -> None: + _write_inventory(output_dir, inventory) work = output_dir / WORK_DIRNAME work.mkdir(parents=True, exist_ok=True) pipeline = Pipeline( @@ -80,7 +95,7 @@ def test_plan_round_trips_including_reserved_assignments(tmp_path: Path) -> None plan.write(tmp_path) assert ConversionPlan.load(tmp_path) == plan - assert json.loads((tmp_path / "metadata" / "conversion_plan.json").read_text())["schema_version"] == "2" + assert json.loads((tmp_path / "metadata" / "conversion_plan.json").read_text())["schema_version"] == "3" def test_a_plan_from_another_schema_version_is_refused(tmp_path: Path) -> None: @@ -102,7 +117,7 @@ def test_recorded_plan_is_bound_to_the_source_graphs_and_insights(tmp_path: Path assert recorded is not None assert recorded.schema_version == SCHEMA_VERSION assert recorded.source_graphs_sha256 == "graphs-1" - assert recorded.agentic_insights_sha256 == "insights-1" + assert recorded.agentic_insights_sha256 == _insights("insights-1")["agentic_insights_sha256"] assert result["source_graphs_sha256"] == "graphs-1" @@ -115,13 +130,40 @@ def test_per_node_assignments_are_reserved_for_phase_2() -> None: assert any("reserved for Phase 2" in violation for violation in violations) -def test_authored_plans_cannot_set_the_bound_hashes() -> None: +def test_an_edited_copy_of_the_recorded_plan_cannot_change_the_bound_hashes(tmp_path: Path) -> None: + _setup(tmp_path, _inventory()) plan = {**_decide("agentic"), "source_graphs_sha256": "x", "agentic_insights_sha256": "y"} - violations = routing.validate_plan(plan, _inventory()) + assert routing.validate_plan(plan, _inventory()) == [] + assert routing.record_plan(tmp_path, plan=plan)["ok"] is True - assert any("source_graphs_sha256" in violation and "library" in violation for violation in violations) - assert any("agentic_insights_sha256" in violation and "library" in violation for violation in violations) + recorded = ConversionPlan.load(tmp_path) + assert recorded is not None + assert recorded.source_graphs_sha256 == "graphs-1" + assert recorded.agentic_insights_sha256 == _insights("insights-1")["agentic_insights_sha256"] + + +def test_an_unknown_key_in_an_authored_plan_is_still_refused() -> None: + violations = routing.validate_plan({**_decide("agentic"), "decisions": {}}, _inventory()) + + assert violations == ["unknown top-level key: 'decisions'"] + + +@pytest.mark.parametrize( + "components", + [None, {"component-1": {}}, [{"members": ["solo"], "decision": "agentic"}], ["component-1"]], + ids=["missing", "an-object", "no-component-id", "not-an-object"], +) +def test_a_malformed_recorded_plan_is_refused_on_load(tmp_path: Path, components: Any) -> None: + metadata = tmp_path / "metadata" + metadata.mkdir() + document: dict[str, Any] = {"schema_version": SCHEMA_VERSION} + if components is not None: + document["components"] = components + (metadata / "conversion_plan.json").write_text(json.dumps(document), encoding="utf-8") + + with pytest.raises(ValueError, match="malformed"): + ConversionPlan.load(tmp_path) def test_package_refuses_a_plan_when_the_insights_changed_after_route( @@ -130,7 +172,7 @@ def test_package_refuses_a_plan_when_the_insights_changed_after_route( """Only the H1 binding catches this: the inventory fingerprint ignores the insights block.""" _setup(tmp_path, _inventory("insights-1")) assert routing.record_plan(tmp_path, plan=_decide("deterministic"))["ok"] is True - (tmp_path / "metadata" / "inventory.json").write_text(json.dumps(_inventory("insights-2")), encoding="utf-8") + _write_inventory(tmp_path, _inventory("insights-2")) code = package_main(["--output-dir", str(tmp_path), "--no-download-workspace-files", "--keep-intermediates"]) @@ -145,7 +187,7 @@ def test_apply_plan_refuses_a_stale_plan(tmp_path: Path) -> None: recorded = ConversionPlan.load(tmp_path) assert recorded is not None report_bytes = (tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_bytes() - (tmp_path / "metadata" / "inventory.json").write_text(json.dumps(_inventory("insights-2")), encoding="utf-8") + _write_inventory(tmp_path, _inventory("insights-2")) with pytest.raises(ValueError, match="different agentic insights"): apply_plan(tmp_path, recorded) diff --git a/tests/unit/test_mcp_route.py b/tests/unit/test_mcp_route.py index 1dfd663a..f962d714 100644 --- a/tests/unit/test_mcp_route.py +++ b/tests/unit/test_mcp_route.py @@ -1,8 +1,8 @@ -"""Tests that the reshaped MCP ``route`` / ``fill_agentic`` commands forward to the adapter CLI. +"""Tests that the MCP ``route`` / ``fill_agentic`` commands forward to the adapter CLI. -``route`` is one command: no plan => the adapter emits the recommendation; a plan (inline or path) -=> the adapter records + edits via ``--plan-path``. ``fill_agentic`` performs the cross-pipeline -combine fill. +``route`` with no plan lets the adapter write the recommendation into ``conversion_plan.json``; a +plan (inline or path) is forwarded via ``--plan-path``. Either way the recorded plan and the routing +review page come back to the caller. ``fill_agentic`` fills a routed-agentic unit. """ from __future__ import annotations @@ -37,22 +37,33 @@ def captured(monkeypatch): def fake_run_adapter(args, **_kwargs): calls.append([str(a) for a in args]) if args and args[0] == "route" and "--plan-path" not in [str(a) for a in args]: - return _Result(json.dumps({"components": [{"component_id": "component-1"}], "default_plan": {}})) + return _Result(json.dumps({"ok": True, "violations": [], "pending": ["component-1"], "applied": False})) return _Result(json.dumps({"ok": True, "violations": [], "components": 1, "edit": {"agentic_pipelines": []}})) monkeypatch.setattr(runner, "run_adapter", fake_run_adapter) return calls -def test_route_without_a_plan_emits_the_recommendation(captured, tmp_path: Path) -> None: +def test_route_without_a_plan_forwards_no_plan_path(captured, tmp_path: Path) -> None: out = server._cmd_route({"output_dir": str(tmp_path)}) assert out["ok"] is True - assert out["result"]["components"][0]["component_id"] == "component-1" argv = captured[0] assert argv[0] == "route" and "--plan-path" not in argv assert "--output-dir" in argv +def test_route_returns_the_recorded_plan_and_review_page_inline(captured, tmp_path: Path) -> None: + metadata = tmp_path / "metadata" + metadata.mkdir() + (metadata / "conversion_plan.json").write_text('{"components": []}', encoding="utf-8") + (metadata / "routing_review.html").write_text("", encoding="utf-8") + (metadata / "inventory.json").write_text("{}", encoding="utf-8") + + out = server._cmd_route({"output_dir": str(tmp_path)}) + + assert sorted(out["bundle"]["files"]) == ["conversion_plan.json", "routing_review.html"] + + def test_route_inline_plan_is_staged_and_forwarded_as_plan_path(captured, tmp_path: Path) -> None: plan = {"components": [{"component_id": "component-1", "members": ["a"], "decision": "agentic"}]} out = server._cmd_route({"output_dir": str(tmp_path), "plan": plan}) @@ -103,7 +114,7 @@ def test_route_registered_in_command_map() -> None: # --------------------------------------------------------------------------- # -# fill_agentic (combine). +# fill_agentic. # --------------------------------------------------------------------------- # @@ -125,7 +136,7 @@ def test_fill_agentic_inline_pipelines_are_staged(captured_fill, tmp_path: Path) ) assert out["ok"] is True argv = captured_fill[0] - assert argv[0] == "fill-agentic" and argv[1] == "combine" + assert argv[0] == "fill-agentic" and argv[1] == "--output-dir" assert argv[argv.index("--members") + 1] == "parent,child" assert "--pipelines-path" in argv diff --git a/tests/unit/test_package_routing_preflight.py b/tests/unit/test_package_routing_preflight.py index c287fe01..001fcad6 100644 --- a/tests/unit/test_package_routing_preflight.py +++ b/tests/unit/test_package_routing_preflight.py @@ -21,6 +21,7 @@ from flowx.bundler.dab_writer import main as package_main from flowx.discovery_insights import enrich_inventory from flowx.discovery_inventory import STRATEGY_PROPERTY, build_source_inventory +from flowx.discovery_serde import canonical_sha256 from flowx.ir_serde import pipeline_to_dict from flowx.models.discovery import CONCEPT_NOTEBOOK, SourceGraph, SourceNode from flowx.models.ir import NotebookActivity, Pipeline @@ -125,7 +126,7 @@ def test_package_with_a_deterministic_plan_and_no_record_passes(tmp_path: Path) "members": ["solo"], "decision": "deterministic", "outcome": "deterministic", - "combine_sha256": None, + "output_sha256": None, } ] assert audit["baseline_report_sha256"] is None @@ -156,7 +157,6 @@ def test_route_combine_modify_then_package_keeps_the_routing_record(tmp_path: Pa adapter_main( [ "fill-agentic", - "combine", "--output-dir", str(tmp_path), "--members", @@ -178,7 +178,7 @@ def test_route_combine_modify_then_package_keeps_the_routing_record(tmp_path: Pa assert component_audit["members"] == ["solo"] assert component_audit["decision"] == "agentic" assert component_audit["outcome"] == "agentic-applied" - assert "combine_sha256" in component_audit + assert "output_sha256" in component_audit assert "fingerprint" in component_audit assert audit["baseline_report_sha256"] == baseline_sha256 assert audit["translation_report_sha256"] == hashlib.sha256(stamped_path.read_bytes()).hexdigest() @@ -199,7 +199,7 @@ def test_route_after_modify_leaves_the_stamped_report_and_package_asks_to_re_run stamped_path = tmp_path / WORK_DIRNAME / "translation_report.stamped.json" assert adapter_main(route) == 0 - combine = ["fill-agentic", "combine", "--output-dir", str(tmp_path), "--members", "solo"] + combine = ["fill-agentic", "--output-dir", str(tmp_path), "--members", "solo"] assert adapter_main([*combine, "--pipelines-path", str(authored_path)]) == 0 assert adapter_main(["modify", str(report_path), "--output-dir", str(tmp_path)]) == 0 assert enrich_inventory(tmp_path, insights={"overview": "Solo loads one table."})["ok"] is True @@ -243,33 +243,51 @@ def _route(decision: str) -> int: assert _package(tmp_path) == 0 +def test_package_refuses_an_agentic_component_with_no_agent_output( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + report_path = _setup(tmp_path, _inventory("agentic")) + _record(tmp_path, "agentic") + plan = json.loads((tmp_path / "metadata" / "conversion_plan.json").read_text(encoding="utf-8")) + apply_plan_to_report(tmp_path, plan) + assert routing_record(json.loads(report_path.read_text(encoding="utf-8"))) is not None + + assert _package(tmp_path) == 1 + + error = capsys.readouterr().err + assert "component 'component-1' (solo) is routed agentic but has no agent output yet" in error + assert not (tmp_path / "databricks.yml").exists() + assert not (tmp_path / "solo" / "databricks.yml").exists() + _fill_solo(tmp_path) + assert _package(tmp_path) == 0 + + def test_package_with_a_current_plan_writes_an_audit_with_hashes(tmp_path: Path) -> None: report_path = _setup(tmp_path, _inventory("agentic")) baseline_sha256 = hashlib.sha256(report_path.read_bytes()).hexdigest() _record(tmp_path, "agentic") plan = json.loads((tmp_path / "metadata" / "conversion_plan.json").read_text(encoding="utf-8")) apply_plan_to_report(tmp_path, plan) + _fill_solo(tmp_path) record = routing_record(json.loads(report_path.read_text(encoding="utf-8"))) assert record is not None - assert _package(tmp_path) in (0, 1) # 1 only signals bundle-invariant warnings, not the preflight + assert _package(tmp_path) == 0 assert (tmp_path / "databricks.yml").exists() audit = json.loads((tmp_path / "metadata" / "route_audit.json").read_text(encoding="utf-8")) assert audit["inventory_sha256"] == audit["recorded_against_inventory_sha256"] == plan["inventory_sha256"] - assert ( - audit["conversion_plan_sha256"] - == hashlib.sha256((tmp_path / "metadata" / "conversion_plan.json").read_bytes()).hexdigest() - ) + assert audit["conversion_plan_sha256"] == record["conversion_plan_sha256"] == canonical_sha256(plan) assert audit["baseline_report_sha256"] == baseline_sha256 assert audit["baseline_gaps_sha256"] == record["baseline_gaps_sha256"] assert audit["translation_report_sha256"] == hashlib.sha256(report_path.read_bytes()).hexdigest() + assert "gaps_introduced" not in audit and "gaps_count" not in audit (component,) = audit["components"] recorded = record["components"][component["component_id"]] assert component["decision"] == "agentic" - assert component["outcome"] == recorded["outcome"] == "agentic-not-viable" + assert component["outcome"] == recorded["outcome"] == "agentic-applied" assert component["fingerprint"] == recorded["fingerprint"] - assert component["combine_sha256"] is None + assert component["output_sha256"] == recorded["output_sha256"] is not None def test_package_refuses_tampered_baseline(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: @@ -306,7 +324,7 @@ def test_package_refuses_stale_unstamped_report(tmp_path: Path, capsys: pytest.C def test_package_refuses_edited_combine_store(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - """Package refuses when agentic_combines.json pipelines were edited after the combine.""" + """Package refuses when agentic_conversion.json pipelines were edited after the combine.""" _setup(tmp_path, _inventory("agentic")) plan_path = tmp_path / "plan.json" plan_path.write_text( @@ -323,7 +341,6 @@ def test_package_refuses_edited_combine_store(tmp_path: Path, capsys: pytest.Cap adapter_main( [ "fill-agentic", - "combine", "--output-dir", str(tmp_path), "--members", @@ -335,16 +352,19 @@ def test_package_refuses_edited_combine_store(tmp_path: Path, capsys: pytest.Cap == 0 ) - combines_path = tmp_path / "metadata" / "agentic_combines.json" + combines_path = tmp_path / "metadata" / "agentic_conversion.json" combines = json.loads(combines_path.read_text(encoding="utf-8")) - combines["component-1"]["pipelines"][0]["name"] = "tampered_name" + combines["components"]["component-1"]["pipelines"][0]["name"] = "tampered_name" combines_path.write_text(json.dumps(combines, indent=2), encoding="utf-8") assert _package(tmp_path) == 1 - assert "the combine store has been edited (component-1); re-run fill-agentic combine" in capsys.readouterr().err + assert ( + "metadata/agentic_conversion.json has been edited (component-1); re-run fill-agentic for that component" + in capsys.readouterr().err + ) assert not (tmp_path / "databricks.yml").exists() - combine = ["fill-agentic", "combine", "--output-dir", str(tmp_path), "--members", "solo"] + combine = ["fill-agentic", "--output-dir", str(tmp_path), "--members", "solo"] assert adapter_main([*combine, "--pipelines-path", str(authored_path)]) == 0 assert _package(tmp_path) == 0 @@ -367,7 +387,6 @@ def test_package_refuses_stale_stamped_report(tmp_path: Path, capsys: pytest.Cap adapter_main( [ "fill-agentic", - "combine", "--output-dir", str(tmp_path), "--members", @@ -425,10 +444,19 @@ def _route_solo_agentic_and_modify(output_dir: Path, report_path: Path, *modify_ ) route = ["route", "--output-dir", str(output_dir), "--plan-path", str(plan_path)] assert adapter_main(route) == 0 + _fill_solo(output_dir) assert adapter_main(["modify", str(report_path), "--output-dir", str(output_dir), *modify_args]) == 0 return route +def _fill_solo(output_dir: Path) -> None: + """Fill the routed-agentic 'solo' component with the authored pipeline through the CLI.""" + authored_path = output_dir / "authored.json" + authored_path.write_text(json.dumps([_authored_pipeline()]), encoding="utf-8") + fill = ["fill-agentic", "--output-dir", str(output_dir), "--members", "solo", "--pipelines-path"] + assert adapter_main([*fill, str(authored_path)]) == 0 + + def test_package_asks_to_re_run_modify_after_a_re_convert_and_re_route( tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: @@ -460,3 +488,63 @@ def test_package_checks_a_modify_out_copy_by_its_routing_record(tmp_path: Path) assert exit_code == 0 assert (tmp_path / "databricks.yml").exists() + + +def test_package_refuses_a_routed_report_whose_plan_is_missing( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + report_path = _setup(tmp_path, _inventory("agentic")) + _route_solo_agentic_and_modify(tmp_path, report_path) + (tmp_path / "metadata" / "conversion_plan.json").unlink() + capsys.readouterr() + + assert _package(tmp_path) == 1 + + assert "carries a routing record) but metadata/conversion_plan.json is missing" in capsys.readouterr().err + assert not (tmp_path / "databricks.yml").exists() + + +def test_package_revalidates_a_recorded_plan_that_lost_its_component( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + _setup(tmp_path, _inventory()) + _record(tmp_path, "deterministic") + plan_path = tmp_path / "metadata" / "conversion_plan.json" + plan = json.loads(plan_path.read_text(encoding="utf-8")) + plan["components"] = [] + plan_path.write_text(json.dumps(plan), encoding="utf-8") + + assert _package(tmp_path) == 1 + + assert "conversion_plan.json is not a valid decision" in capsys.readouterr().err + assert not (tmp_path / "databricks.yml").exists() + + +def test_package_refuses_a_plan_with_a_pending_decision(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + _setup(tmp_path, _inventory()) + plan = {"components": [{"component_id": "component-1", "members": ["solo"], "decision": None}]} + assert routing.record_plan(tmp_path, plan=plan)["ok"] is True + + assert _package(tmp_path) == 1 + + assert "components ['component-1'] have no decision yet" in capsys.readouterr().err + assert not (tmp_path / "databricks.yml").exists() + + +def test_package_refuses_a_configured_report_of_an_unfilled_agentic_component( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + report_path = _setup(tmp_path, _inventory("agentic")) + plan_path = tmp_path / "plan.json" + plan_path.write_text( + json.dumps({"components": [{"component_id": "component-1", "members": ["solo"], "decision": "agentic"}]}), + encoding="utf-8", + ) + assert adapter_main(["route", "--output-dir", str(tmp_path), "--plan-path", str(plan_path)]) == 0 + assert adapter_main(["modify", str(report_path), "--output-dir", str(tmp_path)]) == 0 + capsys.readouterr() + + assert _package(tmp_path) == 1 + + assert "is routed agentic but has no agent output yet" in capsys.readouterr().err + assert not (tmp_path / "databricks.yml").exists() diff --git a/tests/unit/test_route_agentic_flow.py b/tests/unit/test_route_agentic_flow.py index 3b19da3f..91c3f151 100644 --- a/tests/unit/test_route_agentic_flow.py +++ b/tests/unit/test_route_agentic_flow.py @@ -42,11 +42,11 @@ WORK_DIRNAME, agentic_pipeline_names, alter_report, - apply_combine_fill, + apply_agentic_output, apply_plan, apply_plan_to_report, - combine_group_fill, prompt_for_decisions, + replace_unit_pipelines, routing_record, validate_report_structurally, ) @@ -469,7 +469,7 @@ def test_merge_still_fills_convert_gaps_in_a_deterministic_pipeline(tmp_path: Pa assert _record_outcomes(report_path) == {"component-1": "agentic-not-viable", "component-2": "deterministic"} -def test_a_merged_top_level_gap_survives_a_re_route_and_package_accepts_it( +def test_a_merged_top_level_gap_is_stored_apart_and_survives_a_re_route_with_the_baseline_unchanged( tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: report = _report_with_child_gap() @@ -477,22 +477,66 @@ def test_a_merged_top_level_gap_survives_a_re_route_and_package_accepts_it( pipeline["tags"] = {"source": "adf"} _route_parent_agentic(tmp_path, report) report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME + baseline_path = tmp_path / WORK_DIRNAME / "route_baseline" / REPORT_FILENAME + baseline_before = baseline_path.read_bytes() + record_before = routing_record(json.loads(report_path.read_text(encoding="utf-8"))) results_dir = tmp_path / "agentic_results" _write_merge_result(results_dir, "child", "Load", "load") assert merge_agentic_results(report_path, results_dir) == (1, 0) + assert baseline_path.read_bytes() == baseline_before + assert routing_record(json.loads(report_path.read_text(encoding="utf-8"))) == record_before + stored = json.loads((tmp_path / "metadata" / "agentic_conversion.json").read_text(encoding="utf-8")) + assert [(fill["pipeline"], fill["activity_name"]) for fill in stored["gap_fills"]] == [("child", "Load")] + recorded = ConversionPlan.load(tmp_path) assert recorded is not None assert apply_plan(tmp_path, recorded)["altered"] is False - for path in (report_path, tmp_path / WORK_DIRNAME / "route_baseline" / REPORT_FILENAME): - child = next(p for p in json.loads(path.read_text(encoding="utf-8"))["pipelines"] if p["name"] == "child") - assert child["tasks"][0]["type"] == "NotebookActivity" + child = next(p for p in json.loads(report_path.read_text(encoding="utf-8"))["pipelines"] if p["name"] == "child") + assert child["tasks"][0]["type"] == "NotebookActivity" + assert baseline_path.read_bytes() == baseline_before + _route_two_components(tmp_path, parent="deterministic", child="deterministic") capsys.readouterr() package_main(["--output-dir", str(tmp_path), "--no-download-workspace-files", "--keep-intermediates"]) assert "preflight failed" not in capsys.readouterr().err assert (tmp_path / "child" / "databricks.yml").exists() +def test_a_gap_fill_survives_switching_every_component_back_to_deterministic(tmp_path: Path) -> None: + _route_parent_agentic(tmp_path, _report_with_child_gap()) + report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME + results_dir = tmp_path / "agentic_results" + _write_merge_result(results_dir, "child", "Load", "load") + assert merge_agentic_results(report_path, results_dir) == (1, 0) + + _route_two_components(tmp_path, parent="deterministic", child="deterministic") + + report = json.loads(report_path.read_text(encoding="utf-8")) + child = next(p for p in report["pipelines"] if p["name"] == "child") + assert child["tasks"][0]["type"] == "NotebookActivity" + assert _record_outcomes(report_path) == {"component-1": "deterministic", "component-2": "deterministic"} + + +def test_a_re_convert_drops_gap_fills_merged_against_the_old_baseline(tmp_path: Path) -> None: + _route_parent_agentic(tmp_path, _report_with_child_gap()) + report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME + results_dir = tmp_path / "agentic_results" + _write_merge_result(results_dir, "child", "Load", "load") + assert merge_agentic_results(report_path, results_dir) == (1, 0) + reconverted = _report_with_child_gap() + reconverted["pipelines"][0]["tasks"][0]["task_key"] = "extract_again" + _write_work(tmp_path, reconverted, gaps=[]) + + recorded = ConversionPlan.load(tmp_path) + assert recorded is not None + assert apply_plan(tmp_path, recorded)["dropped_gap_fills"] == 1 + + stored = json.loads((tmp_path / "metadata" / "agentic_conversion.json").read_text(encoding="utf-8")) + assert stored["gap_fills"] == [] + child = next(p for p in json.loads(report_path.read_text(encoding="utf-8"))["pipelines"] if p["name"] == "child") + assert child["tasks"][0]["type"] == "PlaceholderActivity" + + def test_a_merge_written_to_a_separate_copy_leaves_the_baseline_and_live_report_alone(tmp_path: Path) -> None: _route_parent_agentic(tmp_path, _report_with_child_gap()) report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME @@ -587,7 +631,7 @@ def test_route_takes_a_fresh_convert_as_the_new_baseline(tmp_path: Path) -> None def test_reroute_under_the_same_decisions_keeps_fills_and_updates_the_plan_hash(tmp_path: Path) -> None: _setup_routed_agentic(tmp_path, decision="agentic") - assert apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()])["ok"] is True + assert apply_agentic_output(tmp_path, ["parent", "child"], [_lfc_pipeline()])["ok"] is True report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME first_record = routing_record(json.loads(report_path.read_text(encoding="utf-8"))) assert first_record is not None @@ -621,14 +665,14 @@ def test_reroute_under_the_same_decisions_keeps_fills_and_updates_the_plan_hash( # --------------------------------------------------------------------------- # -def test_combine_group_fill_replaces_group_pipelines_with_authored_ones() -> None: +def test_replace_unit_pipelines_replaces_group_pipelines_with_authored_ones() -> None: report = _report_two_pipelines() - merged = combine_group_fill(report, {"parent", "child"}, [_lfc_pipeline()]) + merged = replace_unit_pipelines(report, {"parent", "child"}, [_lfc_pipeline()]) names = [pipeline["name"] for pipeline in merged["pipelines"]] assert names == ["orders_lfc"] -def test_combine_group_fill_keeps_pipelines_outside_the_group() -> None: +def test_replace_unit_pipelines_keeps_pipelines_outside_the_group() -> None: report = { "pipelines": [ {"name": "parent", "tasks": [_copy_task("Extract", "extract")]}, @@ -636,14 +680,14 @@ def test_combine_group_fill_keeps_pipelines_outside_the_group() -> None: {"name": "unrelated", "tasks": [_notebook_task("Keep", "keep")]}, ] } - merged = combine_group_fill(report, {"parent", "child"}, [_lfc_pipeline()]) + merged = replace_unit_pipelines(report, {"parent", "child"}, [_lfc_pipeline()]) names = sorted(pipeline["name"] for pipeline in merged["pipelines"]) assert names == ["orders_lfc", "unrelated"] def test_combine_fill_of_two_pipelines_into_one_lfc_passes_structural_validation() -> None: report = _report_two_pipelines() - merged = combine_group_fill(report, {"parent", "child"}, [_lfc_pipeline()]) + merged = replace_unit_pipelines(report, {"parent", "child"}, [_lfc_pipeline()]) result = validate_report_structurally(merged) assert result.ok, [f"{finding.code}: {finding.message}" for finding in result.findings] @@ -652,28 +696,28 @@ def test_combine_fill_with_a_dangling_pipeline_reference_is_caught() -> None: dangling = _lfc_pipeline() # Point the pipeline_task at a resource that is never declared in this activity's resources list. dangling["tasks"][0]["task"] = {"pipeline_task": {"pipeline_id": "${resources.pipelines.ghost.id}"}} - merged = combine_group_fill(_report_two_pipelines(), {"parent", "child"}, [dangling]) + merged = replace_unit_pipelines(_report_two_pipelines(), {"parent", "child"}, [dangling]) result = validate_report_structurally(merged) assert not result.ok assert any(finding.code == "dangling_pipeline_reference" for finding in result.violations) -def test_apply_combine_fill_writes_merged_report_when_valid(tmp_path: Path) -> None: +def test_apply_agentic_output_writes_merged_report_when_valid(tmp_path: Path) -> None: _setup_routed_agentic(tmp_path, decision="agentic") - result = apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()]) + result = apply_agentic_output(tmp_path, ["parent", "child"], [_lfc_pipeline()]) assert result["ok"] is True assert result["component_id"] == "component-1" report = json.loads((tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_text(encoding="utf-8")) assert [pipeline["name"] for pipeline in report["pipelines"]] == ["orders_lfc"] -def test_apply_combine_fill_rejects_and_does_not_write_on_dangling_reference(tmp_path: Path) -> None: +def test_apply_agentic_output_rejects_and_does_not_write_on_dangling_reference(tmp_path: Path) -> None: _setup_routed_agentic(tmp_path, decision="agentic") report_before = (tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_bytes() dangling = _lfc_pipeline() dangling["tasks"][0]["task"] = {"pipeline_task": {"pipeline_id": "${resources.pipelines.ghost.id}"}} - result = apply_combine_fill(tmp_path, ["parent", "child"], [dangling]) + result = apply_agentic_output(tmp_path, ["parent", "child"], [dangling]) assert result["ok"] is False assert any("dangling_pipeline_reference" in violation for violation in result["violations"]) # The report on disk is untouched when validation fails. @@ -687,7 +731,7 @@ def test_combine_rejects_an_authored_pipeline_missing_the_source_tag(tmp_path: P untagged = _lfc_pipeline() del untagged["tags"] - result = apply_combine_fill(tmp_path, ["parent", "child"], [untagged]) + result = apply_agentic_output(tmp_path, ["parent", "child"], [untagged]) assert result["ok"] is False assert any("tags.source" in violation and "adf" in violation for violation in result["violations"]) @@ -701,7 +745,7 @@ def test_combine_rejects_an_authored_pipeline_with_wrong_source_tag(tmp_path: Pa mistagged = _lfc_pipeline() mistagged["tags"] = {"source": "airflow"} - result = apply_combine_fill(tmp_path, ["parent", "child"], [mistagged]) + result = apply_agentic_output(tmp_path, ["parent", "child"], [mistagged]) assert result["ok"] is False assert any("tags.source" in violation for violation in result["violations"]) @@ -711,15 +755,15 @@ def test_combine_is_idempotent_running_twice_yields_no_duplicate(tmp_path: Path) """FIX 3: re-running combine with the same members + authored pipeline does not duplicate it.""" _setup_routed_agentic(tmp_path, decision="agentic") - first = apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()]) + first = apply_agentic_output(tmp_path, ["parent", "child"], [_lfc_pipeline()]) assert first["ok"] is True - assert first["already_combined"] is False + assert first["already_applied"] is False # The recorded plan still lists {parent, child}, so the plan/membership check passes again; the # report, however, now contains only the authored pipeline. A naive re-run would re-append it. - second = apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()]) + second = apply_agentic_output(tmp_path, ["parent", "child"], [_lfc_pipeline()]) assert second["ok"] is True - assert second["already_combined"] is True + assert second["already_applied"] is True report = json.loads((tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_text(encoding="utf-8")) names = [pipeline["name"] for pipeline in report["pipelines"]] @@ -729,16 +773,16 @@ def test_combine_is_idempotent_running_twice_yields_no_duplicate(tmp_path: Path) def test_a_different_combine_on_an_applied_component_replaces_and_rebuilds(tmp_path: Path) -> None: """A second combine with a different hash replaces the stored entry and rebuilds.""" _setup_routed_agentic(tmp_path, decision="agentic") - first = apply_combine_fill(tmp_path, ["parent", "child"], [_named_lfc_pipeline("orders_lfc")]) - assert first["ok"] is True and first["already_combined"] is False + first = apply_agentic_output(tmp_path, ["parent", "child"], [_named_lfc_pipeline("orders_lfc")]) + assert first["ok"] is True and first["already_applied"] is False report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME first_names = [p["name"] for p in json.loads(report_path.read_text(encoding="utf-8"))["pipelines"]] assert first_names == ["orders_lfc"] - second = apply_combine_fill(tmp_path, ["parent", "child"], [_named_lfc_pipeline("orders_lfc_v2")]) + second = apply_agentic_output(tmp_path, ["parent", "child"], [_named_lfc_pipeline("orders_lfc_v2")]) assert second["ok"] is True - assert second["already_combined"] is False + assert second["already_applied"] is False second_names = [p["name"] for p in json.loads(report_path.read_text(encoding="utf-8"))["pipelines"]] assert second_names == ["orders_lfc_v2"] @@ -752,12 +796,12 @@ def test_combine_idempotent_when_authored_name_collides_with_a_former_member(tmp """ _setup_routed_agentic(tmp_path, decision="agentic") - first = apply_combine_fill(tmp_path, ["parent", "child"], [_named_lfc_pipeline("parent")]) - assert first["ok"] is True and first["already_combined"] is False + first = apply_agentic_output(tmp_path, ["parent", "child"], [_named_lfc_pipeline("parent")]) + assert first["ok"] is True and first["already_applied"] is False - second = apply_combine_fill(tmp_path, ["parent", "child"], [_named_lfc_pipeline("parent")]) + second = apply_agentic_output(tmp_path, ["parent", "child"], [_named_lfc_pipeline("parent")]) assert second["ok"] is True - assert second["already_combined"] is True + assert second["already_applied"] is True report = json.loads((tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_text(encoding="utf-8")) names = [pipeline["name"] for pipeline in report["pipelines"]] @@ -771,7 +815,7 @@ def test_combine_idempotent_when_authored_name_collides_with_a_former_member(tmp def test_combine_rejects_a_deterministic_component(tmp_path: Path) -> None: _setup_routed_agentic(tmp_path, decision="deterministic") - result = apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()]) + result = apply_agentic_output(tmp_path, ["parent", "child"], [_lfc_pipeline()]) assert result["ok"] is False assert "not agentic" in result["error"] # A deterministic component is never swapped out. @@ -781,21 +825,21 @@ def test_combine_rejects_a_deterministic_component(tmp_path: Path) -> None: def test_combine_rejects_a_partial_member_set(tmp_path: Path) -> None: _setup_routed_agentic(tmp_path, decision="agentic") - result = apply_combine_fill(tmp_path, ["parent"], [_lfc_pipeline()]) + result = apply_agentic_output(tmp_path, ["parent"], [_lfc_pipeline()]) assert result["ok"] is False assert "do not exactly match" in result["error"] def test_combine_rejects_a_superset_member_set(tmp_path: Path) -> None: _setup_routed_agentic(tmp_path, decision="agentic") - result = apply_combine_fill(tmp_path, ["parent", "child", "extra"], [_lfc_pipeline()]) + result = apply_agentic_output(tmp_path, ["parent", "child", "extra"], [_lfc_pipeline()]) assert result["ok"] is False assert "do not exactly match" in result["error"] def test_combine_rejects_a_typoed_member(tmp_path: Path) -> None: _setup_routed_agentic(tmp_path, decision="agentic") - result = apply_combine_fill(tmp_path, ["parent", "chld"], [_lfc_pipeline()]) + result = apply_agentic_output(tmp_path, ["parent", "chld"], [_lfc_pipeline()]) assert result["ok"] is False assert "do not exactly match" in result["error"] @@ -808,7 +852,7 @@ def test_combine_rejects_a_stale_fingerprint_plan(tmp_path: Path) -> None: inventory["pipelines"].append({"name": "late_addition", "activities": [], "motifs": []}) inventory_path.write_text(json.dumps(inventory, indent=2), encoding="utf-8") - result = apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()]) + result = apply_agentic_output(tmp_path, ["parent", "child"], [_lfc_pipeline()]) assert result["ok"] is False assert "stale" in result["error"] @@ -818,7 +862,7 @@ def test_combine_marks_the_component_applied(tmp_path: Path) -> None: report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME assert _record_outcomes(report_path) == {"component-1": "agentic-not-viable"} - assert apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()])["ok"] is True + assert apply_agentic_output(tmp_path, ["parent", "child"], [_lfc_pipeline()])["ok"] is True assert _record_outcomes(report_path) == {"component-1": "agentic-applied"} @@ -839,7 +883,7 @@ def _write_merge_result(results_dir: Path, pipeline: str, activity_name: str, ta def test_merge_after_a_combine_is_refused(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: _setup_routed_agentic(tmp_path, decision="agentic") - assert apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()])["ok"] is True + assert apply_agentic_output(tmp_path, ["parent", "child"], [_lfc_pipeline()])["ok"] is True report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME combined_report = report_path.read_bytes() results_dir = tmp_path / "agentic_results" @@ -865,12 +909,12 @@ def _colliding_authored_pipelines() -> list[dict[str, Any]]: def test_combine_rerun_is_idempotent_when_authored_names_reuse_every_member(tmp_path: Path) -> None: _setup_routed_agentic(tmp_path, decision="agentic") - first = apply_combine_fill(tmp_path, ["parent", "child"], _colliding_authored_pipelines()) - assert first["ok"] is True and first["already_combined"] is False - second = apply_combine_fill(tmp_path, ["parent", "child"], _colliding_authored_pipelines()) + first = apply_agentic_output(tmp_path, ["parent", "child"], _colliding_authored_pipelines()) + assert first["ok"] is True and first["already_applied"] is False + second = apply_agentic_output(tmp_path, ["parent", "child"], _colliding_authored_pipelines()) assert second["ok"] is True - assert second["already_combined"] is True + assert second["already_applied"] is True report = json.loads((tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_text(encoding="utf-8")) assert [pipeline["name"] for pipeline in report["pipelines"]] == ["parent", "child"] @@ -883,7 +927,7 @@ def test_merge_after_a_combine_reusing_every_member_name_is_refused( tmp_path: Path, pipeline: str | None, activity_name: str ) -> None: _setup_routed_agentic(tmp_path, decision="agentic") - assert apply_combine_fill(tmp_path, ["parent", "child"], _colliding_authored_pipelines())["ok"] is True + assert apply_agentic_output(tmp_path, ["parent", "child"], _colliding_authored_pipelines())["ok"] is True report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME combined_report = report_path.read_bytes() results_dir = tmp_path / "agentic_results" @@ -900,7 +944,7 @@ def test_merge_after_a_combine_reusing_every_member_name_is_refused( def test_merge_into_an_authored_combine_pipeline_is_refused(tmp_path: Path) -> None: _setup_routed_agentic(tmp_path, decision="agentic") - assert apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()])["ok"] is True + assert apply_agentic_output(tmp_path, ["parent", "child"], [_lfc_pipeline()])["ok"] is True results_dir = tmp_path / "agentic_results" _write_merge_result(results_dir, "orders_lfc", "Ingest orders", "ingest_orders") @@ -912,7 +956,7 @@ def test_combine_requires_route_to_have_applied_the_plan(tmp_path: Path) -> None _setup_routed_agentic(tmp_path, decision="agentic") _write_work(tmp_path, _report_two_pipelines()) # convert ran again, so the report has no record - result = apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()]) + result = apply_agentic_output(tmp_path, ["parent", "child"], [_lfc_pipeline()]) assert result["ok"] is False assert "routing record" in result["error"] @@ -920,22 +964,22 @@ def test_combine_requires_route_to_have_applied_the_plan(tmp_path: Path) -> None def test_combine_requires_a_recorded_plan(tmp_path: Path) -> None: _write_work(tmp_path, _report_two_pipelines()) # report only; no plan recorded - result = apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()]) + result = apply_agentic_output(tmp_path, ["parent", "child"], [_lfc_pipeline()]) assert result["ok"] is False assert "conversion_plan.json" in result["error"] def test_an_invalid_replacement_combine_is_refused_and_keeps_the_applied_one(tmp_path: Path) -> None: _setup_routed_agentic(tmp_path, decision="agentic") - assert apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()])["ok"] is True + assert apply_agentic_output(tmp_path, ["parent", "child"], [_lfc_pipeline()])["ok"] is True report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME - combines_path = tmp_path / "metadata" / "agentic_combines.json" + combines_path = tmp_path / "metadata" / "agentic_conversion.json" report_before = report_path.read_bytes() combines_before = combines_path.read_bytes() dangling = _named_lfc_pipeline("orders_lfc_v2") dangling["tasks"][0]["task"] = {"pipeline_task": {"pipeline_id": "${resources.pipelines.ghost.id}"}} - result = apply_combine_fill(tmp_path, ["parent", "child"], [dangling]) + result = apply_agentic_output(tmp_path, ["parent", "child"], [dangling]) assert result["ok"] is False assert any("dangling_pipeline_reference" in violation for violation in result["violations"]) @@ -945,38 +989,38 @@ def test_an_invalid_replacement_combine_is_refused_and_keeps_the_applied_one(tmp def test_combine_stores_sorted_members_and_hashes_every_pipeline_field(tmp_path: Path) -> None: _setup_routed_agentic(tmp_path, decision="agentic") - assert apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()])["ok"] is True + assert apply_agentic_output(tmp_path, ["parent", "child"], [_lfc_pipeline()])["ok"] is True described = _lfc_pipeline() described["description"] = "Ingest orders through Lakeflow Connect" - result = apply_combine_fill(tmp_path, ["parent", "child"], [described]) + result = apply_agentic_output(tmp_path, ["parent", "child"], [described]) - assert result["ok"] is True and result["already_combined"] is False - combines = json.loads((tmp_path / "metadata" / "agentic_combines.json").read_text(encoding="utf-8")) - assert combines["component-1"]["members"] == ["child", "parent"] - assert combines["component-1"]["pipelines"] == [described] + assert result["ok"] is True and result["already_applied"] is False + combines = json.loads((tmp_path / "metadata" / "agentic_conversion.json").read_text(encoding="utf-8")) + assert combines["components"]["component-1"]["members"] == ["child", "parent"] + assert combines["components"]["component-1"]["pipelines"] == [described] report = json.loads((tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_text(encoding="utf-8")) assert report["pipelines"] == [described] def test_combine_replaces_a_hand_edited_store_entry(tmp_path: Path) -> None: _setup_routed_agentic(tmp_path, decision="agentic") - assert apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()])["ok"] is True - combines_path = tmp_path / "metadata" / "agentic_combines.json" + assert apply_agentic_output(tmp_path, ["parent", "child"], [_lfc_pipeline()])["ok"] is True + combines_path = tmp_path / "metadata" / "agentic_conversion.json" combined = combines_path.read_bytes() edited = json.loads(combined) - edited["component-1"]["pipelines"][0]["tasks"] = [] + edited["components"]["component-1"]["pipelines"][0]["tasks"] = [] combines_path.write_text(json.dumps(edited, indent=2), encoding="utf-8") - result = apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()]) + result = apply_agentic_output(tmp_path, ["parent", "child"], [_lfc_pipeline()]) - assert result["ok"] is True and result["already_combined"] is False + assert result["ok"] is True and result["already_applied"] is False assert combines_path.read_bytes() == combined def test_a_combine_routed_back_to_deterministic_records_no_combine_hash(tmp_path: Path) -> None: _route_parent_agentic(tmp_path, _report_two_pipelines(), child="agentic") - assert apply_combine_fill(tmp_path, ["parent"], [_named_lfc_pipeline("parent_lfc")])["ok"] is True + assert apply_agentic_output(tmp_path, ["parent"], [_named_lfc_pipeline("parent_lfc")])["ok"] is True _route_two_components(tmp_path, parent="deterministic", child="agentic") @@ -985,7 +1029,7 @@ def test_a_combine_routed_back_to_deterministic_records_no_combine_hash(tmp_path assert record is not None (parent_entry,) = [entry for entry in record["components"].values() if entry["members"] == ["parent"]] assert parent_entry["outcome"] == "deterministic" - assert parent_entry["combine_sha256"] is None + assert parent_entry["output_sha256"] is None assert [pipeline["name"] for pipeline in report["pipelines"]] == ["parent", "child"] @@ -994,13 +1038,13 @@ def test_combine_refuses_a_name_that_another_components_combine_would_drop(tmp_p child_lfc = _named_lfc_pipeline("child_lfc") child_lfc["tasks"][0]["resources"][0]["resource_key"] = "child_ingestion" child_lfc["tasks"][0]["task"] = {"pipeline_task": {"pipeline_id": "${resources.pipelines.child_ingestion.id}"}} - assert apply_combine_fill(tmp_path, ["child"], [child_lfc])["ok"] is True + assert apply_agentic_output(tmp_path, ["child"], [child_lfc])["ok"] is True report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME - combines_path = tmp_path / "metadata" / "agentic_combines.json" + combines_path = tmp_path / "metadata" / "agentic_conversion.json" report_before = report_path.read_bytes() combines_before = combines_path.read_bytes() - result = apply_combine_fill(tmp_path, ["parent"], [_named_lfc_pipeline("child")]) + result = apply_agentic_output(tmp_path, ["parent"], [_named_lfc_pipeline("child")]) assert result["ok"] is False assert "clash" in result["error"] @@ -1024,28 +1068,28 @@ def test_combine_refuses_authored_names_that_clash(tmp_path: Path, authored_name "pipeline_task": {"pipeline_id": f"${{resources.pipelines.orders_ingestion_{index}.id}}"} } - result = apply_combine_fill(tmp_path, ["parent"], authored) + result = apply_agentic_output(tmp_path, ["parent"], authored) assert result["ok"] is False assert "clash" in result["error"] assert report_path.read_bytes() == report_before - assert not (tmp_path / "metadata" / "agentic_combines.json").exists() + assert not (tmp_path / "metadata" / "agentic_conversion.json").exists() -def test_combine_same_hash_returns_already_combined_true_with_unchanged_message(tmp_path: Path) -> None: - """Same authored pipelines hash returns already_combined: True with message 'already applied, unchanged'.""" +def test_combine_same_hash_returns_already_applied_true_with_unchanged_message(tmp_path: Path) -> None: + """Same authored pipelines hash returns already_applied: True with message 'already applied, unchanged'.""" _setup_routed_agentic(tmp_path, decision="agentic") report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME - combines_path = tmp_path / "metadata" / "agentic_combines.json" + combines_path = tmp_path / "metadata" / "agentic_conversion.json" - first = apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()]) - assert first["ok"] is True and first["already_combined"] is False + first = apply_agentic_output(tmp_path, ["parent", "child"], [_lfc_pipeline()]) + assert first["ok"] is True and first["already_applied"] is False report_before = report_path.read_bytes() combines_before = combines_path.read_bytes() - second = apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()]) + second = apply_agentic_output(tmp_path, ["parent", "child"], [_lfc_pipeline()]) assert second["ok"] is True - assert second["already_combined"] is True + assert second["already_applied"] is True assert second.get("message") == "already applied, unchanged" assert report_path.read_bytes() == report_before @@ -1056,10 +1100,10 @@ def test_combine_empty_pipelines_list_is_refused(tmp_path: Path) -> None: """Empty pipelines list is refused and writes nothing.""" _setup_routed_agentic(tmp_path, decision="agentic") report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME - combines_path = tmp_path / "metadata" / "agentic_combines.json" + combines_path = tmp_path / "metadata" / "agentic_conversion.json" report_before = report_path.read_bytes() - result = apply_combine_fill(tmp_path, ["parent", "child"], []) + result = apply_agentic_output(tmp_path, ["parent", "child"], []) assert result["ok"] is False assert "cannot be empty" in result["error"] @@ -1071,7 +1115,7 @@ def test_combine_validation_failure_writes_nothing(tmp_path: Path) -> None: """When structural validation fails, neither the store nor the report is written.""" _setup_routed_agentic(tmp_path, decision="agentic") report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME - combines_path = tmp_path / "metadata" / "agentic_combines.json" + combines_path = tmp_path / "metadata" / "agentic_conversion.json" report_before = report_path.read_bytes() bad_pipeline = copy.deepcopy(_lfc_pipeline()) @@ -1079,7 +1123,7 @@ def test_combine_validation_failure_writes_nothing(tmp_path: Path) -> None: bad_task = bad_pipeline["tasks"][0] bad_task["depends_on"] = [{"task_key": "nonexistent"}] - result = apply_combine_fill(tmp_path, ["parent", "child"], [bad_pipeline]) + result = apply_agentic_output(tmp_path, ["parent", "child"], [bad_pipeline]) assert result["ok"] is False assert "violations" in result @@ -1092,7 +1136,7 @@ def test_combine_re_apply_after_reroute_rebuilds_unchanged(tmp_path: Path) -> No _setup_routed_agentic(tmp_path, decision="agentic") report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME - first = apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()]) + first = apply_agentic_output(tmp_path, ["parent", "child"], [_lfc_pipeline()]) assert first["ok"] is True report_after_combine = report_path.read_bytes() @@ -1106,7 +1150,7 @@ def test_combine_re_apply_after_reroute_rebuilds_unchanged(tmp_path: Path) -> No def test_merge_agentic_refuses_routed_agentic_pipeline(tmp_path: Path) -> None: """merge_agentic refuses a result landing in a routed-agentic pipeline.""" _setup_routed_agentic(tmp_path, decision="agentic") - apply_combine_fill(tmp_path, ["parent", "child"], [_lfc_pipeline()]) + apply_agentic_output(tmp_path, ["parent", "child"], [_lfc_pipeline()]) report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME results_dir = tmp_path / "agentic_results" @@ -1115,3 +1159,136 @@ def test_merge_agentic_refuses_routed_agentic_pipeline(tmp_path: Path) -> None: with pytest.raises(ValueError) as exc: merge_agentic_results(report_path, results_dir) assert "routed agentic" in str(exc.value) + + +# --------------------------------------------------------------------------- # +# The agent's output history, gaps of applied units, and fill-time checks. +# --------------------------------------------------------------------------- # + + +def _stored_output(output_dir: Path, unit_id: str) -> dict[str, Any]: + stored = json.loads((output_dir / "metadata" / "agentic_conversion.json").read_text(encoding="utf-8")) + return stored["components"][unit_id] + + +def test_the_first_fill_is_not_a_replacement_and_a_different_output_is(tmp_path: Path) -> None: + _setup_routed_agentic(tmp_path, decision="agentic") + assert apply_agentic_output(tmp_path, ["parent", "child"], [_lfc_pipeline()])["ok"] is True + first = _stored_output(tmp_path, "component-1") + assert first["replaced"] == [] + described = _lfc_pipeline() + described["description"] = "a different pattern" + + assert apply_agentic_output(tmp_path, ["parent", "child"], [described])["ok"] is True + + second = _stored_output(tmp_path, "component-1") + assert second["replaced"] == [{"from": first["output_sha256"], "to": second["output_sha256"]}] + record = routing_record(json.loads((tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_text(encoding="utf-8"))) + assert record is not None and "replacements" not in record["components"]["component-1"] + + +def test_the_output_history_survives_switching_to_deterministic_and_back(tmp_path: Path) -> None: + _route_parent_agentic(tmp_path, _report_two_pipelines()) + assert apply_agentic_output(tmp_path, ["parent"], [_named_lfc_pipeline("parent_lfc")])["ok"] is True + changed = _named_lfc_pipeline("parent_lfc") + changed["description"] = "changed" + assert apply_agentic_output(tmp_path, ["parent"], [changed])["ok"] is True + + _route_two_components(tmp_path, parent="deterministic", child="deterministic") + assert routing_record(json.loads((tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_text(encoding="utf-8"))) is None + _route_two_components(tmp_path, parent="agentic", child="deterministic") + + assert len(_stored_output(tmp_path, "component-2")["replaced"]) == 1 + assert _record_outcomes(tmp_path / WORK_DIRNAME / REPORT_FILENAME)["component-2"] == "agentic-applied" + + +def test_an_applied_unit_leaves_no_routed_gaps(tmp_path: Path) -> None: + _route_parent_agentic(tmp_path, _report_two_pipelines()) + gaps_path = tmp_path / WORK_DIRNAME / GAPS_FILENAME + assert [gap["pipeline"] for gap in json.loads(gaps_path.read_text(encoding="utf-8"))] == ["parent"] + + assert apply_agentic_output(tmp_path, ["parent"], [_named_lfc_pipeline("parent_lfc")])["ok"] is True + + assert json.loads(gaps_path.read_text(encoding="utf-8")) == [] + + +@pytest.mark.parametrize( + "authored_names", + [["Child"], ["Sales Load", "Sales-Load"]], + ids=["with-a-deterministic-pipeline", "within-the-authored-list"], +) +def test_fill_refuses_names_that_share_a_bundle_folder(tmp_path: Path, authored_names: list[str]) -> None: + _route_parent_agentic(tmp_path, _report_two_pipelines()) + report_path = tmp_path / WORK_DIRNAME / REPORT_FILENAME + report_before = report_path.read_bytes() + authored = [_named_lfc_pipeline(name) for name in authored_names] + for index, pipeline in enumerate(authored): + pipeline["tasks"][0]["resources"][0]["resource_key"] = f"ingestion_{index}" + pipeline["tasks"][0]["task"] = { + "pipeline_task": {"pipeline_id": f"${{resources.pipelines.ingestion_{index}.id}}"} + } + + result = apply_agentic_output(tmp_path, ["parent"], authored) + + assert result["ok"] is False + assert any("shares the bundle folder" in violation for violation in result["violations"]) + assert report_path.read_bytes() == report_before + assert not (tmp_path / "metadata" / "agentic_conversion.json").exists() + + +def test_fill_refuses_an_authored_pipeline_that_still_holds_a_placeholder(tmp_path: Path) -> None: + _route_parent_agentic(tmp_path, _report_two_pipelines()) + authored = _named_lfc_pipeline("parent_lfc") + authored["tasks"].append( + { + "name": "Loop", + "task_key": "loop", + "type": "ForEachActivity", + "inner_activities": [{"name": "Todo", "task_key": "todo", "type": "PlaceholderActivity"}], + } + ) + + result = apply_agentic_output(tmp_path, ["parent"], [authored]) + + assert result["ok"] is False + assert result["violations"] == [ + "parent_lfc: still has placeholder tasks ['Todo']; convert every task before filling the unit" + ] + + +def test_the_merge_refusal_points_at_fill_agentic_not_a_re_convert() -> None: + assert "fill-agentic" in ROUTED_AGENTIC_MERGE_REFUSED + assert "re-run convert" not in ROUTED_AGENTIC_MERGE_REFUSED and "combine" not in ROUTED_AGENTIC_MERGE_REFUSED + + +def test_route_refuses_a_report_converted_from_a_different_discover(tmp_path: Path) -> None: + stale = _report_two_pipelines() + stale["pipelines"][1]["name"] = "removed_since" + _write_work(tmp_path, stale, gaps=[]) + metadata = tmp_path / "metadata" + metadata.mkdir(parents=True, exist_ok=True) + (metadata / "inventory.json").write_text(json.dumps(_two_component_inventory(), indent=2), encoding="utf-8") + report_before = (tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_bytes() + + with pytest.raises(ValueError, match=r"different discover \(missing \['child'\], extra \['removed_since'\]\)"): + _route_two_components(tmp_path, parent="agentic", child="deterministic") + assert (tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_bytes() == report_before + + +def test_package_refuses_an_edited_gap_fill(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + report = _report_with_child_gap() + for pipeline in report["pipelines"]: + pipeline["tags"] = {"source": "adf"} + _route_parent_agentic(tmp_path, report) + results_dir = tmp_path / "agentic_results" + _write_merge_result(results_dir, "child", "Load", "load") + assert merge_agentic_results(tmp_path / WORK_DIRNAME / REPORT_FILENAME, results_dir) == (1, 0) + store_path = tmp_path / "metadata" / "agentic_conversion.json" + store = json.loads(store_path.read_text(encoding="utf-8")) + store["gap_fills"][0]["task"]["notebook_path"] = "/Workspace/Shared/tampered" + store_path.write_text(json.dumps(store), encoding="utf-8") + capsys.readouterr() + + assert package_main(["--output-dir", str(tmp_path), "--no-download-workspace-files"]) == 1 + + assert "agentic_conversion.json has been edited (gap fill 'Load')" in capsys.readouterr().err diff --git a/tests/unit/test_route_audit.py b/tests/unit/test_route_audit.py index bdaf8401..51748a48 100644 --- a/tests/unit/test_route_audit.py +++ b/tests/unit/test_route_audit.py @@ -1,10 +1,10 @@ -"""Tests for the routing/gaps audit artifact the package phase persists (FIX 5). +"""Tests for the routing audit artifact the package phase persists. Package prunes the transient ``.work/`` folder by default, which erases the routing trail -(translation report + ``gaps.json``). ``_write_route_audit`` summarises the recorded plan's routed -components + decisions + the gaps routing introduced into ``metadata/route_audit.json`` so the trail -survives the prune. It must no-op (write nothing) when no plan was recorded, keeping the no-route -path byte-identical. +(translation report + ``gaps.json``). ``_write_route_audit`` summarises the recorded plan's routing +units, their decisions and outcomes, the agent's output history, the gap fills and the routing +conversation into ``metadata/route_audit.json`` so the trail survives the prune. It must no-op (write +nothing) when no plan was recorded, keeping the no-route path byte-identical. """ from __future__ import annotations @@ -13,46 +13,36 @@ from pathlib import Path from flowx.bundler.dab_writer import _write_route_audit -from flowx.models.conversion_plan import ComponentPlan, ConversionPlan +from flowx.discovery_serde import canonical_sha256 +from flowx.models.conversion_plan import ComponentPlan, ConversationEntry, ConversionPlan, SuggestedGrouping -def _write_plan(output_dir: Path) -> None: - ConversionPlan( +def _plan(*, accepted: bool = False) -> ConversionPlan: + return ConversionPlan( inventory_sha256="abc123", source_graphs_sha256="graphs123", agentic_insights_sha256="insights123", components=[ - ComponentPlan(component_id="component-1", members=["parent", "child"], decision="agentic"), - ComponentPlan(component_id="component-2", members=["solo"], decision="deterministic"), + ComponentPlan(component_id="component-1", members=["child", "parent"], decision="agentic"), + ComponentPlan( + component_id="component-2", members=["solo"], decision="agentic" if accepted else "deterministic" + ), ], - ).write(output_dir) - - -def _write_gaps(output_dir: Path) -> None: - work = output_dir / ".work" - work.mkdir(parents=True, exist_ok=True) - gaps = [ - { - "activity_name": "Extract", - "activity_type": "CopyActivity", - "raw_definition": {"name": "Extract", "type": "CopyActivity"}, - "pipeline": "parent", - }, - { - "activity_name": "Load", - "activity_type": "NotebookActivity", - "raw_definition": {"name": "Load"}, - "pipeline": "child", - }, - {"activity_name": "Fail", "activity_type": "Fail", "raw_definition": {"name": "Fail"}}, - {"activity_name": "Hook", "activity_type": "WebHook", "raw_definition": {}, "pipeline": "solo"}, - ] - (work / "gaps.json").write_text(json.dumps(gaps, indent=2), encoding="utf-8") + suggested_groupings=[ + SuggestedGrouping( + grouping_id="grouping-1", + components=["component-1", "component-2"], + members=["child", "parent", "solo"], + accepted=accepted, + ) + ], + conversation=[ConversationEntry(question="What should the conversion achieve?", answer="Fewer pipelines")], + ) -def test_route_audit_summarises_components_decisions_and_gaps(tmp_path: Path) -> None: - _write_plan(tmp_path) - _write_gaps(tmp_path) +def test_route_audit_summarises_units_decisions_and_the_conversation(tmp_path: Path) -> None: + plan = _plan() + plan.write(tmp_path) audit_path = _write_route_audit(tmp_path) @@ -61,15 +51,46 @@ def test_route_audit_summarises_components_decisions_and_gaps(tmp_path: Path) -> assert audit["recorded_against_inventory_sha256"] == "abc123" assert audit["source_graphs_sha256"] == "graphs123" assert audit["agentic_insights_sha256"] == "insights123" - # Both routed components and their decisions are recorded. + assert audit["conversion_plan_sha256"] == canonical_sha256(plan.to_dict()) decisions = {component["component_id"]: component["decision"] for component in audit["components"]} assert decisions == {"component-1": "agentic", "component-2": "deterministic"} - # Only the agentic component's members are surfaced as agentic pipelines. assert audit["agentic_pipelines"] == ["child", "parent"] - # Only gaps in agentic-routed pipelines count as introduced by routing, as a compact summary. - assert audit["gaps_count"] == 2 - assert {gap["pipeline"] for gap in audit["gaps_introduced"]} == {"parent", "child"} - assert all("raw_definition" not in gap for gap in audit["gaps_introduced"]) + assert audit["conversation"] == [{"question": "What should the conversion achieve?", "answer": "Fewer pipelines"}] + assert "gaps_introduced" not in audit and "gaps_count" not in audit + + +def test_route_audit_lists_an_accepted_grouping_as_one_unit(tmp_path: Path) -> None: + _plan(accepted=True).write(tmp_path) + + audit = json.loads(_write_route_audit(tmp_path).read_text(encoding="utf-8")) + + (unit,) = audit["components"] + assert unit["component_id"] == "grouping-1" + assert unit["members"] == ["child", "parent", "solo"] + assert unit["grouped_components"] == ["component-1", "component-2"] + assert unit["decision"] == "agentic" + + +def test_route_audit_copies_the_output_replacement_history(tmp_path: Path) -> None: + _plan().write(tmp_path) + pipelines = [{"name": "orders_lfc", "tasks": []}] + store = { + "components": { + "component-1": { + "members": ["child", "parent"], + "pipelines": pipelines, + "output_sha256": canonical_sha256(pipelines), + "replaced": [{"from": "old", "to": canonical_sha256(pipelines)}], + } + }, + "gap_fills": [], + } + (tmp_path / "metadata" / "agentic_conversion.json").write_text(json.dumps(store), encoding="utf-8") + + audit = json.loads(_write_route_audit(tmp_path).read_text(encoding="utf-8")) + + component = next(entry for entry in audit["components"] if entry["component_id"] == "component-1") + assert component["replaced"] == [{"from": "old", "to": canonical_sha256(pipelines)}] def test_route_audit_noops_without_a_recorded_plan(tmp_path: Path) -> None: @@ -80,15 +101,3 @@ def test_route_audit_noops_without_a_recorded_plan(tmp_path: Path) -> None: assert audit_path is None assert not (tmp_path / "metadata" / "route_audit.json").exists() - - -def test_route_audit_handles_missing_gaps_file(tmp_path: Path) -> None: - # A recorded plan but no gaps.json (e.g. all-deterministic route) still writes an audit with zero gaps. - _write_plan(tmp_path) - - audit_path = _write_route_audit(tmp_path) - - assert audit_path is not None - audit = json.loads(audit_path.read_text(encoding="utf-8")) - assert audit["gaps_count"] == 0 - assert audit["gaps_introduced"] == [] diff --git a/tests/unit/test_routing_groupings.py b/tests/unit/test_routing_groupings.py new file mode 100644 index 00000000..5dd32b95 --- /dev/null +++ b/tests/unit/test_routing_groupings.py @@ -0,0 +1,274 @@ +"""Suggested groupings: whole components the insights link, converted together agentically as one unit. + +Route suggests a grouping from two kinds of link in the enriched insights: an ``inferred`` pipeline +relationship between pipelines of different components, and one simplification pattern recommended +for pipelines in more than one component. Linked components form one suggestion (so suggestions +never overlap); the user accepts one by setting ``accepted`` and deciding its components agentic, and +it then routes, fills, packages and audits as one unit. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from flowx import routing +from flowx.bundler.dab_writer import main as package_main +from flowx.discovery_insights import enrich_inventory +from flowx.discovery_inventory import STRATEGY_PROPERTY, build_source_inventory +from flowx.ir_serde import pipeline_to_dict +from flowx.models.conversion_plan import ConversionPlan +from flowx.models.discovery import CONCEPT_NOTEBOOK, SourceGraph, SourceNode +from flowx.models.ir import ControlEdge, Lineage, NotebookActivity, Pipeline +from flowx.route_agentic import REPORT_FILENAME, WORK_DIRNAME, apply_agentic_output, apply_plan, routing_record + +_PIPELINES = ("child", "extract_a", "extract_b", "extract_c", "mart", "orchestrator") +_LAKEFLOW_CONNECT = "Lakeflow Connect SQL Server connector" + + +def _node(task_key: str, native_type: str = "DatabricksNotebook") -> SourceNode: + return SourceNode( + source_id=task_key, + task_key=task_key, + concept=CONCEPT_NOTEBOOK, + source="adf", + name=task_key, + native_type=native_type, + properties={STRATEGY_PROPERTY: "deterministic"}, + ) + + +def _inventory(source: str = "adf") -> dict[str, Any]: + """'orchestrator' calls 'child'; the four other pipelines are independent components.""" + graphs = [ + SourceGraph( + name="orchestrator", + source=source, + tasks=[_node("call_child", "ExecutePipeline")], + lineage=Lineage( + control_edges=[ + ControlEdge(source_workflow="orchestrator", target_workflow="child", via_task_key="call_child") + ] + ), + ), + *(SourceGraph(name=name, source=source, tasks=[_node(f"{name}_load")]) for name in _PIPELINES[:-1]), + ] + return build_source_inventory(graphs, source=source, source_dir="/src") + + +def _pattern(name: str, *, simplification: bool = True) -> dict[str, Any]: + pattern: dict[str, Any] = {"pattern": name, "fit": "One managed pipeline", "simplification_pattern": simplification} + if simplification: + pattern["release_state"] = "ga" + return pattern + + +def _insights(*, extra_patterns: dict[str, str] | None = None) -> dict[str, Any]: + """extract_a + extract_b share a simplification pattern; extract_c feeds mart through a shared table.""" + pattern_by_pipeline = {"extract_a": _LAKEFLOW_CONNECT, "extract_b": _LAKEFLOW_CONNECT, **(extra_patterns or {})} + return { + "overview": "Extractors land tables the mart reads.", + "pipeline_insights": [ + {"pipeline": pipeline, "intent": f"Load {pipeline}", "recommended_patterns": [_pattern(pattern)]} + for pipeline, pattern in sorted(pattern_by_pipeline.items()) + ] + + [{"pipeline": "mart", "recommended_patterns": [_pattern("Databricks SQL", simplification=False)]}], + "pipeline_relationships": [ + { + "from_pipeline": "extract_c", + "to_pipeline": "mart", + "lineage_edge": { + "edge_type": "inferred", + "edge_identity": "sales.orders", + "evidence": "extract_c writes sales.orders, which mart reads", + "confidence": "high", + }, + "relationship_summary": "mart reads what extract_c lands", + }, + { + "from_pipeline": "orchestrator", + "to_pipeline": "child", + "lineage_edge": {"edge_type": "control", "edge_identity": "call_child"}, + }, + { + "from_pipeline": "child", + "to_pipeline": "orchestrator", + "lineage_edge": { + "edge_type": "inferred", + "edge_identity": "audit.log", + "evidence": "both write audit.log", + "confidence": "low", + }, + }, + ], + } + + +def _enriched(output_dir: Path, **insight_options: Any) -> dict[str, Any]: + """Write the inventory, enrich it for real, and return the enriched inventory.""" + metadata = output_dir / "metadata" + metadata.mkdir(parents=True, exist_ok=True) + (metadata / "inventory.json").write_text(json.dumps(_inventory(), indent=2), encoding="utf-8") + result = enrich_inventory(output_dir, insights=_insights(**insight_options)) + assert result["ok"] is True, result + return json.loads((metadata / "inventory.json").read_text(encoding="utf-8")) + + +def _write_report(output_dir: Path) -> None: + work = output_dir / WORK_DIRNAME + work.mkdir(parents=True, exist_ok=True) + pipelines = [ + pipeline_to_dict( + Pipeline( + name=name, + tasks=[NotebookActivity(name="load", task_key="load", notebook_path=f"/Workspace/Shared/{name}")], + tags={"source": "adf"}, + ) + ) + for name in _PIPELINES + ] + (work / REPORT_FILENAME).write_text(json.dumps({"pipelines": pipelines}, indent=2), encoding="utf-8") + (work / "gaps.json").write_text("[]", encoding="utf-8") + + +def _plan(inventory: dict[str, Any], agentic: set[str], accepted: set[str]) -> dict[str, Any]: + """The default plan with ``agentic`` components routed agentic and ``accepted`` groupings accepted.""" + recommendation = routing.build_recommendation(inventory) + components = [ + {**component, "decision": "agentic" if component["component_id"] in agentic else "deterministic"} + for component in recommendation["default_plan"]["components"] + ] + groupings = [ + {"grouping_id": grouping["grouping_id"], "accepted": grouping["grouping_id"] in accepted} + for grouping in recommendation["suggested_groupings"] + ] + return {"components": components, "suggested_groupings": groupings} + + +def _route(output_dir: Path, plan: dict[str, Any]) -> dict[str, Any]: + assert routing.record_plan(output_dir, plan=plan)["ok"] is True + recorded = ConversionPlan.load(output_dir) + assert recorded is not None + return apply_plan(output_dir, recorded) + + +def _authored(name: str) -> dict[str, Any]: + return pipeline_to_dict( + Pipeline( + name=name, + tasks=[NotebookActivity(name="ingest", task_key="ingest", notebook_path="/Workspace/Shared/lakeflow")], + tags={"source": "adf"}, + ) + ) + + +def test_groupings_come_from_a_shared_simplification_pattern_and_an_inferred_relationship(tmp_path: Path) -> None: + inventory = _enriched(tmp_path) + + groupings = routing.suggest_groupings(inventory) + + assert [(grouping["grouping_id"], grouping["components"]) for grouping in groupings] == [ + ("grouping-1", ["component-2", "component-3"]), + ("grouping-2", ["component-4", "component-5"]), + ] + assert groupings[0]["members"] == ["extract_a", "extract_b"] + assert groupings[0]["basis"] == [ + {"kind": "shared_pattern", "pattern": _LAKEFLOW_CONNECT, "pipelines": ["extract_a", "extract_b"]} + ] + (inferred,) = groupings[1]["basis"] + assert inferred["kind"] == "inferred_relationship" + assert (inferred["from_pipeline"], inferred["to_pipeline"]) == ("extract_c", "mart") + assert inferred["evidence"] == "extract_c writes sales.orders, which mart reads" + assert inferred["confidence"] == "high" + assert all(grouping["accepted"] is False for grouping in groupings) + + +def test_links_of_both_kinds_join_into_one_suggestion_so_suggestions_never_overlap(tmp_path: Path) -> None: + inventory = _enriched(tmp_path, extra_patterns={"extract_c": _LAKEFLOW_CONNECT}) + + (grouping,) = routing.suggest_groupings(inventory) + + assert grouping["components"] == ["component-2", "component-3", "component-4", "component-5"] + assert grouping["members"] == ["extract_a", "extract_b", "extract_c", "mart"] + assert {basis["kind"] for basis in grouping["basis"]} == {"shared_pattern", "inferred_relationship"} + + +def test_no_grouping_without_insights_or_for_an_airflow_inventory() -> None: + assert routing.suggest_groupings(_inventory()) == [] + airflow = _inventory("airflow") + airflow["insights"] = _insights() + assert routing.suggest_groupings(airflow) == [] + + +def test_an_accepted_grouping_routes_fills_and_packages_as_one_unit( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + inventory = _enriched(tmp_path) + _write_report(tmp_path) + + outcome = _route(tmp_path, _plan(inventory, {"component-2", "component-3"}, {"grouping-1"})) + + assert outcome["outcomes"]["grouping-1"] == "agentic-not-viable" + assert "component-2" not in outcome["outcomes"] and "component-3" not in outcome["outcomes"] + result = apply_agentic_output(tmp_path, ["extract_a", "extract_b"], [_authored("sql_server_ingest")]) + assert result["ok"] is True and result["component_id"] == "grouping-1" + report = json.loads((tmp_path / WORK_DIRNAME / REPORT_FILENAME).read_text(encoding="utf-8")) + names = sorted(pipeline["name"] for pipeline in report["pipelines"]) + assert names == ["child", "extract_c", "mart", "orchestrator", "sql_server_ingest"] + record = routing_record(report) + assert record is not None and record["components"]["grouping-1"]["outcome"] == "agentic-applied" + + assert package_main(["--output-dir", str(tmp_path), "--no-download-workspace-files", "--keep-intermediates"]) == 0 + audit = json.loads((tmp_path / "metadata" / "route_audit.json").read_text(encoding="utf-8")) + grouped = next(unit for unit in audit["components"] if unit["component_id"] == "grouping-1") + assert grouped["grouped_components"] == ["component-2", "component-3"] + assert grouped["outcome"] == "agentic-applied" + + +def test_un_accepting_a_grouping_routes_its_components_separately_again(tmp_path: Path) -> None: + inventory = _enriched(tmp_path) + _write_report(tmp_path) + _route(tmp_path, _plan(inventory, {"component-2", "component-3"}, {"grouping-1"})) + assert apply_agentic_output(tmp_path, ["extract_a", "extract_b"], [_authored("sql_server_ingest")])["ok"] is True + + outcome = _route(tmp_path, _plan(inventory, {"component-2", "component-3"}, set())) + + assert outcome["outcomes"]["component-2"] == outcome["outcomes"]["component-3"] == "agentic-not-viable" + assert "grouping-1" not in outcome["outcomes"] + assert apply_agentic_output(tmp_path, ["extract_a", "extract_b"], [_authored("x")])["ok"] is False + + +def test_a_grouping_with_a_component_decided_deterministic_is_refused(tmp_path: Path) -> None: + inventory = _enriched(tmp_path) + + violations = routing.validate_plan(_plan(inventory, {"component-2"}, {"grouping-1"}), inventory) + + assert any("grouping 'grouping-1'" in violation and "'component-3'" in violation for violation in violations) + + +def test_only_a_suggested_grouping_can_be_accepted(tmp_path: Path) -> None: + inventory = _enriched(tmp_path) + plan = _plan(inventory, set(), set()) + plan["suggested_groupings"].append({"grouping_id": "grouping-9", "accepted": True}) + + violations = routing.validate_plan(plan, inventory) + + assert any("'grouping-9' is not a grouping route suggests" in violation for violation in violations) + + +def test_an_accepted_grouping_is_kept_when_route_runs_again_without_a_plan(tmp_path: Path) -> None: + inventory = _enriched(tmp_path) + assert routing.record_plan(tmp_path, plan=_plan(inventory, {"component-2", "component-3"}, {"grouping-1"}))["ok"] + recorded = json.loads((tmp_path / "metadata" / "conversion_plan.json").read_text(encoding="utf-8")) + + carried = routing.carried_forward_plan(recorded, inventory) + + assert {entry["grouping_id"]: entry["accepted"] for entry in carried["suggested_groupings"]} == { + "grouping-1": True, + "grouping-2": False, + } + decisions = {entry["component_id"]: entry["decision"] for entry in carried["components"]} + assert decisions["component-2"] == decisions["component-3"] == "agentic" diff --git a/tests/unit/test_routing_insights_check.py b/tests/unit/test_routing_insights_check.py new file mode 100644 index 00000000..18f1bc67 --- /dev/null +++ b/tests/unit/test_routing_insights_check.py @@ -0,0 +1,165 @@ +"""Route, fill-agentic and package refuse when agentic_insights.json and inventory.json disagree. + +Enrich writes ``metadata/agentic_insights.json`` and then ``inventory.json``. A run stopped between +the two, or a hand edit, leaves them out of step; the readers then refuse and say how to recover. The +file's own ``agentic_insights_sha256`` is checked too. Package runs the check whenever enrich ran, +plan or no plan; a run without enrich is unaffected. +""" + +from __future__ import annotations + +import io +import json +from pathlib import Path +from typing import Any + +import pytest + +from flowx import routing +from flowx.adapter.__main__ import main as adapter_cli_main +from flowx.bundler.dab_writer import main as package_main +from flowx.discovery_insights import enrich_inventory +from flowx.discovery_inventory import STRATEGY_PROPERTY, build_source_inventory +from flowx.discovery_serde import canonical_sha256 +from flowx.ir_serde import pipeline_to_dict +from flowx.models.conversion_plan import ConversionPlan +from flowx.models.discovery import CONCEPT_NOTEBOOK, SourceGraph, SourceNode +from flowx.models.ir import NotebookActivity, Pipeline +from flowx.route_agentic import REPORT_FILENAME, WORK_DIRNAME, apply_agentic_output + + +def _setup(output_dir: Path) -> None: + """An enriched one-pipeline factory with its convert report.""" + node = SourceNode( + source_id="load", + task_key="load", + concept=CONCEPT_NOTEBOOK, + source="adf", + name="load", + native_type="DatabricksNotebook", + properties={STRATEGY_PROPERTY: "agentic"}, + ) + inventory = build_source_inventory( + [SourceGraph(name="solo", source="adf", tasks=[node])], source="adf", source_dir="/x" + ) + metadata = output_dir / "metadata" + metadata.mkdir(parents=True, exist_ok=True) + (metadata / "inventory.json").write_text(json.dumps(inventory, indent=2), encoding="utf-8") + assert enrich_inventory(output_dir, insights={"overview": "Solo loads one table."})["ok"] is True + work = output_dir / WORK_DIRNAME + work.mkdir(parents=True, exist_ok=True) + pipeline = Pipeline( + name="solo", + tasks=[NotebookActivity(name="load", task_key="load", notebook_path="/Workspace/Shared/load")], + tags={"source": "adf"}, + ) + (work / REPORT_FILENAME).write_text(json.dumps({"pipelines": [pipeline_to_dict(pipeline)]}), encoding="utf-8") + (work / "gaps.json").write_text("[]", encoding="utf-8") + + +def _insights_path(output_dir: Path) -> Path: + return output_dir / "metadata" / "agentic_insights.json" + + +def _read(path: Path) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def _insights_ahead_of_the_inventory(output_dir: Path) -> None: + """The crash window: enrich replaced agentic_insights.json but stopped before inventory.json.""" + document = _read(_insights_path(output_dir)) + content = {key: value for key, value in document.items() if key != "agentic_insights_sha256"} + content["overview"] = "A later enrich." + _insights_path(output_dir).write_text( + json.dumps({**content, "agentic_insights_sha256": canonical_sha256(content)}), encoding="utf-8" + ) + + +def _insights_missing(output_dir: Path) -> None: + _insights_path(output_dir).unlink() + + +def _insights_edited_in_both(output_dir: Path) -> None: + """Both files agree, but the document no longer matches its own hash.""" + document = _read(_insights_path(output_dir)) + document["overview"] = "Edited by hand." + _insights_path(output_dir).write_text(json.dumps(document), encoding="utf-8") + inventory = _read(output_dir / "metadata" / "inventory.json") + inventory["insights"] = document + (output_dir / "metadata" / "inventory.json").write_text(json.dumps(inventory), encoding="utf-8") + + +_BREAKS = { + "insights-ahead": (_insights_ahead_of_the_inventory, "disagree"), + "insights-missing": (_insights_missing, "is missing"), + "self-hash-broken": (_insights_edited_in_both, "does not match its recorded agentic_insights_sha256"), +} + + +def test_route_binds_the_plan_to_the_real_agentic_insights_hash(tmp_path: Path) -> None: + _setup(tmp_path) + plan = {"components": [{"component_id": "component-1", "members": ["solo"], "decision": "deterministic"}]} + + assert routing.record_plan(tmp_path, plan=plan)["ok"] is True + + recorded = ConversionPlan.load(tmp_path) + assert recorded is not None + assert recorded.agentic_insights_sha256 is not None + assert recorded.agentic_insights_sha256 == _read(_insights_path(tmp_path))["agentic_insights_sha256"] + + +def test_an_enriched_or_unenriched_run_with_files_in_step_passes(tmp_path: Path) -> None: + _setup(tmp_path) + inventory = _read(tmp_path / "metadata" / "inventory.json") + assert routing.insights_file_violations(tmp_path, inventory) == [] + del inventory["insights"] + _insights_missing(tmp_path) + assert routing.insights_file_violations(tmp_path, inventory) == [] + + +@pytest.mark.parametrize("case", sorted(_BREAKS)) +def test_route_refuses_and_says_how_to_recover( + tmp_path: Path, case: str, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch +) -> None: + _setup(tmp_path) + breaker, expected = _BREAKS[case] + breaker(tmp_path) + monkeypatch.setattr("sys.stdin", io.StringIO("")) + + assert adapter_cli_main(["route", "--output-dir", str(tmp_path)]) == 1 + + (violation,) = json.loads(capsys.readouterr().out)["violations"] + assert expected in violation + assert "run enrich again with the same insights" in violation and ".enrich.lock" in violation + assert not (tmp_path / "metadata" / "conversion_plan.json").exists() + + +@pytest.mark.parametrize("case", sorted(_BREAKS)) +def test_package_refuses_even_without_a_plan(tmp_path: Path, case: str, capsys: pytest.CaptureFixture[str]) -> None: + _setup(tmp_path) + breaker, expected = _BREAKS[case] + breaker(tmp_path) + + code = package_main(["--output-dir", str(tmp_path), "--no-download-workspace-files", "--keep-intermediates"]) + + assert code == 1 + error = capsys.readouterr().err + assert expected in error and "run enrich again" in error + assert not (tmp_path / "databricks.yml").exists() + + +def test_fill_agentic_refuses_when_the_files_disagree(tmp_path: Path) -> None: + _setup(tmp_path) + plan = {"components": [{"component_id": "component-1", "members": ["solo"], "decision": "agentic"}]} + assert adapter_cli_main(["route", "--output-dir", str(tmp_path), "--plan-path", _write(tmp_path, plan)]) == 0 + _insights_ahead_of_the_inventory(tmp_path) + + result = apply_agentic_output(tmp_path, ["solo"], [{"name": "solo", "tags": {"source": "adf"}, "tasks": []}]) + + assert result["ok"] is False and "disagree" in result["error"] + + +def _write(output_dir: Path, plan: dict[str, Any]) -> str: + path = output_dir / "plan.json" + path.write_text(json.dumps(plan), encoding="utf-8") + return str(path) diff --git a/tests/unit/test_routing_record.py b/tests/unit/test_routing_record.py index 1a8c9fd3..e87c535e 100644 --- a/tests/unit/test_routing_record.py +++ b/tests/unit/test_routing_record.py @@ -19,6 +19,7 @@ from flowx.adapter.__main__ import main as adapter_cli_main from flowx.discovery_insights import inventory_fingerprint from flowx.discovery_inventory import STRATEGY_PROPERTY, build_source_inventory +from flowx.discovery_serde import canonical_sha256 from flowx.models.conversion_plan import ( SCHEMA_VERSION, ComponentPlan, @@ -79,6 +80,7 @@ def _inventory() -> dict[str, Any]: } ] } + inventory["insights"]["agentic_insights_sha256"] = canonical_sha256(inventory["insights"]) return inventory @@ -97,6 +99,8 @@ def _write_inventory(output_dir: Path, inventory: dict[str, Any]) -> Path: metadata.mkdir(parents=True, exist_ok=True) path = metadata / "inventory.json" path.write_text(json.dumps(inventory, indent=2), encoding="utf-8") + if "insights" in inventory: + (metadata / "agentic_insights.json").write_text(json.dumps(inventory["insights"], indent=2), encoding="utf-8") return path @@ -145,22 +149,42 @@ def test_non_dict_payload_is_a_violation() -> None: assert validate_plan([1, 2], _inventory()) == ["conversion plan must be a JSON object, got list"] -def test_unknown_top_level_key_including_library_owned_fields() -> None: +def test_library_owned_top_level_keys_are_ignored_and_unknown_ones_rejected() -> None: raw = _authored_plan() raw["schema_version"] = "1" + raw["findings"] = ["edited"] raw["bogus"] = True violations = validate_plan(raw, _inventory()) - assert any("unknown top-level key: 'schema_version' (set by the library, not the author)" in v for v in violations) - assert any("unknown top-level key: 'bogus'" in v for v in violations) + assert violations == ["unknown top-level key: 'bogus'"] -def test_library_owned_component_fields_are_rejected() -> None: +def test_library_owned_component_fields_are_ignored_and_recomputed(tmp_path: Path) -> None: + _write_inventory(tmp_path, _inventory()) raw = _authored_plan() - raw["components"][0]["recommended"] = "deterministic" + raw["components"][0]["recommended"] = "agentic" raw["components"][0]["options"] = {} - violations = validate_plan(raw, _inventory()) - assert any("unknown field 'recommended' (set by the library, not the author)" in v for v in violations) - assert any("unknown field 'options' (set by the library, not the author)" in v for v in violations) + raw["components"][0]["typo"] = 1 + assert validate_plan(raw, _inventory()) == ["components[0]: unknown field 'typo'"] + del raw["components"][0]["typo"] + + assert record_plan(tmp_path, plan=raw)["ok"] is True + + recorded = json.loads((tmp_path / "metadata" / PLAN_FILENAME).read_text(encoding="utf-8")) + assert recorded["components"][0]["recommended"] == "deterministic" + assert recorded["components"][0]["options"]["deterministic"]["capable"] is True + + +def test_a_pending_decision_is_valid_and_recorded_as_pending(tmp_path: Path) -> None: + _write_inventory(tmp_path, _inventory()) + raw = _authored_plan() + raw["components"][0]["decision"] = None + del raw["components"][1]["decision"] + + result = record_plan(tmp_path, plan=raw) + + assert result["ok"] is True and result["pending"] == ["component-1", "component-2"] + recorded = ConversionPlan.load(tmp_path) + assert recorded is not None and recorded.pending_components() == ["component-1", "component-2"] def test_invalid_decision_value_is_rejected() -> None: @@ -337,18 +361,20 @@ def _write_empty_report(output_dir: Path) -> None: (work / "translation_report.json").write_text(json.dumps(report, indent=2), encoding="utf-8") -def test_cli_route_without_plan_emits_components( +def test_cli_route_without_plan_writes_the_recommendation_into_the_plan( tmp_path: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch ) -> None: import io _write_inventory(tmp_path, _inventory()) - monkeypatch.setattr("sys.stdin", io.StringIO("")) # not a TTY -> dry-run recommendation + monkeypatch.setattr("sys.stdin", io.StringIO("")) # not a TTY code = adapter_cli_main(["route", "--output-dir", str(tmp_path)]) assert code == 0 payload = json.loads(capsys.readouterr().out) - assert [c["component_id"] for c in payload["components"]] == ["component-1", "component-2"] - assert "default_plan" in payload + assert payload["pending"] == ["component-1", "component-2"] and payload["applied"] is False + recorded = json.loads((tmp_path / "metadata" / PLAN_FILENAME).read_text(encoding="utf-8")) + assert [component["component_id"] for component in recorded["components"]] == ["component-1", "component-2"] + assert recorded["components"][0]["options"]["agentic"]["has_simplification"] is True def test_cli_route_with_plan_records_and_reports_the_edit(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: diff --git a/tests/unit/test_routing_review_page.py b/tests/unit/test_routing_review_page.py new file mode 100644 index 00000000..52873215 --- /dev/null +++ b/tests/unit/test_routing_review_page.py @@ -0,0 +1,140 @@ +"""The routing review page: one standard HTML view, the same eight sections in the same order every run. + +Route writes ``metadata/routing_review.html`` from the recorded plan, the inventory's insights and the +report's routing record. It escapes every authored value, carries no timestamp (the same inputs give +the same bytes), and says per unit what package will do. +""" + +from __future__ import annotations + +import io +import json +import re +from pathlib import Path +from typing import Any + +import pytest + +from flowx.adapter.__main__ import main as adapter_cli_main +from flowx.discovery_inventory import STRATEGY_PROPERTY, build_source_inventory +from flowx.discovery_serde import canonical_sha256 +from flowx.models.conversion_plan import ComponentPlan, ConversationEntry, ConversionPlan, SuggestedGrouping +from flowx.models.discovery import CONCEPT_NOTEBOOK, SourceGraph, SourceNode +from flowx.reporting.routing_review import render_routing_review + +_SECTIONS = [ + "1. Summary", + "2. What the source does", + "3. Components", + "4. Suggested groupings", + "5. How it will be converted", + "6. Routing conversation", + "7. Findings", + "8. How to steer", +] + + +def _plan(**overrides: Any) -> ConversionPlan: + fields: dict[str, Any] = { + "inventory_sha256": "inv", + "components": [ + ComponentPlan(component_id="component-1", members=["extract"], recommended="agentic", decision=None), + ComponentPlan( + component_id="component-2", members=["mart"], recommended="deterministic", decision="agentic" + ), + ComponentPlan(component_id="component-3", members=["solo"], decision="deterministic"), + ], + } + fields.update(overrides) + return ConversionPlan(**fields) + + +def _headings(page: str) -> list[str]: + return re.findall(r"

    (.*?)

    ", page) + + +def test_every_page_has_the_same_eight_sections_in_order() -> None: + assert _headings(render_routing_review(_plan(), None, None)) == _SECTIONS + assert _headings(render_routing_review(ConversionPlan(), {"pipelines": []}, None)) == _SECTIONS + + +def test_authored_text_is_escaped() -> None: + plan = _plan( + conversation=[ConversationEntry(question="", answer="a & ")], + components=[ + ComponentPlan(component_id="component-1", members=["x"], decision="agentic", rationale="") + ], + ) + inventory = {"insights": {"overview": ""}} + + page = render_routing_review(plan, inventory, None) + + assert "