From d605849e3f2d7854cf09e6c688d2b5feecd33678 Mon Sep 17 00:00:00 2001 From: Yury Date: Sat, 10 Oct 2026 22:19:08 +0300 Subject: [PATCH] perf: build declaration inlays from model facts Record authored type syntax through decoding and cloning. Reuse the snapshot's authored subjects and typed-data index without composing source or populating grammar for inlays. Preserve fragment and YAML-alias source boundaries. A/B vs bb6941f, three rounds/best of three: inlays/unwrap time -27.4% (4.9% noise), peak -34.8%, kept -30.2%. Mixed times stay within noise; snapshot-first peak +10.0%, kept +0.1%, accepted as construction-order overlap. Source-first peak/kept +0.1%. large/service time stays within 13.0% noise, peak +0.4%, kept +0.6%. Gate: Ruff, format, strict mypy, 5861 passed/68 skipped/1 xfailed. Focused pure-Python YAML: 305 passed. Inlay and both mixed linearity checks pass. Record the acceptance decision, workload boundaries, raw phase evidence and allocation attribution. --- docs/05-type-model.md | 7 + docs/13-public-api.md | 4 + docs/15-implementation-plan.md | 13 +- docs/21-language-service.md | 17 +- .../service-baseline-integration.md | 42 +- .../service-model-inlays-phase-metrics.json | 690 ++++++++++++++++++ .../2026-10-10/service-model-inlays.md | 168 +++++ .../service_inlay_allocation_probe.py | 84 +++ .../2026-10-10/service-recovery-plan.md | 15 +- fastraml/service/hover.py | 77 +- fastraml/types/base.py | 4 + fastraml/types/shape.py | 4 + tests/bench/test_corpus.py | 10 + tests/unit/test_service_inlays.py | 111 +++ tests/unit/test_shape_decode.py | 26 + 15 files changed, 1247 insertions(+), 25 deletions(-) create mode 100644 docs/reports/2026-10-10/service-model-inlays-phase-metrics.json create mode 100644 docs/reports/2026-10-10/service-model-inlays.md create mode 100644 docs/reports/2026-10-10/service_inlay_allocation_probe.py diff --git a/docs/05-type-model.md b/docs/05-type-model.md index 5f00f85..1b9781e 100644 --- a/docs/05-type-model.md +++ b/docs/05-type-model.md @@ -47,6 +47,13 @@ returned to a caller holds no type-expression node, unless it was parsed with each would only add memory, 2.4 MB on the `validate` workload's `service` configuration. Read only `value` and `position`, which both forms have. +`BaseShape.type_written` records authored type syntax during declaration decoding, +before resolution or unwrap, and survives `clone`/`clone_detached`. Scalar type +names and multiple-inheritance sequences are explicit, as are accepted `type:` +and `schema:` fields even when their value is null. A bare empty/null declaration +or a mapping with only inferred facets has no authored type. This fact is distinct +from the resolved kind or the presence of bound expression references. + ## 2. Kinds and facets | Kind | Kind-specific facets | diff --git a/docs/13-public-api.md b/docs/13-public-api.md index a2dedbe..d8e466d 100644 --- a/docs/13-public-api.md +++ b/docs/13-public-api.md @@ -162,6 +162,10 @@ Consumers must honor these contracts: partial: the expression's `value` and `position`, not a YAML `Node`. With `retain_source` it stays the `Node`. Read only `value` and `position` (docs/05 § 1). +6. `BaseShape.type_written` records whether a declaration supplied type syntax, + independently of its resolved kind or expression references. Cloning preserves + it; bare empty/null declarations are inferred, while an explicit `type:` or + `schema:` field is authored even with a null value (docs/05 § 1). Parsing, `build_graph`, `build_occurrences`, `Linter` runs, `to_openapi`, and the `lsp` verb raise the garbage collector's full-collection threshold while diff --git a/docs/15-implementation-plan.md b/docs/15-implementation-plan.md index 840c0c9..43f1a4e 100644 --- a/docs/15-implementation-plan.md +++ b/docs/15-implementation-plan.md @@ -22,7 +22,13 @@ XML Schema external types are unsupported ([01](01-scope-and-coverage.md) § 3). **Service baseline acceptance and recovery.** The simpler source-cache branch passes the master-controlled rebuild and mixed-request checks after sharing its query source owner. The user accepted its documented first-use trade-off; -CI verification and integration are pending. +all CI checks passed and the baseline is merged. The first isolated recovery +experiment implements model-backed declaration inlays against that accepted +control. Its correctness, scaling and A/B results are recorded in the +[inlay recovery report](reports/2026-10-10/service-model-inlays.md). The user +accepted its snapshot-first mixed allocation-peak trade-off. Remaining recovery +candidates are shared semantic/hierarchy indices and outline caching, cursor-local +source lookup, and accurate authored section ranges, each measured independently. The record-backed representation replacement remains parked; its recurring rebuild and first-use costs are recorded in the [service cost review](reports/2026-10-10/service-authoring-cost-review.md). @@ -75,8 +81,9 @@ the media-type fixes and the type-walk work landed together: owner removes the old duplicate hover/structural tree. The accepted baseline's remaining first-use cost and post-query allocation trade-off are recorded in the [integration report](reports/2026-10-10/service-baseline-integration.md). - Model-backed declaration inlays and cursor-local source lookup are the next - separately measured recovery candidates. + Model-backed declaration inlays remove that first-use work from hints alone; + their accepted request-order peak trade-off is recorded in the inlay recovery + report. Cursor-local source lookup remains a separately measured candidate. ## 3. Potential future work diff --git a/docs/21-language-service.md b/docs/21-language-service.md index 344871d..aaf4c4b 100644 --- a/docs/21-language-service.md +++ b/docs/21-language-service.md @@ -56,13 +56,14 @@ the TCK's 1011 files. `parser:` limits. It keeps the YAML trees too, `retain_source=True`, only when an enabled lint rule reads them; no rule in the default set does. Decoders record the small syntax facts that `deprecated-schemas` reads (docs/18 § 6). -The first query that needs a file's tree composes it on demand. Hover, inlays, +The first query that needs a file's tree composes it on demand. Hover, folding and selection share the workspace's source owner when their input and composition policy agree (§ 4). Parsing does not populate that query cache or retain all producer trees. Snapshot-local hover indices keep requested trees; snapshots borrow the workspace's cache weakly rather than keep its other files alive. Without a live owner, a held snapshot can compose its own retained text. -It records the set of files it read: every retained text, every +Inlays read model facts and typed data without requesting a tree (§ 4.4). +A snapshot records the set of files it read: every retained text, every fragment, and every include it tried, found or not. It is built when a query first asks for it and kept until one of those files changes. A file that appears, a buffer opened or a file created, also drops every snapshot that @@ -350,6 +351,18 @@ returns hints only within the requested source range. It reuses hover's authored source sites and typed `DataNode` token index, and never resolves a name or repeats type matching in the adapter. +The decoder's `type_written` flag distinguishes explicit type syntax from +inference. Declaration hints read per-file authored model subjects and expression +spans, without composing source or populating source grammar/built-in indices. +Inferred hints need a real single-line key span; a fragment-root body span is not +an authored declaration key. Reference hints still use the authored expression +span in that fragment. + +Expanded YAML aliases keep their anchor's child spans. Declaration hints omit +those borrowed children, including nested object and array members, while keeping +any declaration hints authored at the anchor itself. This check reads model +placement and owned declaration edges; it does not build the source grammar. + A declaration without an authored type can show its inferred type, such as `[object]`. Explicit built-in types and multiple-inheritance lists are not repeated. Named type references show the underlying effective type diff --git a/docs/reports/2026-10-10/service-baseline-integration.md b/docs/reports/2026-10-10/service-baseline-integration.md index 812809b..8cfbf59 100644 --- a/docs/reports/2026-10-10/service-baseline-integration.md +++ b/docs/reports/2026-10-10/service-baseline-integration.md @@ -145,5 +145,43 @@ against that accepted baseline, without accepting record capture. ## 3. Selective recovery -Pending acceptance of the production baseline. The first planned experiment is -model-backed declaration inlays and `type_written`, without record capture. +All production-baseline CI checks passed, including Linux, Windows, pure-Python +YAML, TCK and benchmarks. [PR #2](https://github.com/deiteris/FastRAML/pull/2) +merged at `bb6941fce593082a4b38a33f4b346dc35268a8d9`. This exact revision is the +comparison control for the first recovery experiment. + +### 3.1 Model-backed declaration inlays + +Branch: `perf/service-model-inlays`, based on the accepted baseline. Objective: +remove the whole-file grammar/source construction that inlays currently request +just to identify authored types and declaration sites. This does not port shared +semantic indices, accurate outline sites or the record-backed representation. + +The decoder records whether a declaration supplied its type; clones preserve the +fact. A bare empty/null declaration is inferred, while an authored `type:` key +(including its null value) is explicit syntax. The existing hover subject/sites +index supplies declarations, with per-URI subject references to avoid scanning +the whole model for each queried file. Inferred labels require an authored +single-line key; fragment-root body spans must not become invented declaration +keys. Named-reference hints continue to use the model's type-expression span. + +Expected work: one boolean slot/assignment per shape and clone, one per-URI +subject reference per distinct authored site when hover is first populated, one +hint construction per declaration in the queried file/snapshot, then range +lookup. Inlays should make zero source compositions or grammar/built-in-token +index populations. Typed-value navigation remains lazy and shared as before. + +Acceptance compares `inlays`, both mixed request orders, ordinary parse/validate +and routine `large/service` against `bb6941f`. A reach test must forbid source +construction during both timed and allocation inlay passes. Behavior cases cover +explicit versus inferred syntax, multiple inheritance, cloning, included types +and template materializations. Preserve the remaining gates and record the +allocation cost of the new model flag, even when time stays within noise. + +The isolated port and measurements are now recorded in the +[inlay recovery report](service-model-inlays.md). Cold inlay time falls 27.4%, +peak 34.8% and retained allocation 30.2%; recurring rebuild time stays within +noise. A 10.0% snapshot-first mixed peak increase is attributed to query-order +allocation overlap, with final retention effectively unchanged. After the +shared-snapshot/order explanation, the user explicitly accepted that new trade-off +and instructed integration on 2026-10-10; normal CI verification gates the merge. diff --git a/docs/reports/2026-10-10/service-model-inlays-phase-metrics.json b/docs/reports/2026-10-10/service-model-inlays-phase-metrics.json new file mode 100644 index 0000000..03189b2 --- /dev/null +++ b/docs/reports/2026-10-10/service-model-inlays-phase-metrics.json @@ -0,0 +1,690 @@ +{ + "corpus": "hover", + "scale": 1, + "inputs": { + "files": 1, + "bytes": 311308, + "entry_bytes": 311308, + "focus_bytes": 311308 + }, + "detail_timers": false, + "mixed": false, + "source_first": false, + "platform": "Windows-10-10.0.19045-SP0", + "trees": [ + [ + "baseline", + "C:\\Sources\\pyRAML\\.claude\\worktrees\\service-source-simple" + ], + [ + "model-inlays", + "C:\\Sources\\pyRAML\\.claude\\worktrees\\service-model-inlays" + ] + ], + "results": { + "baseline": [ + { + "phases": [ + { + "change": 0.00035399990156292915, + "collect": 0.013006799854338169, + "snapshot": 0.17130300030112267, + "diagnostics": 1.600012183189392e-06, + "occurrences": 0.01887140003964305, + "total": 0.20353680010885 + }, + { + "change": 0.00041579967364668846, + "collect": 0.013568300288170576, + "snapshot": 0.16224819980561733, + "diagnostics": 1.4998950064182281e-06, + "occurrences": 0.019019300118088722, + "total": 0.19525309978052974 + }, + { + "change": 0.0004069996066391468, + "collect": 0.012496300041675568, + "snapshot": 0.16499580023810267, + "diagnostics": 9.997747838497162e-07, + "occurrences": 0.019017400220036507, + "total": 0.19691749988123775 + } + ], + "queries": [ + { + "outline.first": 0.03003360005095601, + "lenses.first": 0.0013667996972799301, + "inlays.first": 0.1542587000876665, + "folding.first": 0.015804599970579147, + "hover.first_after_auto": 1.6300007700920105e-05, + "outline.warm": 0.030866699758917093, + "inlays.warm": 9.470013901591301e-05, + "folding.warm": 0.011901100166141987, + "hover.warm": 1.3499986380338669e-05 + }, + { + "outline.first": 0.030701899901032448, + "lenses.first": 0.0013407999649643898, + "inlays.first": 0.15674180025234818, + "folding.first": 0.01629119971767068, + "hover.first_after_auto": 2.0499806851148605e-05, + "outline.warm": 0.033507800195366144, + "inlays.warm": 0.0001088995486497879, + "folding.warm": 0.012932800222188234, + "hover.warm": 1.9000377506017685e-05 + }, + { + "outline.first": 0.03134239977225661, + "lenses.first": 0.0014504999853670597, + "inlays.first": 0.15326920012012124, + "folding.first": 0.01534479996189475, + "hover.first_after_auto": 1.5600118786096573e-05, + "outline.warm": 0.0352956997230649, + "inlays.warm": 0.00010270019993185997, + "folding.warm": 0.011890999972820282, + "hover.warm": 1.4299992471933365e-05 + } + ], + "memory": { + "snapshot": { + "kept": 9627176, + "peak": 16383910 + }, + "diagnostics_occurrences": { + "kept": 11984222, + "peak": 16383910 + }, + "automatic_queries": { + "kept": 25839653, + "peak": 30600909 + }, + "sparse_hovers": { + "kept": 25840085, + "peak": 30600909 + } + }, + "dimensions": { + "read_files": 1, + "shapes": 7200, + "endpoints": 400, + "captured_files": 0, + "retained_source_records": 0 + }, + "counts": { + "v1.snapshot.yaml_composes": 1, + "input.b0de2b90a79d": 2, + "v1.snapshot.converted_nodes": 32007, + "v1.automatic.yaml_composes": 1, + "v1.automatic.converted_nodes": 32007, + "v2.snapshot.yaml_composes": 1, + "input.7cd2b060b6fd": 2, + "v2.snapshot.converted_nodes": 32007, + "v2.automatic.yaml_composes": 1, + "v2.automatic.converted_nodes": 32007, + "v3.snapshot.yaml_composes": 1, + "input.b2c9a97bfc7a": 2, + "v3.snapshot.converted_nodes": 32007, + "v3.automatic.yaml_composes": 1, + "v3.automatic.converted_nodes": 32007, + "broken.snapshot_composes": 1, + "broken.three_source_request_composes": 1 + }, + "python": "3.12.13 (main, May 10 2026, 19:35:37) [MSC v.1944 64 bit (AMD64)]", + "backend": "libyaml", + "import": "C:\\Sources\\pyRAML\\.claude\\worktrees\\service-source-simple\\fastraml\\__init__.py" + }, + { + "phases": [ + { + "change": 0.00035159988328814507, + "collect": 0.012389299925416708, + "snapshot": 0.15698739979416132, + "diagnostics": 2.100132405757904e-06, + "occurrences": 0.021853200159966946, + "total": 0.19158359989523888 + }, + { + "change": 0.0004306999035179615, + "collect": 0.013914200011640787, + "snapshot": 0.16586640011519194, + "diagnostics": 1.7997808754444122e-06, + "occurrences": 0.01705110026523471, + "total": 0.19726420007646084 + }, + { + "change": 0.00042559998109936714, + "collect": 0.014231100212782621, + "snapshot": 0.15821809973567724, + "diagnostics": 1.1003576219081879e-06, + "occurrences": 0.017208999954164028, + "total": 0.19008490024134517 + } + ], + "queries": [ + { + "outline.first": 0.03164030006155372, + "lenses.first": 0.001641799695789814, + "inlays.first": 0.1517718997783959, + "folding.first": 0.016369699966162443, + "hover.first_after_auto": 1.7399899661540985e-05, + "outline.warm": 0.03163710003718734, + "inlays.warm": 0.00010229973122477531, + "folding.warm": 0.011863300111144781, + "hover.warm": 1.2100208550691605e-05 + }, + { + "outline.first": 0.029142200015485287, + "lenses.first": 0.0013697999529540539, + "inlays.first": 0.14390150038525462, + "folding.first": 0.015065299812704325, + "hover.first_after_auto": 1.4099758118391037e-05, + "outline.warm": 0.02941670035943389, + "inlays.warm": 9.289989247918129e-05, + "folding.warm": 0.011653599794954062, + "hover.warm": 1.2400094419717789e-05 + }, + { + "outline.first": 0.02961320010945201, + "lenses.first": 0.001409000251442194, + "inlays.first": 0.14207250019535422, + "folding.first": 0.015125500038266182, + "hover.first_after_auto": 1.3899989426136017e-05, + "outline.warm": 0.02918960014358163, + "inlays.warm": 9.300000965595245e-05, + "folding.warm": 0.011605000123381615, + "hover.warm": 1.0200310498476028e-05 + } + ], + "memory": { + "snapshot": { + "kept": 9625038, + "peak": 16383969 + }, + "diagnostics_occurrences": { + "kept": 11982319, + "peak": 16383969 + }, + "automatic_queries": { + "kept": 25837891, + "peak": 30599065 + }, + "sparse_hovers": { + "kept": 25838323, + "peak": 30599065 + } + }, + "dimensions": { + "read_files": 1, + "shapes": 7200, + "endpoints": 400, + "captured_files": 0, + "retained_source_records": 0 + }, + "counts": { + "v1.snapshot.yaml_composes": 1, + "input.b0de2b90a79d": 2, + "v1.snapshot.converted_nodes": 32007, + "v1.automatic.yaml_composes": 1, + "v1.automatic.converted_nodes": 32007, + "v2.snapshot.yaml_composes": 1, + "input.7cd2b060b6fd": 2, + "v2.snapshot.converted_nodes": 32007, + "v2.automatic.yaml_composes": 1, + "v2.automatic.converted_nodes": 32007, + "v3.snapshot.yaml_composes": 1, + "input.b2c9a97bfc7a": 2, + "v3.snapshot.converted_nodes": 32007, + "v3.automatic.yaml_composes": 1, + "v3.automatic.converted_nodes": 32007, + "broken.snapshot_composes": 1, + "broken.three_source_request_composes": 1 + }, + "python": "3.12.13 (main, May 10 2026, 19:35:37) [MSC v.1944 64 bit (AMD64)]", + "backend": "libyaml", + "import": "C:\\Sources\\pyRAML\\.claude\\worktrees\\service-source-simple\\fastraml\\__init__.py" + }, + { + "phases": [ + { + "change": 0.0003551999107003212, + "collect": 0.012495799921452999, + "snapshot": 0.15563159994781017, + "diagnostics": 1.200009137392044e-06, + "occurrences": 0.01638000039383769, + "total": 0.18486380018293858 + }, + { + "change": 0.000412600114941597, + "collect": 0.011598299723118544, + "snapshot": 0.159115900285542, + "diagnostics": 1.8998980522155762e-06, + "occurrences": 0.016906000208109617, + "total": 0.18803470022976398 + }, + { + "change": 0.00042290007695555687, + "collect": 0.013815500307828188, + "snapshot": 0.1590086999349296, + "diagnostics": 1.3997778296470642e-06, + "occurrences": 0.01655249996110797, + "total": 0.18980100005865097 + } + ], + "queries": [ + { + "outline.first": 0.029083999805152416, + "lenses.first": 0.001335000153630972, + "inlays.first": 0.1422079000622034, + "folding.first": 0.01504219975322485, + "hover.first_after_auto": 1.4200340956449509e-05, + "outline.warm": 0.029282900039106607, + "inlays.warm": 9.769992902874947e-05, + "folding.warm": 0.011528399772942066, + "hover.warm": 1.2100208550691605e-05 + }, + { + "outline.first": 0.030026200227439404, + "lenses.first": 0.0013417000882327557, + "inlays.first": 0.14204619964584708, + "folding.first": 0.014940300025045872, + "hover.first_after_auto": 1.3400334864854813e-05, + "outline.warm": 0.02870139991864562, + "inlays.warm": 9.180000051856041e-05, + "folding.warm": 0.01149709988385439, + "hover.warm": 1.00000761449337e-05 + }, + { + "outline.first": 0.02922269981354475, + "lenses.first": 0.001395100262016058, + "inlays.first": 0.1458729999139905, + "folding.first": 0.014974900055676699, + "hover.first_after_auto": 2.550007775425911e-05, + "outline.warm": 0.02982780011370778, + "inlays.warm": 9.180000051856041e-05, + "folding.warm": 0.011525500100106001, + "hover.warm": 1.0999850928783417e-05 + } + ], + "memory": { + "snapshot": { + "kept": 9618740, + "peak": 16383969 + }, + "diagnostics_occurrences": { + "kept": 11976068, + "peak": 16383969 + }, + "automatic_queries": { + "kept": 25832204, + "peak": 30593284 + }, + "sparse_hovers": { + "kept": 25832636, + "peak": 30593284 + } + }, + "dimensions": { + "read_files": 1, + "shapes": 7200, + "endpoints": 400, + "captured_files": 0, + "retained_source_records": 0 + }, + "counts": { + "v1.snapshot.yaml_composes": 1, + "input.b0de2b90a79d": 2, + "v1.snapshot.converted_nodes": 32007, + "v1.automatic.yaml_composes": 1, + "v1.automatic.converted_nodes": 32007, + "v2.snapshot.yaml_composes": 1, + "input.7cd2b060b6fd": 2, + "v2.snapshot.converted_nodes": 32007, + "v2.automatic.yaml_composes": 1, + "v2.automatic.converted_nodes": 32007, + "v3.snapshot.yaml_composes": 1, + "input.b2c9a97bfc7a": 2, + "v3.snapshot.converted_nodes": 32007, + "v3.automatic.yaml_composes": 1, + "v3.automatic.converted_nodes": 32007, + "broken.snapshot_composes": 1, + "broken.three_source_request_composes": 1 + }, + "python": "3.12.13 (main, May 10 2026, 19:35:37) [MSC v.1944 64 bit (AMD64)]", + "backend": "libyaml", + "import": "C:\\Sources\\pyRAML\\.claude\\worktrees\\service-source-simple\\fastraml\\__init__.py" + } + ], + "model-inlays": [ + { + "phases": [ + { + "change": 0.0003483002074062824, + "collect": 0.012836799956858158, + "snapshot": 0.15877159964293242, + "diagnostics": 1.400243490934372e-06, + "occurrences": 0.01784879993647337, + "total": 0.18980689998716116 + }, + { + "change": 0.0004428001120686531, + "collect": 0.01281779957935214, + "snapshot": 0.16061530029401183, + "diagnostics": 1.7997808754444122e-06, + "occurrences": 0.018542600329965353, + "total": 0.19242030009627342 + }, + { + "change": 0.00040400028228759766, + "collect": 0.013918299693614244, + "snapshot": 0.16185700008645654, + "diagnostics": 1.200009137392044e-06, + "occurrences": 0.01756170019507408, + "total": 0.19374220026656985 + } + ], + "queries": [ + { + "outline.first": 0.03015019977465272, + "lenses.first": 0.0013476000167429447, + "inlays.first": 0.05040870001539588, + "folding.first": 0.07552629988640547, + "hover.first_after_auto": 0.03375460021197796, + "outline.warm": 0.030985400080680847, + "inlays.warm": 9.689992293715477e-05, + "folding.warm": 0.01166009996086359, + "hover.warm": 1.0999850928783417e-05 + }, + { + "outline.first": 0.02942320005968213, + "lenses.first": 0.0013283002190291882, + "inlays.first": 0.05154359992593527, + "folding.first": 0.0771862999536097, + "hover.first_after_auto": 0.03445399971678853, + "outline.warm": 0.03309000004082918, + "inlays.warm": 0.00010379962623119354, + "folding.warm": 0.013205799739807844, + "hover.warm": 1.4800112694501877e-05 + }, + { + "outline.first": 0.03033600002527237, + "lenses.first": 0.0014142002910375595, + "inlays.first": 0.05079880030825734, + "folding.first": 0.07602539984509349, + "hover.first_after_auto": 0.03289539972320199, + "outline.warm": 0.030575500335544348, + "inlays.warm": 9.809993207454681e-05, + "folding.warm": 0.012516699731349945, + "hover.warm": 1.3300217688083649e-05 + } + ], + "memory": { + "snapshot": { + "kept": 9689719, + "peak": 16383918 + }, + "diagnostics_occurrences": { + "kept": 12047000, + "peak": 16383918 + }, + "automatic_queries": { + "kept": 23240843, + "peak": 33678909 + }, + "sparse_hovers": { + "kept": 25867423, + "peak": 33678909 + } + }, + "dimensions": { + "read_files": 1, + "shapes": 7200, + "endpoints": 400, + "captured_files": 0, + "retained_source_records": 0 + }, + "counts": { + "v1.snapshot.yaml_composes": 1, + "input.b0de2b90a79d": 2, + "v1.snapshot.converted_nodes": 32007, + "v1.automatic.yaml_composes": 1, + "v1.automatic.converted_nodes": 32007, + "v2.snapshot.yaml_composes": 1, + "input.7cd2b060b6fd": 2, + "v2.snapshot.converted_nodes": 32007, + "v2.automatic.yaml_composes": 1, + "v2.automatic.converted_nodes": 32007, + "v3.snapshot.yaml_composes": 1, + "input.b2c9a97bfc7a": 2, + "v3.snapshot.converted_nodes": 32007, + "v3.automatic.yaml_composes": 1, + "v3.automatic.converted_nodes": 32007, + "broken.snapshot_composes": 1, + "broken.three_source_request_composes": 1 + }, + "python": "3.12.13 (main, May 10 2026, 19:35:37) [MSC v.1944 64 bit (AMD64)]", + "backend": "libyaml", + "import": "C:\\Sources\\pyRAML\\.claude\\worktrees\\service-model-inlays\\fastraml\\__init__.py" + }, + { + "phases": [ + { + "change": 0.0004841000773012638, + "collect": 0.01583729963749647, + "snapshot": 0.16017170017585158, + "diagnostics": 1.4998950064182281e-06, + "occurrences": 0.016534500289708376, + "total": 0.1930291000753641 + }, + { + "change": 0.0004133000038564205, + "collect": 0.01269460003823042, + "snapshot": 0.16011400008574128, + "diagnostics": 1.600012183189392e-06, + "occurrences": 0.01739259995520115, + "total": 0.19061610009521246 + }, + { + "change": 0.000892499927431345, + "collect": 0.013632100075483322, + "snapshot": 0.15939819999039173, + "diagnostics": 9.0012326836586e-07, + "occurrences": 0.016515399795025587, + "total": 0.19043909991160035 + } + ], + "queries": [ + { + "outline.first": 0.03128980007022619, + "lenses.first": 0.0016167000867426395, + "inlays.first": 0.05480969976633787, + "folding.first": 0.08616989990696311, + "hover.first_after_auto": 0.03448030026629567, + "outline.warm": 0.0315635995939374, + "inlays.warm": 0.0001105000264942646, + "folding.warm": 0.012500799726694822, + "hover.warm": 1.800013706088066e-05 + }, + { + "outline.first": 0.03088079998269677, + "lenses.first": 0.0014346004463732243, + "inlays.first": 0.050342199858278036, + "folding.first": 0.07512659998610616, + "hover.first_after_auto": 0.033044100273400545, + "outline.warm": 0.030587899964302778, + "inlays.warm": 0.00010180007666349411, + "folding.warm": 0.013231299817562103, + "hover.warm": 1.930026337504387e-05 + }, + { + "outline.first": 0.030108400154858828, + "lenses.first": 0.0014154999516904354, + "inlays.first": 0.05031870026141405, + "folding.first": 0.07451419997960329, + "hover.first_after_auto": 0.03290400002151728, + "outline.warm": 0.03089809976518154, + "inlays.warm": 0.00010700011625885963, + "folding.warm": 0.011692400090396404, + "hover.warm": 1.1899974197149277e-05 + } + ], + "memory": { + "snapshot": { + "kept": 9685677, + "peak": 16383918 + }, + "diagnostics_occurrences": { + "kept": 12042770, + "peak": 16383918 + }, + "automatic_queries": { + "kept": 23237365, + "peak": 33675431 + }, + "sparse_hovers": { + "kept": 25863898, + "peak": 33675431 + } + }, + "dimensions": { + "read_files": 1, + "shapes": 7200, + "endpoints": 400, + "captured_files": 0, + "retained_source_records": 0 + }, + "counts": { + "v1.snapshot.yaml_composes": 1, + "input.b0de2b90a79d": 2, + "v1.snapshot.converted_nodes": 32007, + "v1.automatic.yaml_composes": 1, + "v1.automatic.converted_nodes": 32007, + "v2.snapshot.yaml_composes": 1, + "input.7cd2b060b6fd": 2, + "v2.snapshot.converted_nodes": 32007, + "v2.automatic.yaml_composes": 1, + "v2.automatic.converted_nodes": 32007, + "v3.snapshot.yaml_composes": 1, + "input.b2c9a97bfc7a": 2, + "v3.snapshot.converted_nodes": 32007, + "v3.automatic.yaml_composes": 1, + "v3.automatic.converted_nodes": 32007, + "broken.snapshot_composes": 1, + "broken.three_source_request_composes": 1 + }, + "python": "3.12.13 (main, May 10 2026, 19:35:37) [MSC v.1944 64 bit (AMD64)]", + "backend": "libyaml", + "import": "C:\\Sources\\pyRAML\\.claude\\worktrees\\service-model-inlays\\fastraml\\__init__.py" + }, + { + "phases": [ + { + "change": 0.00035850005224347115, + "collect": 0.012868999969214201, + "snapshot": 0.1652979999780655, + "diagnostics": 1.4998950064182281e-06, + "occurrences": 0.017011700198054314, + "total": 0.1955387000925839 + }, + { + "change": 0.0004213000647723675, + "collect": 0.013499299995601177, + "snapshot": 0.1775956000201404, + "diagnostics": 1.80024653673172e-06, + "occurrences": 0.0177110000513494, + "total": 0.2092290003784001 + }, + { + "change": 0.00041510025039315224, + "collect": 0.014422900043427944, + "snapshot": 0.17142869997769594, + "diagnostics": 9.997747838497162e-07, + "occurrences": 0.01806890033185482, + "total": 0.2043366003781557 + } + ], + "queries": [ + { + "outline.first": 0.030308700166642666, + "lenses.first": 0.001349099911749363, + "inlays.first": 0.052336900029331446, + "folding.first": 0.07405529962852597, + "hover.first_after_auto": 0.032701299991458654, + "outline.warm": 0.0300671998411417, + "inlays.warm": 9.669968858361244e-05, + "folding.warm": 0.011727000121027231, + "hover.warm": 1.1499971151351929e-05 + }, + { + "outline.first": 0.03219290031120181, + "lenses.first": 0.0015094000846147537, + "inlays.first": 0.052826700266450644, + "folding.first": 0.07425360009074211, + "hover.first_after_auto": 0.03282469976693392, + "outline.warm": 0.029800199903547764, + "inlays.warm": 9.509967640042305e-05, + "folding.warm": 0.011711400002241135, + "hover.warm": 9.799841791391373e-06 + }, + { + "outline.first": 0.031831400003284216, + "lenses.first": 0.0015290002338588238, + "inlays.first": 0.05159540008753538, + "folding.first": 0.07504560006782413, + "hover.first_after_auto": 0.033148800022900105, + "outline.warm": 0.030403800308704376, + "inlays.warm": 9.569991379976273e-05, + "folding.warm": 0.011812100186944008, + "hover.warm": 1.3799872249364853e-05 + } + ], + "memory": { + "snapshot": { + "kept": 9694466, + "peak": 16383918 + }, + "diagnostics_occurrences": { + "kept": 12051653, + "peak": 16383918 + }, + "automatic_queries": { + "kept": 23245296, + "peak": 33683362 + }, + "sparse_hovers": { + "kept": 25871782, + "peak": 33683362 + } + }, + "dimensions": { + "read_files": 1, + "shapes": 7200, + "endpoints": 400, + "captured_files": 0, + "retained_source_records": 0 + }, + "counts": { + "v1.snapshot.yaml_composes": 1, + "input.b0de2b90a79d": 2, + "v1.snapshot.converted_nodes": 32007, + "v1.automatic.yaml_composes": 1, + "v1.automatic.converted_nodes": 32007, + "v2.snapshot.yaml_composes": 1, + "input.7cd2b060b6fd": 2, + "v2.snapshot.converted_nodes": 32007, + "v2.automatic.yaml_composes": 1, + "v2.automatic.converted_nodes": 32007, + "v3.snapshot.yaml_composes": 1, + "input.b2c9a97bfc7a": 2, + "v3.snapshot.converted_nodes": 32007, + "v3.automatic.yaml_composes": 1, + "v3.automatic.converted_nodes": 32007, + "broken.snapshot_composes": 1, + "broken.three_source_request_composes": 1 + }, + "python": "3.12.13 (main, May 10 2026, 19:35:37) [MSC v.1944 64 bit (AMD64)]", + "backend": "libyaml", + "import": "C:\\Sources\\pyRAML\\.claude\\worktrees\\service-model-inlays\\fastraml\\__init__.py" + } + ] + } +} diff --git a/docs/reports/2026-10-10/service-model-inlays.md b/docs/reports/2026-10-10/service-model-inlays.md new file mode 100644 index 0000000..2da4712 --- /dev/null +++ b/docs/reports/2026-10-10/service-model-inlays.md @@ -0,0 +1,168 @@ +# Model-backed declaration inlay experiment + +Date: 2026-10-10. Branch: `perf/service-model-inlays`, starting at the accepted +baseline `bb6941fce593082a4b38a33f4b346dc35268a8d9`. Candidate measurements use +the working-tree changes on that branch. This is the first isolated experiment +in the [recovery plan](../../research/2026-10-10/service-recovery-plan.md). + +## 1. Goal, workload and cache boundaries + +The goal is to remove source composition and whole-file grammar construction from +declaration inlays while preserving normal rebuild cost. The planned scope and +acceptance criteria preceded implementation in the +[integration record](service-baseline-integration.md#31-model-backed-declaration-inlays). + +The focused workload is a cold workspace and validated/unwrapped snapshot followed +by a whole-file inlay request. Its generated input is one 311,308-byte, +13,204-line file: 400 families, 2,000 named types, 400 annotation types, +400 resources and 7,200 shapes. The snapshot and hints remain live for retained +allocation. This establishes total cold inlay cost, not one already-parsed viewport +request. The two mixed workloads use the same input over three comment edits and +the ordered request sequence in docs/12 § 4.2. Their times cover all three cycles; +they are representative service-core scenarios, not observed editor traces. + +The decoder records `BaseShape.type_written`; clones preserve it. Empty/null +declarations infer their kind, while an explicit `type:` or accepted `schema:` +field counts as authored even with a null value. The flag is an additional boolean +slot on every shape, with no source-tree owner or query cache populated by parsing. + +On the first hover-index request, the existing authored-site population also keeps +per-URI subject references, selecting the first materialization at each site. +One placement check per shape detects expanded YAML aliases. A temporary visited-ID +set excludes their owned declaration descendants from inlays, without following +named-reference edges or suppressing the actual declaration at an anchor. This +walk only occurs for borrowed children and releases its set after construction. + +The first inlay request for a URI builds and sorts declaration hints once. Later +ranges use the existing anchor index. The typed-data index still populates once +per snapshot on demand. Both caches die with that snapshot; a root edit rebuilds +them. Inlays compose no source and populate no source grammar/built-in-token index. +Folding or hover still populates the shared source owner on its own first use; +hover's grammar remains snapshot-local. The sharing/invalidation policy of the +accepted source cache is unchanged. + +## 2. Correctness and reach + +Tests cover explicit and inferred syntax, bare and explicit null/empty values, +multiple-inheritance sequences, both clone operations, included fragments, +synthetic fragment keys, keyless inline bodies and template materializations. +YAML aliases originating in opaque data are checked through nested object and +array declarations; a separate case preserves hints at a real declaration anchor. + +The inlay benchmark reach test fails if `Hover._node` or `Hover._source_keys` is +called. It checks both timed and allocation passes at two corpus sizes, while +requiring inferred, typed-data and inherited-constraint labels. The mixed reach +tests still require one semantic and one shared query composition per version. +Removing inlay composition moves the query composition to folding; it does not +remove source work from a sequence that also asks for folding/hover. + +The final Windows gate passes: Ruff checks and formatting, strict mypy, and +**5,861 passed, 68 skipped, 1 xfailed**, with the pinned TCK submodule initialized. +The focused inlay/decoder/corpus/mixed-reach suites also pass with the forced +pure-Python YAML plugin: **305 passed**. The accompanying pull request tracks +platform/CI verification, required before integration. + +## 3. Controlled A/B results + +Windows AMD64, Python 3.12, libyaml. `bench ab` used identical candidate-generated +inputs, three alternating rounds and the best of three untraced repetitions per +round. Allocation ran separately with the harness collector policy. A time change +within the reported round spread is not treated as an improvement or regression. + +| Workload/configuration | Baseline ms | Candidate ms | Time/noise | Peak MB, baseline → candidate | Kept MB, baseline → candidate | +|---|---:|---:|---|---:|---:| +| `inlays/unwrap` | 347.1 | 251.9 | -27.4%, noise 4.9% | 30.022 → 19.569 (-34.8%) | 25.987 → 18.131 (-30.2%) | +| `service-session/unwrap` | 1320.9 | 1314.2 | within 4.5% noise | 30.621 → 33.691 (+10.0%) | 25.856 → 25.879 (+0.1%) | +| `service-source-first/unwrap` | 1379.3 | 1346.6 | within 4.9% noise | 28.115 → 28.137 (+0.1%) | 25.856 → 25.880 (+0.1%) | +| `large/parse` | 342.9 | 339.5 | within 9.2% noise | 20.073 → 20.233 (+0.8%, within allocation spread) | 19.608 → 19.767 (+0.8%, within allocation spread) | +| `large/unwrap+validate` | 391.6 | 407.7 | within 7.3% noise | 20.655 → 20.814 (+0.8%) | 19.714 → 19.873 (+0.8%) | +| `large/service` | 479.0 | 488.0 | within 13.0% noise | 35.643 → 35.803 (+0.4%) | 28.464 → 28.623 (+0.6%) | + +The ordinary model-retaining rows expose about 159 KB of additional retained +allocation from adding the flag to shapes and their copies. In the inlay-only +scenario, avoiding source and grammar dominates that cost. When all mixed queries +eventually populate their indices, final retention is effectively unchanged. + +Commands, from this worktree, with `TEMP` set to the approved temporary directory: + +```powershell +$env:TEMP = 'C:\Users\Snek\AppData\Local\Temp\opencode' +uv run python -m bench ab bb6941f --bench inlays --bench service-session --bench service-source-first --config unwrap --rounds 3 --repeat 3 +uv run python -m bench ab bb6941f --bench large --config parse --config unwrap+validate --config service --rounds 3 --repeat 3 +uv run python -m bench linearity --bench inlays --bench service-session --bench service-source-first --repeat 9 +``` + +## 4. First-use costs and the mixed peak + +The existing unprofiled phase driver alternated three rounds of three requests. +Its baseline worktree is `refactor/service-source-simple` at `42c434e`, whose +`fastraml/` and `bench/` have an empty diff from `bb6941f`. Raw rounds, input +dimensions and counters are in [service-model-inlays-phase-metrics.json](service-model-inlays-phase-metrics.json). +The following are per-phase minima, potentially from different repetitions; +they must not be added into a claimed total or interpreted as an A/B noise test. + +| Boundary in the snapshot-first sequence | Baseline ms | Candidate ms | +|---|---:|---:| +| Recurring edit through diagnostics/occurrences, best complete run | 184.9 | 189.8 | +| First 120-line viewport inlays, after outline/lenses | 142.0 | 50.3 | +| First fold, after inlays | 14.9 | 74.1 | +| First hover, after inlays/folding | 0.013 | 32.7 | +| Warm viewport inlays | 0.092 | 0.095 | +| Warm fold | 11.5 | 11.7 | +| Warm hover | 0.010 | 0.010 | + +The best recurring totals differ by 2.7%, within the probe's 5.6% round spread. +The candidate's earlier inlay response leaves source composition to folding and +grammar construction to hover. A fast inlay alone is not a mixed-sequence speedup; +the controlled mixed A/B time remains within noise. + +The allocation probe [service_inlay_allocation_probe.py](service_inlay_allocation_probe.py) +wraps the same mixed driver and records request-phase traced peaks with the +collector paused. It confirmed that the snapshot-first peak moves from the first +inlay request (30.61 MB baseline) to the first source/folding composition (33.68 MB +candidate). That composition begins with 17.68 MB already allocated for the +model, occurrences and lazy inlay/data caches, and peaks while the composer and +converted query tree coexist. In the baseline, composition precedes creation of +those inlay/data caches. This is a request-order allocation overlap, not a second +retained source owner. The probe's own rows add small diagnostic allocations, so +the ordinary A/B figures are the acceptance numbers. + +The phase driver's post-collection retention after automatic queries is 25.832 MB +baseline versus 23.237 MB candidate; after sparse hovers it is 25.833 versus +25.864 MB. Its automatic-query boundary includes folding but excludes hover, +so the candidate has not yet populated grammar at that point. Source-first +requests compose before inlays, and their controlled peak is essentially unchanged. + +Reproduce phases and attribution: + +```powershell +uv run python docs/reports/2026-10-10/service_cost_probe.py --corpus hover --tree "baseline=C:\Sources\pyRAML\.claude\worktrees\service-source-simple" --tree "model-inlays=C:\Sources\pyRAML\.claude\worktrees\service-model-inlays" --rounds 3 --repeat 3 --output docs/reports/2026-10-10/service-model-inlays-phase-metrics.json +# Run this absolute driver path once with each comparison worktree as cwd. +uv run python C:/Sources/pyRAML/.claude/worktrees/service-model-inlays/docs/reports/2026-10-10/service_inlay_allocation_probe.py +``` + +## 5. Scaling and decision + +Final linearity, best of nine, full versus half size: + +| Workload | Full / half ms | Normalized time | Peak | Retained | +|---|---:|---:|---:|---:| +| `inlays` | 245.9 / 125.1 | 0.983 | 0.985 | 0.997 | +| `service-session` | 1296.5 / 662.6 | 0.978 | 0.991 | 0.998 | +| `service-source-first` | 1364.7 / 693.2 | 0.984 | 0.988 | 0.998 | + +Earlier source-first runs failed with normalized time 1.876 at three repeats and +0.459 at five repeats, while allocation ratios stayed stable. Full/half timings +swung in opposite directions; the final higher-repeat run passes all three. +These failures remain part of the measurement record rather than being presented +as an uninterrupted green run. + +Decision: retain this bounded recovery. It meets the model-only inlay goal, +improves cold inlay time/allocation, and does not show a repeatable recurring +rebuild penalty. After the shared-snapshot ownership and construction-order +explanation, the user explicitly accepted the **+10.0% snapshot-first mixed +peak**, about 3.07 MB on this corpus, and instructed integration on 2026-10-10: +“Ok, let's say this is acceptable, proceed”. This is acceptance of the measured +peak with the collector paused, not a claim of 10% higher steady editor memory. +Normal platform/CI verification remains the merge gate. No further optimization +is proposed here. diff --git a/docs/reports/2026-10-10/service_inlay_allocation_probe.py b/docs/reports/2026-10-10/service_inlay_allocation_probe.py new file mode 100644 index 0000000..6ff1fb4 --- /dev/null +++ b/docs/reports/2026-10-10/service_inlay_allocation_probe.py @@ -0,0 +1,84 @@ +"""Attribute mixed-scenario allocation peaks to request phases, not elapsed time. + +Run this driver in each comparison tree with the same Python interpreter. Its +collector policy and request/result lifetimes match bench.service_session and +bench.harness. The wrappers only record traced allocation; this is diagnostic +evidence, never an acceptance timing. +""" + +from __future__ import annotations + +import argparse +import gc +import json +import sys +import tempfile +import tracemalloc +from contextlib import ExitStack +from pathlib import Path +from unittest.mock import patch + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--source-first', action='store_true') + args = parser.parse_args() + sys.path.insert(0, str(Path.cwd())) + from bench import corpus + from bench.service_session import exercise, prepare + from fastraml.service import inlays, outline, queries + from fastraml.service.workspace import Workspace + + with tempfile.TemporaryDirectory(prefix='inlay-allocation-') as directory: + prepared = prepare(corpus.write_hover(Path(directory), family_count=400)) + # Warm configuration/imports, as the ordinary harness does before tracing. + exercise(prepared, source_first=args.source_first) + gc.collect() + rows = [] + version = [0] + + def watched(name, operation): + def request(*arguments, **kwargs): + before = tracemalloc.get_traced_memory()[0] + tracemalloc.reset_peak() + result = operation(*arguments, **kwargs) + current, peak = tracemalloc.get_traced_memory() + rows.append({'version': version[0], 'request': name, + 'before_MB': before / 1e6, 'after_MB': current / 1e6, + 'peak_MB': peak / 1e6}) + return result + return request + + change = Workspace.change + + def changed(workspace, uri, text, new_version): + version[0] = new_version + return change(workspace, uri, text, new_version) + + with ExitStack() as stack: + stack.enter_context(patch.object(Workspace, 'change', changed)) + for owner, method, name in ( + (Workspace, '_parse', 'snapshot'), + (outline, 'document_symbols', 'outline'), + (inlays, 'inlay_hints', 'inlays'), + (Workspace, 'source', 'source'), + (queries, 'hover', 'hover'), + ): + stack.enter_context(patch.object(owner, method, watched(name, getattr(owner, method)))) + enabled = gc.isenabled() + gc.disable() + tracemalloc.start() + try: + result = exercise(prepared, source_first=args.source_first) + gc.collect() + kept = tracemalloc.get_traced_memory()[0] + assert result[1]['snapshots'] == 3 + finally: + tracemalloc.stop() + if enabled: + gc.enable() + print(json.dumps({'source_first': args.source_first, 'kept_MB': kept / 1e6, 'requests': rows}, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/docs/research/2026-10-10/service-recovery-plan.md b/docs/research/2026-10-10/service-recovery-plan.md index ca4fb2b..c2e8ef9 100644 --- a/docs/research/2026-10-10/service-recovery-plan.md +++ b/docs/research/2026-10-10/service-recovery-plan.md @@ -127,11 +127,18 @@ Current execution: (5,810 passed, 67 skipped, 1 xfailed) and both mixed-workload linearity checks. All GitHub checks passed, including Ubuntu, TCK and benchmarks; PR #1 merged into master as `d2b8a02`. -- Stage B: duplicate hover/workspace ownership is fixed and locally verified. +- Stage B: all CI checks passed and PR #2 merged into master at `bb6941f`. The user explicitly accepted the remaining first-use and +5.4% post-query - allocation trade-off with “Merge, then recover”; GitHub verification and the - production merge are next. Results and the decision are in the integration report. -- Stage C: pending an accepted baseline. + allocation trade-off with “Merge, then recover”. The integration report records + that decision; the parked representation has not been merged. +- Stage C: `perf/service-model-inlays` starts from `bb6941f`, with the first + model-backed inlay/explicit-type experiment documented before implementation. + The bounded port now has correctness, reach, A/B and scaling evidence in the + [inlay report](../../reports/2026-10-10/service-model-inlays.md). Its snapshot-first + mixed allocation peak adds 10.0%; after the ownership/order explanation the user + explicitly accepted that trade-off and instructed integration on 2026-10-10. + The normal CI/platform gate applies before merge. Remaining recovery candidates + stay separate experiments against the integrated baseline. Update these entries as each stage completes. Keep measurements in the dated report, current pending work in docs/15, and execution history here. diff --git a/fastraml/service/hover.py b/fastraml/service/hover.py index ea82472..48138e1 100644 --- a/fastraml/service/hover.py +++ b/fastraml/service/hover.py @@ -35,7 +35,7 @@ from fastraml.service.source import original_tree from fastraml.service.text import Lines from fastraml.types.base import BaseShape, Parameter, facets_of -from fastraml.types.complex_ import ObjectShape, UnknownShape +from fastraml.types.complex_ import ArrayShape, ObjectShape, UnionShape, UnknownShape from fastraml.types.examples import examples_of from fastraml.types.expressions import Array, Optional_, Primitive, Union, parse_expression from fastraml.types.jsonschema_ import JsonShape @@ -45,7 +45,7 @@ from fastraml.views import authored from fastraml.views.occurrences import DECLARATION_KINDS, Kind, Link, Occurrence, Occurrences, Role from fastraml.views.render import type_name -from fastraml.yamlnode import TAG_INCLUDE, TAG_STR, Node, NodeKind, backend_name, compose, pairs +from fastraml.yamlnode import TAG_INCLUDE, TAG_STR, Node, backend_name, compose if TYPE_CHECKING: from collections.abc import Iterable, Iterator @@ -78,6 +78,24 @@ class _Subject: required: bool | None = None +def _declaration_children(base: BaseShape) -> Iterator[BaseShape]: + """Owned declaration children, without following named-reference edges.""" + if base.alias is not None: + return + if isinstance(base.shape, ObjectShape): + yield from (prop.base for _, prop in authored.properties(base)) + yield from (prop.base for _, prop in authored.pattern_properties(base)) + yield from (prop.base for _, prop in authored.facets(base)) + if isinstance(base.shape, ArrayShape): + item = authored.items(base) + if item is not None: + yield item + if isinstance(base.shape, UnionShape): + yield from ( + member for member in base.shape.any_of or () if authored.wrote(base, member.location, member.key_pos) + ) + + def _code(text: str) -> str: """An inline code span that cannot be closed by the author's text.""" if len(text) > _INLINE_LIMIT: @@ -91,6 +109,7 @@ class Hover: __slots__ = ( '_ambiguous_sites', + '_by_uri', '_contexts', '_contexts_ready', '_data', @@ -133,6 +152,7 @@ def __init__( # noqa: PLR0913 - the snapshot's borrowed source owner and compos self._source_generation = source_generation self._source_backend = source_backend self._subjects: dict[int, _Subject] = {} + self._by_uri: dict[str, list[_Subject]] = {} self._sites: dict[tuple[str, int, int], _Subject] = {} self._mappings: dict[tuple[str, int, int], _Subject] = {} self._keys: dict[str, list[Key]] = {} @@ -168,21 +188,32 @@ def _put(self, subject: _Subject) -> None: if key.is_known: site = (entity.location, key.line, key.column) previous = self._sites.setdefault(site, subject) + if previous is subject: + self._by_uri.setdefault(entity.location, []).append(subject) if previous.entity.id != entity.id: self._ambiguous_sites.add(site) + else: + self._by_uri.setdefault(entity.location, []).append(subject) value = entity.value_pos if value.is_known: self._mappings.setdefault((entity.location, value.line, value.column), subject) def _index(self) -> None: raml = self._raml + borrowed: list[BaseShape] = [] for key, name, entity in every_declaration(raml): kind = DECLARATION_KINDS[key] self._put(_Subject(name, _role(kind), entity)) for fragment in raml.fragments.values(): for name, link in fragment.uses.items(): self._put(_Subject(name, 'library namespace', link)) + body = authored.fragment_body(raml, fragment.location) + if isinstance(body, BaseShape): + self._put(_Subject(body.name or '', 'type declaration', body)) for base in raml.shapes: + key, value = base.key_pos, base.value_pos + if key.is_known and value.is_known and (value.line, value.column) < (key.line, key.column): + borrowed.extend(_declaration_children(base)) self._subjects.setdefault(base.id, _Subject(base.name or '', 'type declaration', base)) if isinstance(base.shape, ObjectShape): for name, prop in authored.properties(base): @@ -192,6 +223,24 @@ def _index(self) -> None: for name, prop in authored.facets(base): self._put(_Subject(name, 'custom facet declaration', prop.base, prop.required)) self._index_endpoints() + self._exclude_alias_children(borrowed) + + def _exclude_alias_children(self, borrowed: list[BaseShape]) -> None: + # YAML aliases borrow children placed at their anchor. Keep hover's + # subjects, but do not turn those borrowed data spans into declarations + # (the same placement boundary as parser.syntax.keys). + excluded: set[int] = set() + while borrowed: + base = borrowed.pop() + if base.id in excluded: + continue + excluded.add(base.id) + borrowed.extend(_declaration_children(base)) + if excluded: + self._by_uri = { + uri: [subject for subject in subjects if subject.entity.id not in excluded] + for uri, subjects in self._by_uri.items() + } def _index_endpoints(self) -> None: api = self._raml.entry_point @@ -269,10 +318,8 @@ def data_at(self, uri: str, line: int, column: int) -> list[DataTarget]: def inlay_hints(self, uri: str, span: Position) -> list[Hint]: """Present hidden type/facet facts using the existing authored/data indices.""" - if uri not in self._keys: - self._source_keys(uri) if uri not in self._inlay_declarations: - declared = [hint for source in self._keys[uri] for hint in self._declaration_hints(uri, source)] + declared = [hint for subject in self._by_uri.get(uri, ()) for hint in self._declaration_hints(uri, subject)] declared.sort(key=lambda hint: (hint.position.line, hint.position.column)) self._inlay_declarations[uri] = declared, [(hint.position.line, hint.position.column) for hint in declared] declared, starts = self._inlay_declarations[uri] @@ -326,19 +373,21 @@ def inlay_hints(self, uri: str, span: Position) -> list[Hint]: ) return sorted(hints, key=lambda hint: (hint.position.line, hint.position.column, hint.label)) - def _declaration_hints(self, uri: str, source: Key) -> Iterator[Hint]: - subject = self._sites.get((uri, source.node.line, source.node.column)) - if subject is None or not isinstance(subject.entity, BaseShape): + def _declaration_hints(self, uri: str, subject: _Subject) -> Iterator[Hint]: + if not isinstance(subject.entity, BaseShape): return base = subject.entity if base.id in self._raml.broken or isinstance(base.shape, UnknownShape): return - type_written = source.value.kind is NodeKind.SEQUENCE or ( - source.value.kind is NodeKind.MAPPING - and any(key.value in ('type', 'schema') for key, _ in pairs(source.value)) - ) - if base.type_expr is None and base.type and not type_written: - key = source.node.position + key = base.key_pos + if ( + base.type + and not base.type_written + and base.name is not None + and key.is_known + and key.line == key.end_line + and key.end_column > key.column + ): yield Hint( key.shifted(key.end_column - key.column, 0), (Part('[' + type_label(underlying_type(base)) + ']'),), diff --git a/fastraml/types/base.py b/fastraml/types/base.py index f936e12..3d9f413 100644 --- a/fastraml/types/base.py +++ b/fastraml/types/base.py @@ -294,6 +294,7 @@ class BaseShape: # provenance / tooling 'type_expr', 'type_expr_refs', + 'type_written', 'is_annotation_type', 'anchor', 'location', @@ -353,6 +354,8 @@ def __init__( # noqa: PLR0913 - a model constructor names its fields #: text and span after, unless the source is retained (docs/05 § 1). self.type_expr: Node | WrittenScalar | None = None self.type_expr_refs: list[TypeExprRef] = EMPTY_LIST + #: Authored type syntax, independent of whether P7 creates expression refs. + self.type_written = False self.is_annotation_type = is_annotation_type #: The scope unqualified names in this declaration resolve in. self.anchor = anchor @@ -429,6 +432,7 @@ def clone(self, memo: dict[int, BaseShape], depth: int = 0) -> BaseShape: # shared list would let one clone's narrowing reach the original. clone.allowed_targets = None if self.allowed_targets is None else list(self.allowed_targets) clone.type_expr = self.type_expr + clone.type_written = self.type_written clone._unwrapped = self._unwrapped # The containers are what unwrap mutates, so each gets its own. An diff --git a/fastraml/types/shape.py b/fastraml/types/shape.py index 1fc69aa..773caa2 100644 --- a/fastraml/types/shape.py +++ b/fastraml/types/shape.py @@ -265,6 +265,10 @@ def _make_shape( # noqa: PLR0913, PLR0917 - make_shape's arguments, resolved def _decode_shape(raml: Raml, base: BaseShape, value_node: Node, location: str, default_type: str) -> BaseShape: """Everything `make_shape` reads from the declaration's value.""" type_node, facets = _decode(raml, base, value_node) + base.type_written = type_node is not None and ( + value_node.kind is not NodeKind.SCALAR + or not (is_null(type_node) or (type_node.tag == TAG_STR and not type_node.value)) + ) if type_node is None: kind = identify_shape_type(facets, default_type, location) else: diff --git a/tests/bench/test_corpus.py b/tests/bench/test_corpus.py index 912843c..87c7de4 100644 --- a/tests/bench/test_corpus.py +++ b/tests/bench/test_corpus.py @@ -172,18 +172,28 @@ def compose(text, **kwargs): def test_inlays_reaches_inferred_types_data_types_and_inherited_facets(self, tmp_path, monkeypatch, count): from bench.__main__ import run_one from fastraml.service import inlays + from fastraml.service.hover import Hover labels = [] + passes = 0 original = inlays.inlay_hints + def source(*args, **kwargs): + pytest.fail('model-backed inlays must not compose source or populate grammar/builtin indices') + def hints(snapshot, uri, span): + nonlocal passes + passes += 1 result = original(snapshot, uri, span) labels.extend(hint.label for hint in result) return result + monkeypatch.setattr(Hover, '_node', source) + monkeypatch.setattr(Hover, '_source_keys', source) monkeypatch.setattr(inlays, 'inlay_hints', hints) entry = corpus.write_hover(tmp_path, family_count=count) run_one('inlays', 'unwrap', entry, repeat=1) + assert passes == 2, 'timing and allocation both reach model-backed hints' for label in ('[object]', '[string]', '[string; length: ≥2]'): assert labels.count(label) >= count diff --git a/tests/unit/test_service_inlays.py b/tests/unit/test_service_inlays.py index d6e3252..b4f5a21 100644 --- a/tests/unit/test_service_inlays.py +++ b/tests/unit/test_service_inlays.py @@ -269,3 +269,114 @@ def test_long_union_labels_remain_short_without_losing_alternative_documentation ellipsis = next(part for part in hint.parts if '…' in part.label) assert ellipsis.tooltip is not None assert 'Uses `Zeta`' in ellipsis.tooltip + + +def test_empty_declarations_are_inferred_but_explicit_null_types_and_parent_lists_are_not(memory_workspace): + document = ( + '#%RAML 1.0\ntitle: T\ntypes:\n Empty:\n Null: null\n Quoted: ""\n' + ' Map: {}\n Explicit: {type: null}\n Schema: {schema: null}\n' + ' A: object\n B: object\n Both: [A, B]\n Mapping: {type: [A, B]}\n' + ) + workspace, folder = _buffered(memory_workspace, {'api.raml': document}) + uri = f'{folder}/api.raml' + snapshot = workspace.snapshot(uri) + assert snapshot.error is None + hints = inlays.inlay_hints(snapshot, uri, Position(1, 1, 100, 1)) + assert [(hint.position.line, hint.label) for hint in hints] == [ + (_where(document, f'{name}:')[0], '[string]') for name in ('Empty', 'Null', 'Quoted', 'Map') + ] + + +@pytest.mark.parametrize('header', ['', '#%RAML 1.0 DataType\n']) +def test_included_fragment_reference_hints_use_the_expression_not_a_synthetic_key(memory_workspace, header): + included = header + 'type: lib.Code\nuses:\n lib: lib.raml\n' if header else 'type: string\n' + files = { + 'api.raml': '#%RAML 1.0\ntitle: T\ntypes:\n Included: !include included.raml\n', + 'included.raml': included, + 'lib.raml': '#%RAML 1.0 Library\ntypes:\n Code: {type: string, minLength: 2}\n', + } + workspace, folder = _buffered(memory_workspace, files) + uri = f'{folder}/included.raml' + snapshot = workspace.snapshot(f'{folder}/api.raml') + assert snapshot.error is None + hints = inlays.inlay_hints(snapshot, uri, Position(1, 1, 100, 1)) + if header: + assert [(hint.position, hint.label) for hint in hints] == [ + (Position(2, len('type: lib.Code') + 1, 2, len('type: lib.Code') + 1), '[string; length: ≥2]') + ] + else: + assert not hints + + +def test_keyless_fragment_and_inline_body_spans_are_not_inferred_declaration_keys(memory_workspace): + files = { + 'api.raml': ( + '#%RAML 1.0\ntitle: T\nmediaType: application/json\ntypes:\n' + ' Included: !include included.raml\n/test:\n post:\n body: {properties: {value: string}}\n' + ), + 'included.raml': '#%RAML 1.0 DataType\nproperties: {field: string}\n', + } + workspace, folder = _buffered(memory_workspace, files) + snapshot = workspace.snapshot(f'{folder}/api.raml') + assert snapshot.error is None + assert not inlays.inlay_hints(snapshot, f'{folder}/included.raml', Position(1, 1, 100, 1)) + assert not inlays.inlay_hints(snapshot, f'{folder}/api.raml', Position(8, 1, 8, 100)) + + +def test_template_materializations_supply_one_hint_per_authored_site(memory_workspace): + document = ( + '#%RAML 1.0\ntitle: T\ntraits:\n filtering:\n queryParameters:\n' + ' filter: {minLength: 2}\n/a:\n get:\n is: [filtering]\n' + '/b:\n get:\n is: [filtering]\n' + ) + workspace, folder = _buffered(memory_workspace, {'api.raml': document}) + uri = f'{folder}/api.raml' + snapshot = workspace.snapshot(uri) + assert snapshot.error is None + hints = inlays.inlay_hints(snapshot, uri, Position(1, 1, 100, 1)) + line, column = _where(document, 'filter:') + assert [(hint.position, hint.label) for hint in hints] == [ + (Position(line, column + len('filter'), line, column + len('filter')), '[string]') + ] + + +@pytest.mark.parametrize( + 'declaration', + [ + 'field: {minLength: 2}', + 'field: {properties: {nested: {minLength: 2}}}', + 'field: {items: {properties: {nested: {minLength: 2}}}}', + ], +) +def test_yaml_alias_children_do_not_invent_declaration_hints_in_opaque_data(memory_workspace, declaration): + document = ( + '#%RAML 1.0\ntitle: T\nannotationTypes:\n raw: any\n' + f'(raw): &declaration\n properties:\n {declaration}\n' + 'types:\n Aliased: *declaration\n' + ) + workspace, folder = _buffered(memory_workspace, {'api.raml': document}) + uri = f'{folder}/api.raml' + snapshot = workspace.snapshot(uri) + assert snapshot.error is None + hints = inlays.inlay_hints(snapshot, uri, Position(1, 1, 100, 1)) + line, _ = _where(document, 'field:') + assert not [hint for hint in hints if hint.position.line == line] + line, _ = _where(document, 'Aliased:') + assert [hint.label for hint in hints if hint.position.line == line] == ['[object]'] + + +def test_a_yaml_alias_preserves_hints_at_its_actual_declaration_anchor(memory_workspace): + document = ( + '#%RAML 1.0\ntitle: T\ntypes:\n Original: &declaration\n' + ' properties:\n field: {minLength: 2}\n Aliased: *declaration\n' + ) + workspace, folder = _buffered(memory_workspace, {'api.raml': document}) + uri = f'{folder}/api.raml' + snapshot = workspace.snapshot(uri) + assert snapshot.error is None + hints = inlays.inlay_hints(snapshot, uri, Position(1, 1, 100, 1)) + assert [(hint.position.line, hint.label) for hint in hints] == [ + (_where(document, 'Original:')[0], '[object]'), + (_where(document, 'field:')[0], '[string]'), + (_where(document, 'Aliased:')[0], '[object]'), + ] diff --git a/tests/unit/test_shape_decode.py b/tests/unit/test_shape_decode.py index c8e3e02..b3d34cc 100644 --- a/tests/unit/test_shape_decode.py +++ b/tests/unit/test_shape_decode.py @@ -84,6 +84,32 @@ def test_every_shape_is_registered_and_only_unknowns_are_on_the_worklist(self): assert (base in unresolved) is isinstance(base.shape, UnknownShape) +@pytest.mark.parametrize( + ('declaration', 'written'), + [ + ('', False), + ('null', False), + ('~', False), + ("''", False), + ('{}', False), + ('{properties: {name: string}}', False), + ('string', True), + ('Other', True), + ('[A, B]', True), + ('{type: [A, B]}', True), + ('{type: null}', True), + ('{type: ""}', True), + ('{schema: null}', True), + ('{type: {value: string}}', True), + ], +) +def test_authored_type_syntax_survives_both_clone_operations(declaration, written): + base = shape(f'T: {declaration}\n') + assert base.type_written is written + assert base.clone({}).type_written is written + assert base.clone_detached().type_written is written + + class TestCommonFacets: def test_the_common_facets_land_on_the_base(self): base = shape('T:\n type: string\n displayName: D\n description: Text.\n (ann): 1\n')