diff --git a/bench/__main__.py b/bench/__main__.py index ac245df..7ccc9aa 100644 --- a/bench/__main__.py +++ b/bench/__main__.py @@ -124,6 +124,7 @@ def _at(base: int, scale: float) -> int: Bench('hover', lambda root, scale: corpus.write_hover(root, family_count=_at(400, scale))), Bench('effective-types', lambda root, scale: corpus.write_hover(root, family_count=_at(300, scale))), Bench('inlays', lambda root, scale: corpus.write_hover(root, family_count=_at(400, scale))), + Bench('source-structure', lambda root, scale: corpus.write_hover(root, family_count=_at(400, scale))), Bench('service-session', lambda root, scale: corpus.write_hover(root, family_count=_at(400, scale))), Bench('service-source-first', lambda root, scale: corpus.write_hover(root, family_count=_at(400, scale))), ) @@ -161,6 +162,7 @@ def run_one(bench: str, config: str, entry: Path, repeat: int) -> Measurement: 'hover', 'effective-types', 'inlays', + 'source-structure', 'service-session', 'service-source-first', }: @@ -206,6 +208,7 @@ def _measure_view(bench: str, entry: Path, repeat: int) -> Measurement: 'hover': _measure_hover, 'effective-types': _measure_effective_types, 'inlays': _measure_inlays, + 'source-structure': _measure_source_structure, 'service-session': _measure_service_session, 'service-source-first': lambda entry, repeat: _measure_service_session(entry, repeat, source_first=True), }.get(bench) @@ -369,6 +372,38 @@ def hints() -> object: return measure('inlays', 'unwrap', hints, repeat=repeat) +def _measure_source_structure(entry: Path, repeat: int) -> Measurement: + """Folding and selection requests served from the workspace's source cache.""" + from fastraml.gctuning import tuned_gc # noqa: PLC0415 - feature workload only + from fastraml.service import queries # noqa: PLC0415 - feature workload only + from fastraml.service.workspace import Workspace # noqa: PLC0415 - feature workload only + from fastraml.uris import path_to_file_uri # noqa: PLC0415 - feature workload only + + text = entry.read_text(encoding='utf-8') + uri, folder = path_to_file_uri(entry), path_to_file_uri(entry.parent) + lines = text.splitlines() + probes = 12 + step = max(1, len(lines) // probes) + positions = [ + (number, len(raw) - len(raw.lstrip()) + 1) + for number, raw in enumerate(lines[2:], start=3) + if ':' in raw and not raw.lstrip().startswith('#') + ][::step][:probes] + + def structure() -> object: + workspace = Workspace([folder]) + workspace.open(uri, text, 1) + folded = [queries.folding_ranges_of(workspace.source(uri)) for _ in range(3)] + selected = [queries.selection_ranges_of(workspace.source(uri), line, column) for line, column in positions] + if len(positions) < probes or not folded[0] or any(not spans for spans in selected): + message = 'source-structure workload no longer reaches folding and selection' + raise RuntimeError(message) + return workspace.source(uri), folded, selected + + with tuned_gc(): + return measure('source-structure', 'unwrap', structure, repeat=repeat) + + def _measure_service_session(entry: Path, repeat: int, *, source_first: bool = False) -> Measurement: from bench.service_session import exercise, prepare # noqa: PLC0415 - feature workload only from fastraml.gctuning import tuned_gc # noqa: PLC0415 - feature workload only @@ -547,6 +582,7 @@ def compare(results: Sequence[Measurement], tolerance: float) -> int: 'hover': 'unwrap', 'effective-types': 'unwrap', 'inlays': 'unwrap', + 'source-structure': 'unwrap', 'service-session': 'unwrap', 'service-source-first': 'unwrap', } diff --git a/bench/corpus.py b/bench/corpus.py index 7637f66..fa5f526 100644 --- a/bench/corpus.py +++ b/bench/corpus.py @@ -215,6 +215,19 @@ def _derived(name: str, _ordinal: int, previous: str) -> str: ) +def _derived_schema(name: str, _ordinal: int, previous: str) -> str: + """`_derived` under the deprecated `schema:` spelling (docs/18 § 6).""" + return ( + f' {name}:\n' + f' schema: {previous}\n' + f' properties:\n' + f' label?: string\n' + f' weight:\n' + f' type: number\n' + f' minimum: 0\n' + ) + + def _array(name: str, _ordinal: int, previous: str) -> str: return f' {name}: {previous}[]\n' @@ -262,7 +275,12 @@ def _type_block(ordinal: int, previous: str | None) -> str: name = f'T{ordinal}' if previous is None: return _plain(name, ordinal, '') - return _FORMS[ordinal % len(_FORMS)](name, ordinal, previous) + form = _FORMS[ordinal % len(_FORMS)] + # Every other derived type takes the deprecated `schema:` spelling, so the + # parse exercises the compatibility recording (docs/18 § 6). + if form is _derived and ordinal % 12 == 1: + form = _derived_schema + return form(name, ordinal, previous) _COMMON = """#%RAML 1.0 Library @@ -287,10 +305,13 @@ def _library(index: int, type_count: int, *, depth: int) -> str: *same* file through a different spelling of the path. That is the diamond the compose cache exists for (docs/12 § 1); if canonicalisation ever regresses, this corpus goes quadratic and the linearity check catches it. + Every other library declares its whole table under the deprecated + `schemas:` alias, as a pre-1.0 document would: a fragment takes + `schemas:` or `types:`, never both (docs/18 § 6). """ lines = ['#%RAML 1.0 Library', f'usage: generated library {index}', 'uses:'] lines.append(f' common: {"../" * depth}common.raml') - lines.append('types:') + lines.append('schemas:' if index % 2 else 'types:') previous: str | None = None for ordinal in range(type_count): lines.append(_type_block(ordinal, previous).rstrip('\n')) @@ -307,6 +328,8 @@ def _api_using(libraries: list[str]) -> str: 'uses:', ] lines.extend(f' lib{index}: {path}' for index, path in enumerate(libraries)) + lines.append('schemas:') + lines.extend(_plain(f'Root{ordinal}', ordinal, '').rstrip('\n') for ordinal in range(3)) return '\n'.join(lines) + '\n' @@ -327,7 +350,11 @@ def write_large(root: Path, *, type_count: int = 7000, library_count: int = 150) """`type_count` types spread over `library_count` libraries. Sized after go-raml's published corpus (7124 types, 148 libraries). Halve - `type_count` and `library_count` together for the linearity check. + `type_count` and `library_count` together for the linearity check. Every + other library declares its types under the deprecated `schemas:` alias, + every other inherited type uses the deprecated `schema:` facet, and the + root carries a `schemas:` table of its own, so the parse exercises the + compatibility recording (docs/18 § 6). """ per_library, remainder = divmod(type_count, library_count) files = {'common.raml': _COMMON} @@ -375,12 +402,15 @@ def write_endpoints(root: Path, *, resource_count: int = 500) -> Path: Measures the two-stage build (docs/12 § 1): 6000 method-level trait applications, every one of them a tree merge rather than a model merge. + `Item` is declared in a deprecated `schemas:` table, and every fourth + resource types its body with the deprecated `schema:` facet, so the parse + exercises the compatibility recording (docs/18 § 6). """ lines = [ '#%RAML 1.0', 'title: Generated endpoint benchmark', 'baseUri: https://example.test', - 'types:', + 'schemas:', ' Item:', ' type: object', ' properties:', @@ -398,7 +428,7 @@ def write_endpoints(root: Path, *, resource_count: int = 500) -> Path: lines.append(' 200:') lines.append(' body:') lines.append(' application/json:') - lines.append(' type: Item') + lines.append(' schema: Item' if index % 4 == 0 else ' type: Item') _write(root, {'api.raml': '\n'.join(lines) + '\n'}) return root / 'api.raml' diff --git a/docs/04-fragments-and-namespaces.md b/docs/04-fragments-and-namespaces.md index 795542e..40f1741 100644 --- a/docs/04-fragments-and-namespaces.md +++ b/docs/04-fragments-and-namespaces.md @@ -45,6 +45,7 @@ class ReferenceResolver(Fragment, Protocol): def trait_definition(self, name: str) -> TraitDefinition: ... def security_scheme_definition(self, name: str) -> SecuritySchemeDefinition: ... def library_link(self, prefix: str) -> LibraryLink | None: ... + def visible_names(self, kind: str) -> Iterator[tuple[str, Declaration]]: ... ``` `ReferenceResolver` includes the `SecuritySchemeResolver` capability. Every @@ -55,6 +56,17 @@ declarations, as does an Overlay or Extension in its target tree's declarations implementation over two hooks: the local table and the `uses:` map. Resolver capabilities are checked structurally. +`visible_names(kind)` enumerates names for one declaration table (`types`, +`annotationTypes`, `traits`, `resourceTypes`, or `securitySchemes`). It yields +local declarations first, then each directly imported library's declarations +with that import's prefix. Local dotted names take precedence. Every qualified +candidate must round-trip through ordinary resolution to the same entity; +unresolved imports and inaccessible dotted suffixes contribute no names. +It never follows a library's own imports. Annotation-type enumeration includes +the ordinary-type fallback, with annotation types taking precedence. +Extension resolvers enumerate only the declarations and imports visible at +their chain position. Literal included content delegates to its host resolver. + ## 3. Name resolution `resolve_reference()` resolves declarations in a fragment with a local table. @@ -134,6 +146,12 @@ resolves every name through the includer and is located in the included file. ## 5. Fragment decoding +When a declaration table uses the accepted compatibility spelling `schemas:`, +the decoder records its name, authored URI and key position in +`Raml.syntax_aliases`, keyed by the fragment's entity ID. This is an authored +language fact, independent of source retention. The lint view decides whether +to report it and with what severity. + API decoding first collects global `mediaType`, `protocols`, and `securedBy`, then decodes remaining declarations in source order. A lone `mediaType` may be written as an annotated scalar, `{value: ..., (a): ...}`; its annotations diff --git a/docs/05-type-model.md b/docs/05-type-model.md index 1e0d692..5f00f85 100644 --- a/docs/05-type-model.md +++ b/docs/05-type-model.md @@ -74,6 +74,12 @@ reference or inline declaration, not a bare sequence; use ## 3. Decoding and inference +An accepted `schema:` compatibility facet records its authored URI, position +and spelling in `Raml.syntax_aliases`, keyed by the shape's ID. It needs no +retained node and has no severity or diagnostic attached. A lint rule consumes +this fact rather than rediscovering it from every declaration's YAML tree. +Keys in examples, annotations or other opaque data do not declare such a facet. + `make_shape()` is the general declaration constructor. It records common facets on the base, keeps remaining YAML key/value pairs flat, determines or defers the kind, attaches it, builds the declaration facets (`properties`, `items`, @@ -198,6 +204,14 @@ same selection, but preserves all possible structural alternatives when no discriminator selects one; it does not validate the whole value to guess a branch. +`types/navigation.py` exposes `branches(base, value)` and `children(base, +value)` over positioned `ValueNode` data. Branches follow aliases and recursion +markers and preserve every structural union candidate unless a discriminator +selects one. Children report the governing declaration, expected type, source +key span and field name or array index. Invalid values retain known field +types; ordinary additional fields have no invented declaration. The service +uses this API for typed-data tokens, independently of hover presentation. + RAML field names shared by fragment, endpoint, and type decoders are defined in `facet_names.py`; diagnostic fields and JSON Schema keywords are separate. diff --git a/docs/12-performance.md b/docs/12-performance.md index 20de920..bc39c74 100644 --- a/docs/12-performance.md +++ b/docs/12-performance.md @@ -86,7 +86,7 @@ must receive a parser diagnostic rather than `RecursionError`. ## 4. Benchmark suite -`bench/` generates deterministic corpora and measures thirty-two workloads: +`bench/` generates deterministic corpora and measures thirty-three workloads: | Bench | Primary coverage | |---|---| @@ -120,10 +120,11 @@ must receive a parser diagnostic rather than `RecursionError`. | `hover` | a cold language-service snapshot, fourteen authoring hovers per type family and custom-facet definition queries: inherited summaries, facet explanations, built-ins, custom-facet declarations and supplied keys, nested custom-facet and annotation keys and values, property presence, methods and response statuses ([21](21-language-service.md) § 4.2) | | `effective-types` | a cold service snapshot, code-lens enumeration and full-depth RAML rendering for every named type and annotation type, including nested arrays, unions, scalar item constraints and recursive references ([21](21-language-service.md) § 4.3) | | `inlays` | a cold snapshot, inferred declaration types, expected types at supplied custom-facet/annotation roots and nested data keys, and compact inherited constraints anchored to type references ([21](21-language-service.md) § 4.4) | +| `source-structure` | three folding requests and twelve selection requests at scattered keys, served from the workspace's composed source tree, composed once per current text ([21](21-language-service.md) § 4) | | `service-session`, `service-source-first` | representative mixed editor requests over three root-buffer versions: parser diagnostics, occurrences, outline, links, lens enumeration, 120-line viewport inlays, folding, three sparse hovers and warm queries; source-first additionally requests folding before each snapshot | The six general workloads are `small`, `large`, `endpoints`, `extensions`, -`validate` and `jsonschema`. The other twenty-six are feature workloads: +`validate` and `jsonschema`. The other twenty-seven are feature workloads: each exists because no general workload runs the code it covers. Their reach tests (`tests/bench/test_corpus.py`) count calls or check bound results, and fail if a corpus stops reaching that code at every size it covers. @@ -156,6 +157,9 @@ The small corpus-validity tests run in the ordinary test suite. For `hover`, `unwrap` builds a service snapshot with unwrap and validation, then queries every generated hover site. It measures the cold indices and their reuse; probe selection and corpus generation are outside the measured region. +For `source-structure`, `unwrap` opens the corpus as a buffer and serves the +folding and selection requests from the workspace's composed source tree; +probe selection and corpus generation are outside the measured region. For `service-session` and `service-source-first`, `unwrap` runs the three-version mixed request sequences of § 4.2; their other configurations keep their ordinary parse/view meanings. Input texts and sparse probes are prepared outside measurement. @@ -300,6 +304,19 @@ a change to a hot path, or to any code a claim is made about: `python -m bench linearity --bench NAME` for time and memory instead. Include the workload, both deltas, and the noise in the commit message. + +A service workload's measured region is a cold snapshot: the parse, plus the +first query of each kind, then reuse of the built indices. Read its delta in +two parts. The parse side is where the base revision built and retained every +file's tree and the comparison does not. The first-use side is where a query +composes on demand the tree the base retained at parse time, one composition +per file per snapshot, about a third of a parse of the file; repeated +queries of the same kind are cached and show no delta. The first-use cost is +avoidable only by retaining the trees, the per-snapshot memory cost +(docs/15 § 2). When a change moves composition from parse time to first use, +name which side the delta is on before attributing it to the change's hot +path (docs/21 § 2, docs/21 § 4). + `bench compare` against the committed baseline is only a coarse check for large regressions: the baseline and the comparison run at different times, and the machine drifts in between. Follow § 5.1 before optimizing and § 5.2 when diff --git a/docs/13-public-api.md b/docs/13-public-api.md index 4609c6d..a2dedbe 100644 --- a/docs/13-public-api.md +++ b/docs/13-public-api.md @@ -130,6 +130,12 @@ returns RAML text ([20](20-join.md) § 8). It is not part of `fastraml.__all__`. Pass `unwrap=True, validate=True` together unless you specifically need declared, unflattened types. Validation alone must clone and unwrap declarations privately. +Default lint no longer needs whole trees merely to detect `schema:` and +`schemas:`: the decoders record their accepted spelling and position in +`Raml.syntax_aliases` (docs/04 § 5, docs/05 § 3), independent of source +retention. A consumer needing only that compact fact should read it rather +than retaining the whole source tree. + A configuration file's `parser:` section is `ParserConfig`. `limits(options)` applies its `max_include_size`, `max_depth` and `regex_engine`; its `workspace_root` and `remote` are left to the host, which weighs them against diff --git a/docs/15-implementation-plan.md b/docs/15-implementation-plan.md index 032fe63..840c0c9 100644 --- a/docs/15-implementation-plan.md +++ b/docs/15-implementation-plan.md @@ -20,7 +20,9 @@ that each extend the same master remains deferred (docs/19 § 7). XML Schema external types are unsupported ([01](01-scope-and-coverage.md) § 3). **Service baseline acceptance and recovery.** The simpler source-cache branch -is undergoing comparison with master using the portable mixed-request workloads. +passes the master-controlled rebuild and mixed-request checks after sharing its +query source owner. The user accepted its documented first-use trade-off; +CI verification and integration are pending. The record-backed representation replacement remains parked; its recurring rebuild and first-use costs are recorded in the [service cost review](reports/2026-10-10/service-authoring-cost-review.md). @@ -68,6 +70,13 @@ the media-type fixes and the type-walk work landed together: endpoint build still holds; skip the swap where the node is retained anyway. - `validate`: time up to +4.9% on some configurations in a 3-round run; a 5-round rerun of `unwrap+validate` was within noise. +- Text-only service snapshots pay an on-demand composition for a queried input + unless a compatible source entry is already ready (docs/21 § 4). The shared + owner removes the old duplicate hover/structural tree. The accepted baseline's + remaining first-use cost and post-query allocation trade-off are recorded in the + [integration report](reports/2026-10-10/service-baseline-integration.md). + Model-backed declaration inlays and cursor-local source lookup are the next + separately measured recovery candidates. ## 3. Potential future work @@ -78,21 +87,14 @@ parser rules without an owning design document and tests. The language service and its LSP are built ([21](21-language-service.md)). The ordered plan for the rest, completion, editing features and MCP, is [research/language-service-plan.md](research/language-service-plan.md): M1 to -M4 are done, and completion (M5) comes next. Its design records are +M4 are done, and completion (M5) comes next, with its prerequisites in: the +resolver's visible-names enumeration (docs/04 § 2) and the workspace's +composed source cache (docs/21 § 4). Its design records are [archive/language-server.md](archive/language-server.md) and [archive/language-service-architecture.md](archive/language-service-architecture.md); its parser prerequisite, a trustworthy `parse_lenient` model, is analysed in [research/partial-models.md](research/partial-models.md). -**Deprecated aliases as a model fact (proposed).** `deprecated-schemas` is on -by default and reads `source_info`, so every lint-enabled parse, the language -service's included, retains each declaration's YAML subtree: +7.66 MB on -`large`, against +0.56 MB for the text alone. It needs only where a `schema:` -or `schemas:` key was accepted, which the decoder knows. Recording those -(file, position, key) on the model would let the default rule set run on -`retain_text`; the three off-by-default rules that inspect how a declaration -was written keep `retain_source`. - **Sampled examples in the tree (undecided).** The viewer's request and response panel (parked on `feat/viewer-request-samples`) needs a working body for each payload, and only Python can produce a validated one. The candidate design: diff --git a/docs/18-linting.md b/docs/18-linting.md index 8a53af9..9b8c333 100644 --- a/docs/18-linting.md +++ b/docs/18-linting.md @@ -281,7 +281,12 @@ CLI builds the graph as part of each lint invocation. Lint requires an unwrapped model. Source-sensitive rules additionally require `ParseOptions(retain_source=True)`; `Linter.requires_source` says whether an -enabled rule is one. +enabled rule is one. Those are the opt-in style rules +`prefer-array-expression`, `avoid-explicit-inferred-type` and +`prefer-inline-alias`; no rule in the default set reads the trees. + +`deprecated-schemas` reads the decoder's `Raml.syntax_aliases` records and +works with any retention choice. It is not a source-sensitive rule. ## 7. What lint is not diff --git a/docs/21-language-service.md b/docs/21-language-service.md index 0fcb906..344871d 100644 --- a/docs/21-language-service.md +++ b/docs/21-language-service.md @@ -20,7 +20,8 @@ and several views, and only `cli/` imports it (`docs/02` § 2; | Module | Holds | |---|---| -| `service/workspace.py` | buffers, the overlay loader, roots, snapshots, staleness | +| `service/workspace.py` | buffers, the overlay loader, roots, snapshots and staleness | +| `service/source.py` | on-demand composed source shared by workspace and snapshot consumers | | `service/text.py` | converting a column between fastRAML and a protocol | | `service/queries.py` | the queries, in fastRAML positions | | `service/outline.py` | the outline, over the authorship view (`docs/16` § 10) | @@ -53,7 +54,14 @@ the TCK's 1011 files. **Snapshots.** A snapshot is one `parse_lenient` of one root, with `unwrap=True, validate=True, retain_text=True` and the configuration's `parser:` limits. It keeps the YAML trees too, `retain_source=True`, only when -an enabled lint rule reads them (`deprecated-schemas`, in the default set). +an enabled lint rule reads them; no rule in the default set does. Decoders +record the small syntax facts that `deprecated-schemas` reads (docs/18 § 6). +The first query that needs a file's tree composes it on demand. Hover, inlays, +folding and selection share the workspace's source owner when their input and +composition policy agree (§ 4). Parsing does not populate that query cache or +retain all producer trees. Snapshot-local hover indices keep requested trees; +snapshots borrow the workspace's cache weakly rather than keep its other files +alive. Without a live owner, a held snapshot can compose its own retained text. It records the set of files it read: every retained text, every fragment, and every include it tried, found or not. It is built when a query first asks for it and kept until one of those files changes. A file that @@ -122,11 +130,32 @@ composed YAML and the parser's structural grammar. | `document_symbols` | the fragment's metadata, declaration tables, documentation and resources, grouped by section, from `key_pos` and `value_pos`; `type_expr`, else `type_name`, for a type's detail | | `workspace_symbols` | the declarations of every snapshot, matched case-insensitively, once each | | `links` | each `Link` occurrence in the file, with the file it resolved to | -| `folding_ranges`, `selection_ranges` | the buffer's composed `Node` tree alone | +| `folding_ranges`, `selection_ranges` | the workspace's composed source tree for the file's current text, composed once per text | +| `visible_names` | the fragment resolver's local and directly imported declaration candidates (docs/04 § 2) | | `type_at`, `supertypes`, `subtypes` | a type's `inherits` and `alias`, and the declarations naming it | | `diagnostics` | `RamlError.chains()` and the lint findings | | `tree` | `build_tree` (`docs/16` § 6) as JSON text, only on an unwrapped model | +**Shared source.** `Workspace.source(uri)` composes once for its current input, +including a failed composition. Cache compatibility requires the canonical URI, +equal text after the parser's single leading-BOM removal, depth limit and YAML +backend. Equal text from another disk decode can reuse the same tree. Buffer +changes, close and watched disk changes discard that URI's entry; changing the +composition policy discards the cache. Unchanged dependency entries survive a +root edit. These trees carry no semantic binding or include receiving context; +those belong to the snapshot's hover index. + +A source-before-snapshot request and a later hover share their composed tree; +the parse itself still composes independently. An eligible original RAML/YAML +container already retained for a syntax-reading lint rule can replace an earlier +standalone owner on demand, without changing the older tree. JSON include +normalization and synthetic non-YAML/schema scalars are not that standalone +composition policy, so they are not substituted for generic source trees. +An old snapshot may borrow an identical ready entry, but a miss after that input +or its policy was invalidated remains private to that snapshot and cannot replace the +current workspace's entry. Query answers therefore keep their own input positions +without making an old view a source for current text. + **Stopped parses.** A query on a snapshot that stopped early answers from the stages it completed. A declaration is an occurrence after decoding; a type name used in an expression is one only once P7 bound it. No query raises on a @@ -426,7 +455,7 @@ current only those it reads: | definition, highlight, hover, type hierarchy, supertypes | the first snapshot that answers | | references, subtypes | every snapshot, answers deduplicated | | workspace symbols | every root | -| folding, selection | the buffer's text alone | +| folding, selection | the workspace's composed source tree for the buffer's current text, composed once per text (§ 4) | A file that roots bind differently (a master an Overlay merges into, a template applied with different arguments) answers from the first. References @@ -450,6 +479,10 @@ could save. - `test_service_text.py`: conversion in each encoding, both ways. - `test_service_workspace.py`: roots, buffers over the disk, the sandbox, and when a snapshot is stale. +- `test_service_source_cache.py`: both source/semantic request orders, failed + composition, equal closed-file inputs, depth/backend compatibility, unchanged + dependencies and unrelated edits, private old-snapshot misses, weak owner + lifetime, retained original trees and distinct JSON-include normalization. - `test_service_queries.py`: each query on one document with a library, a DataType include, a trait and a resource type; every query on a parse stopped at each stage; an Extension's outline; and, over the TCK, that every diff --git a/docs/reports/2026-10-10/service-baseline-integration.md b/docs/reports/2026-10-10/service-baseline-integration.md index 37e0d7a..812809b 100644 --- a/docs/reports/2026-10-10/service-baseline-integration.md +++ b/docs/reports/2026-10-10/service-baseline-integration.md @@ -32,10 +32,116 @@ Linearity, three repeats, actual `unwrap` mixed scenarios: This verifies the historical uncached behavior is reached and scales. It does not establish acceptance of the simplified production candidate. +All GitHub CI checks subsequently passed, including Ubuntu 3.12/3.13, TCK, +optional extras, pure-Python YAML and the benchmark gate. Measurement-only +[PR #1](https://github.com/deiteris/FastRAML/pull/1) merged into master at +`d2b8a02fc7b3e7cb462906bd998d15b649296348`. + ## 2. Simplified candidate -Pending comparison against the measurement-only master control and inspection -of first-use/source ownership costs. Starting candidate: `fb0fb73`. +Starting candidate: `fb0fb73`, with the measurement-only commit integrated locally. +The production implementation is unchanged in this first comparison. Control: +`549af67` (the production-equivalent measurement parent of merged master +`d2b8a02`). Three alternating rounds, best of three, identical candidate-generated +corpora: + +The mixed corpus is one **311,308-byte (304.012 KiB), 13,204-line** RAML file: +400 families, 2,000 named types, 400 annotation types and 400 resources. Each +reported time covers **three complete edit/query cycles**, not one parse or one +editor request. Each cycle includes two outlines, two viewport inlay requests, +two folding requests, three hovers, eight selection positions, links, lens +enumeration, diagnostics and occurrences. The source-first case adds one folding +request before each snapshot. These request counts are a cache/reuse scenario, +not a measured client request trace. + +Original master recomposes for both folds and every selection: 11 compositions +per version, 33 over the snapshot-first sequence. The simpler candidate composes +three times per version (parse, hover/inlays and workspace source), nine overall. +Thus the aggregate is deliberately sensitive to repeated uncached source work; +it must not be quoted as individual interactive-request latency. + +| Workload (`unwrap`) | Control ms | Candidate ms | Time/noise | Peak | Kept | +|---|---:|---:|---|---:|---:| +| `service-session` | 3,108.7 | 1,560.2 | -49.8%, noise 4.4% | +2.1% | +26.8% | +| `service-source-first` | 3,304.7 | 1,561.5 | -52.7%, noise 18.9% | -10.9% | +26.9% | + +Kept allocation is 24.519 versus 31.097 MB on the first order. The mixed sequence +therefore does not yet accept the simpler candidate's memory ownership: the +hover index and workspace each retain a full tree of the same current input. +The local gate passes after measurement integration: 5,824 passed, 68 skipped, +1 xfailed. + +### 2.1 Bounded acceptance fix: one composed-source owner + +Before further ports, share a queried file's plain `Node` tree between the +workspace's folding/selection requests and snapshot hover/inlay consumers, when +URI, normalized input, depth limit and YAML backend agree. Failed composition is +also a cached result. Retained full-source snapshots can offer their original +tree on demand; text-only snapshots still parse without retaining all producer +trees or building authoring records. + +The workspace owns the source cache. Snapshots borrow it weakly, so holding an old +snapshot does not keep the entire workspace/cache alive. Snapshot-local hover +nodes retain the requested subtree they need. Input invalidation advances a cache +generation; an old snapshot may read an identical ready entry but must not publish +a stale cache miss into the current workspace. This needs no per-file registry +of unused authoring structures during parsing. + +Tests must protect both request orders, failed composition, limits/backend +compatibility, unchanged dependencies, old snapshots and source-cache release. +Measure mixed requests and normal rebuilds separately after the fix. Remaining +first-use trade-offs still require their own acceptance decision. + +### 2.2 Shared-source results and acceptance + +The shared-source implementation supplies one on-demand plain tree to both +consumer groups. Invalidations track changes only for read/cached URIs, so an +unrelated watched file does not prevent a current snapshot from publishing a +cache miss. Policy changes invalidate publication separately. Fourteen focused +cases protect the named ownership/failure/compatibility decisions; the mixed +reach test requires six compositions over three versions in either order +(one semantic and one query composition per version). + +Isolated A/B against the measurement-integrated candidate `94ebc60`: + +| Workload | Time/noise | Peak | Kept | +|---|---|---:|---:| +| `service-session` | -12.5%, noise 2.8% | -26.5% | -16.9% | +| `service-source-first` | within 22.0% noise | -22.7% | -16.9% | + +Final master control is `d2b8a02`. Three rounds/best of three, identical inputs: + +| Workload/configuration | Master ms | Shared-source candidate ms | Time/noise | Peak | Kept | +|---|---:|---:|---|---:|---:| +| `large/service` | 451.0 | 454.9 | within 12.0% noise | -20.1% | -24.0% | +| `service-session/unwrap` | 3,078.7 | 1,307.7 | -57.5%, noise 8.3% | -25.0% | +5.4% | +| `service-source-first/unwrap` | 3,352.6 | 1,377.8 | -58.9%, noise 8.3% | -31.1% | +5.4% | +| `hover/unwrap` | 268.1 | 331.6 | +23.7%, noise 2.4% | +16.6% | +4.1% | +| `inlays/unwrap` | 279.3 | 343.7 | +23.0%, noise 12.3% | +13.0% | +3.4% | +| `effective-types/unwrap` | 209.2 | 208.9 | within 2.8% noise | +0.0% | -27.4% | + +The mixed candidate keeps 25.851 MB versus master's 24.521 MB after requests, +instead of the original candidate's 31.097 MB. There is no longer a second retained +full tree for these consumers. The remaining allocation difference is not claimed +to have been attributed exclusively to one model field or position store. + +`large` and `endpoints`, on `parse`, `unwrap+validate` and `unwrap+lint`, remain +within time noise (0.8–3.4%). Ordinary model-retaining rows add 0.5–0.8% peak and +0.7–0.8% kept allocation; full lint peaks add 0.1–0.2%, with findings-only retained +allocation within noise. These corpora include the candidate's deprecated +spellings on both sides. + +The Windows gate passes: **5,838 passed, 68 skipped, 1 xfailed**. Linearity passes +for `source-structure`, `service-session` and `service-source-first`: normalized +time 1.008 / 1.118 / 1.004, peak 0.998 / 0.993 / 0.984, kept 1.000 / 0.998 / 0.998. + +After being shown the remaining cold-query cost and +5.4% post-query retained +allocation, the user explicitly selected **“Merge, then recover”**. Acceptance is +therefore against the primary rebuild goal, improved complete mixed sequences, +lower diagnostic/peak memory and corrected ownership, with the lazy first-use +trade-off recorded rather than hidden. Linux/CI verification is still required +before the production merge. Model-backed inlays are the first recovery experiment +against that accepted baseline, without accepting record capture. ## 3. Selective recovery diff --git a/docs/reports/2026-10-10/service-source-simple-rework.md b/docs/reports/2026-10-10/service-source-simple-rework.md new file mode 100644 index 0000000..a55809a --- /dev/null +++ b/docs/reports/2026-10-10/service-source-simple-rework.md @@ -0,0 +1,145 @@ +# Simple rework of the source-retention replacement + +Date: 2026-10-10. Working branch: `refactor/service-source-simple`, from +`2b6502e`. The original replacement was parked on +`refactor/service-authoring` at `bc4b49d` after the 2026-10-09 review +concluded it overreached: its acceptance benchmarks failed (large/service +time +39.0% against noise 3.2%, hover +44.4%), and the language-service +design it was meant to serve (docs/archive/language-server.md § 5) needs no +canonical-record machinery. + +## 1. What landed + +Six commits, each independently useful, none building the record +architecture: + +| Commit | Change | +|---|---| +| `3a3553c` | Shared value descent extracted to `fastraml/types/navigation.py` (parked-branch pick; representation-agnostic). | +| `2873223` | Post-parse include reads no longer enter the parse's text store (`fastraml/parser/includes.py`); restores the docs/20 § 2 contract that a file is decoded at most once per parse. | +| `2e724b5` | The deprecated `schemas:`/`schema:` spellings are recorded in `Raml.syntax_aliases` as they are decoded; `deprecated-schemas` reads the records, so no rule in the default set requires retained trees and service snapshots parse text-only by default (docs/21 § 2). | +| `885a16f` | `visible_names`: a fragment resolver's local and directly imported declaration candidates, for completion (docs/04 § 2). | +| `efd9630` | `Workspace.source`: a file's current text composed once per text and held by the workspace, invalidated on change, close or disk change; folding and selection serve from it (docs/21 § 4). New `source-structure` feature workload with its reach test. | +| `661e063` | The general corpora carry the deprecated spellings (docs/18 § 6), so the parse workloads exercise the compatibility recording, pinned by a reach test. | + +## 2. First-use mechanics + +The service composes a file's YAML tree in two independent caches, each +built lazily on first interaction: + +- The hover index (`fastraml/service/hover.py`, `_node`) composes the + file's snapshot text into a per-snapshot cache. The first hover or inlay + that needs the tree pays it: any field hover reaches it through the + field documentation, and a hover in a literal include additionally + triggers the one-time context discovery that composes every fragment + file. +- The workspace source cache (`fastraml/service/workspace.py`, + `Workspace.source`) composes the file's current text into a per-buffer + cache. The first folding or selection request for that text pays it. + +Cost, measured on the hover corpus at 400 families (entry 304 KiB, +13,204 lines): one `compose` is 60.1 ms best of 5 (61.1, 60.7, 62.9, +62.3, 60.1), against a full unwrap+validate parse of the same file at +158.8 ms best of 5. One composition is about 38 percent of one parse, or +0.2 ms per KiB. Nothing is built eagerly; the cache is built by whichever +interaction first needs the tree. + +## 3. Numbers + +`bench ab 2b6502e`, 3 rounds, best of 3 per round, at `efd9630`; the +`large/service` row is re-measured on the spelling-carrying corpora of the +last commit: + +| Workload | A ms | B ms | time | noise | peak | kept | +|---|---:|---:|---:|---:|---:|---:| +| `hover/unwrap` | 274.8 | 349.4 | +27.1% | 3.3% | +16.6% | +4.0% | +| `inlays/unwrap` | 291.1 | 354.9 | +21.9% | 4.8% | +12.9% | +3.3% | +| `effective-types/unwrap` | 209.9 | 212.7 | noise | 2.6% | noise | -27.5% | +| `large/service` | 476.0 | 474.0 | noise | 6.1% | -20.1% | -24.0% | + +New workload, no base number, so linearity (`--repeat 3`): +`source-structure/unwrap` time 1.004 (113.4 / 56.5 ms), peak 0.998, +retained 1.000. + +General parse workloads, on corpora that carry the deprecated spellings +(pinned by `test_general_corpora_exercise_the_deprecated_spellings`): every +other `large` library declares its types under the `schemas:` alias, every +other inherited type uses the `schema:` facet, and `endpoints` declares its +payload type in a `schemas:` table with every fourth body typed by +`schema:`. With the spellings in the input, `large` across all eight +configurations, and `endpoints` and `small` on parse, unwrap and +unwrap+lint, are within noise on time. Allocation peak and kept move +0.4 to ++0.8 percent on the configurations that hold the model — the +`syntax_aliases` records themselves, about 1.3 thousand of them, roughly +120 KB on `large` — and are noise where the model is dropped. The include +text-store fix still touches only `load_include`, which join calls and no +bench workload does, and the lint rule's record read, against the base's +retained-tree walk, is within noise on the lint configurations. + +Attribution: at `2e724b5` (before the last two commits) the same A/B +already shows hover +25.6% and inlays +23.5%, and comparing `2e724b5` to +`efd9630` moves neither. The entire first-use delta is `2e724b5`'s: +snapshots stopped retaining parse trees, so the first hover or inlay per +snapshot composes the file on demand. The deltas match one composition +each (hover +74.6 ms, inlays +63.8 ms, against 60 ms of compose). + +## 4. Interpreting service-workload deltas + +The parked branch's number misled because the workload lumps three costs +into one number. A service request's cost decomposes as: + +1. The parse: the snapshot rebuild, paid per edit. +2. First use: one composition per file per snapshot, paid by whichever + interaction needs the tree first. +3. Repeated use: cached, about zero. + +Against that, the three designs differ on two axes: + +| | parse side | first use | repeated use | +|---|---|---|---| +| `2b6502e` (retained trees) | composes every file's tree and keeps them | free | hover/inlay cached; folding and selection re-compose per request | +| parked `bc4b49d` (records) | builds a canonical record for every file, plus the model | projects a tree from the record, more than a compose | cached from the record | +| `efd9630` (this branch) | lighter than the base | one text composition, about 38% of a parse | everything cached | + +The parked branch regressed the parse side (paid on every edit, which is +why large/service moved +39.0%) and the first-use side (projection is +dearer than compose). This branch regresses only the first-use side, by +exactly one composition, and improves the parse side's memory. The hover +workload's cold-snapshot convention (docs/12 § 4) is the right place for +that first-use number, because an editor session rebuilds the snapshot on +every edit, so first use recurs once per file per edit. The repeated-use +side is not A/B-able — no existing workload drives folding or selection +repeatedly, and a new workload has no base number by docs/12 § 5 — so the +branch pins it instead: the `source-structure` reach test counts `compose` +calls and requires each measured pass to compose the text once, shared by +fifteen requests. + +Residual inefficiency: a clean file is composed twice per snapshot, once +from the snapshot text for the hover index and once from the buffer text +for the workspace cache. Sharing one tree between them when the texts are +identical is the first revisit candidate (docs/15 § 2). + +## 5. Decision + +Keep `2e724b5`. The trade is one first-use composition per file per +snapshot (about 60 ms on the 304 KiB corpus entry, one time per edit) for +a persistent memory reduction in the long-running server: large/service +peak -20.1%, kept -24.0% (measured on the spelling-carrying corpus), and +effective-types kept -27.5%, with the edit loop's time within noise. +Accepted with the revisit note in docs/15 § 2; the interpretation rule +above is recorded in docs/12 § 5. + +## 6. Verification + +Gate (`uv run ruff check . && uv run ruff format --check . && uv run mypy +fastraml/ && uv run pytest -q`) green at `efd9630`. Measurement commands: + +```bash +uv run python -m bench ab 2b6502e --bench hover --bench inlays --bench effective-types --config unwrap --rounds 3 --repeat 3 +uv run python -m bench ab 2b6502e --bench large --config service --rounds 3 --repeat 3 +uv run python -m bench ab 2b6502e --bench large --config parse --config unwrap --config validate --config unwrap+validate --config unwrap+graph --config unwrap+lint --config unwrap+occurrences --rounds 3 --repeat 3 +uv run python -m bench ab 2b6502e --bench endpoints --config parse --config unwrap --config unwrap+lint --rounds 3 --repeat 3 +uv run python -m bench ab 2b6502e --bench small --config parse --config unwrap --config unwrap+lint --rounds 3 --repeat 3 +uv run python -m bench ab 2b6502e --bench datatype-fragments --config parse --config unwrap --config unwrap+lint --rounds 3 --repeat 3 +uv run python -m bench linearity --bench source-structure --repeat 3 +``` diff --git a/docs/research/2026-10-10/service-recovery-plan.md b/docs/research/2026-10-10/service-recovery-plan.md index 96d24a3..ca4fb2b 100644 --- a/docs/research/2026-10-10/service-recovery-plan.md +++ b/docs/research/2026-10-10/service-recovery-plan.md @@ -125,8 +125,12 @@ Current execution: - Stage A: measurement work preserved as `7c954ba` on the parked branch; the master-based `test/service-workload-baseline` port passes the Windows gate (5,810 passed, 67 skipped, 1 xfailed) and both mixed-workload linearity checks. - GitHub/Linux verification and measurement-only integration are next. -- Stage B: pending master-controlled acceptance measurements. + All GitHub checks passed, including Ubuntu, TCK and benchmarks; PR #1 merged + into master as `d2b8a02`. +- Stage B: duplicate hover/workspace ownership is fixed and locally verified. + The user explicitly accepted the remaining first-use and +5.4% post-query + allocation trade-off with “Merge, then recover”; GitHub verification and the + production merge are next. Results and the decision are in the integration report. - Stage C: pending an accepted baseline. Update these entries as each stage completes. Keep measurements in the dated diff --git a/fastraml/parser/fragments.py b/fastraml/parser/fragments.py index 2deb538..389dad1 100644 --- a/fastraml/parser/fragments.py +++ b/fastraml/parser/fragments.py @@ -66,7 +66,7 @@ resolve_ref_uri, strip_uri_suffix, ) -from fastraml.parser.references import resolve_library_reference, resolve_reference +from fastraml.parser.references import resolve_library_reference, resolve_reference, visible_references from fastraml.parser.resourcetypes import ResourceTypeDefinition, make_resource_type_definition from fastraml.parser.security import SecuritySchemeDefinition, make_security_scheme_definition from fastraml.parser.traits import TraitDefinition, make_trait_definition @@ -220,6 +220,8 @@ class ReferenceResolver(Fragment, SecuritySchemeResolver, Protocol): and therefore all of them may resolve a qualified name. """ + def visible_names(self, kind: str) -> Iterator[tuple[str, Declaration]]: ... + def reference_type(self, name: str) -> BaseShape: ... def reference_annotation_type(self, name: str) -> BaseShape: ... @@ -394,6 +396,9 @@ class _UsesOnlyFragment(_BaseFragment): __slots__ = () + def visible_names(self, kind: str) -> Iterator[tuple[str, Declaration]]: + return _visible_declarations(None, None, self.uses, kind) + def reference_type(self, name: str) -> BaseShape: return resolve_library_reference(self.uses, name, _pick_type) @@ -445,6 +450,32 @@ def _pick_security_scheme(library: Library, name: str) -> SecuritySchemeDefiniti } +def _visible_table(library: Library, kind: str) -> Mapping[str, Declaration]: + table: Mapping[str, Declaration] = getattr(library, _DECLARATION_TABLES[kind]) + return table + + +def _visible_declarations( + local: Mapping[str, Declaration] | None, + types: Mapping[str, Declaration] | None, + uses: Mapping[str, LibraryLink], + kind: str, +) -> Iterator[tuple[str, Declaration]]: + primary = visible_references(local, uses, lambda library: _visible_table(library, kind)) + if kind != FACET_ANNOTATION_TYPES: + yield from primary + return + seen: set[str] = set() + for name, entity in primary: + seen.add(name) + yield name, entity + # Annotation lookup tries the whole annotation namespace before falling + # back to the type namespace, including before a local dotted data type. + for name, entity in visible_references(types, uses, lambda library: _visible_table(library, FACET_TYPES)): + if name not in seen: + yield name, entity + + class _NameResolver: """The five name resolvers, written once over two hooks. @@ -469,6 +500,11 @@ def _resolve[T](self, kind: str, name: str, pick: Callable[[Library, str], T | N def library_link(self, prefix: str) -> LibraryLink | None: return self._libraries().get(prefix) + def visible_names(self, kind: str) -> Iterator[tuple[str, Declaration]]: + local: Mapping[str, Declaration] | None = self._declared(kind) + types = self._declared(FACET_TYPES) if kind == FACET_ANNOTATION_TYPES else None + return _visible_declarations(local, types, self._libraries(), kind) + def reference_type(self, name: str) -> BaseShape: return self._resolve(FACET_TYPES, name, _pick_type) @@ -542,6 +578,8 @@ def _decode_declarations(self, key: Node, value: Node, declarations: _Declaratio raml = self._raml name = key.value if name in (FACET_TYPES, FACET_SCHEMAS): + if name == FACET_SCHEMAS: + raml.record_syntax_alias(self, key, self.location) unmarshal_types(raml, declarations.types(key, value), self.location, self.types) elif name == FACET_ANNOTATION_TYPES: unmarshal_types(raml, value, self.location, self.annotation_types, is_annotation=True) diff --git a/fastraml/parser/includes.py b/fastraml/parser/includes.py index 1aeb2fa..830ed80 100644 --- a/fastraml/parser/includes.py +++ b/fastraml/parser/includes.py @@ -34,7 +34,9 @@ ) if TYPE_CHECKING: - from fastraml.parser.fragments import LibraryLink, ReferenceResolver + from collections.abc import Iterator + + from fastraml.parser.fragments import Declaration, LibraryLink, ReferenceResolver from fastraml.parser.resourcetypes import ResourceTypeDefinition from fastraml.parser.security import SecuritySchemeDefinition from fastraml.parser.traits import TraitDefinition @@ -285,15 +287,19 @@ def load_include(raml: Raml, node: Node, target: str, location: str) -> Node: through no cache and recording nothing: for a reader outside the parse, such as `join` comparing a target no applied template reached (docs/20 § 2). """ - return _compose_include(raml, node, _load(raml, node, target, location), target, location)[0] + return _compose_include(raml, node, _load(raml, node, target, location), target, location, store_text=False)[0] -def _compose_include(raml: Raml, node: Node, data: bytes, target: str, location: str) -> tuple[Node, str]: +def _compose_include( # noqa: PLR0913 - include coordinates plus text retention + raml: Raml, node: Node, data: bytes, target: str, location: str, *, store_text: bool = True +) -> tuple[Node, str]: """The include's content, and its RAML header line, `''` when it has none.""" try: text = decode_source(data) except UnicodeDecodeError as err: raise node_error('include is not UTF-8', location, node, info={'path': target}) from err + if store_text: + raml.store_source_text(target, text) ref = node.value if not _composes_as_yaml(ref): # Spec section Resolving Includes: any other file is included as a scalar. @@ -385,6 +391,9 @@ def security_scheme_definition(self, name: str) -> SecuritySchemeDefinition: def library_link(self, prefix: str) -> LibraryLink | None: return self.host.library_link(prefix) + def visible_names(self, kind: str) -> Iterator[tuple[str, Declaration]]: + return self.host.visible_names(kind) + def content_anchor(anchor: ReferenceResolver | None, location: str) -> ReferenceResolver | None: """`anchor`, or an `IncludedContent` of it when what is decoded at diff --git a/fastraml/parser/references.py b/fastraml/parser/references.py index 009d561..3da0250 100644 --- a/fastraml/parser/references.py +++ b/fastraml/parser/references.py @@ -13,7 +13,7 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from collections.abc import Callable, Mapping + from collections.abc import Callable, Iterator, Mapping from fastraml.parser.fragments import Library, LibraryLink @@ -22,6 +22,7 @@ 'cut_last', 'resolve_library_reference', 'resolve_reference', + 'visible_references', ] @@ -109,6 +110,32 @@ def resolve_library_reference[T]( return _pick_from_library(uses, prefix, suffix, pick) +def visible_references[T]( + local: Mapping[str, T] | None, + uses: Mapping[str, LibraryLink], + table: Callable[[Library], Mapping[str, T]], +) -> Iterator[tuple[str, T]]: + """Enumerate names that resolve in this namespace, with local precedence. + + Only one library hop, including dotted import prefixes. Check the round + trip rather than inventing a second interpretation of dotted names. + """ + yield from (local or {}).items() + for prefix, link in uses.items(): + if link.link is None: + continue + for name, entity in table(link.link).items(): + qualified = f'{prefix}.{name}' + if local is not None and qualified in local: + continue + try: + resolved = resolve_reference(local, uses, qualified, lambda library, key: table(library).get(key)) + except UnresolvedReferenceError: + continue + if resolved is entity: + yield qualified, entity + + def _pick_from_library[T]( uses: Mapping[str, LibraryLink] | None, prefix: str, diff --git a/fastraml/registry.py b/fastraml/registry.py index 10bc476..d02bb72 100644 --- a/fastraml/registry.py +++ b/fastraml/registry.py @@ -23,6 +23,7 @@ from fastraml.domains import DomainLocation from fastraml.errors import Accumulator, RamlError from fastraml.loaders import SchemeLoader +from fastraml.sourceinfo import KeywordUse from fastraml.yamlnode import AUTHORED_NODES, DEFAULT_MAX_DEPTH, mark_subtree if TYPE_CHECKING: @@ -211,6 +212,7 @@ class Raml: 'include_refs', 'shapes', 'substitutions', + 'syntax_aliases', # --- work queues ----------------------------------------------------- '_discriminator_shapes', 'unresolved_shapes', @@ -297,6 +299,9 @@ def __init__( # noqa: PLR0913 - the parse's configuration, keyword-only, one fi #: Each scalar a template substitution produced, and the caller's #: values in it: where a name in it was written (docs/08 § 5.1). self.substitutions: Substitutions = {} + #: Accepted compatibility spellings by entity ID; only used spellings + #: allocate a record. The parser records syntax, a view judges it. + self.syntax_aliases: dict[int, list[KeywordUse]] = {} # A worklist, drained from the left in P7 while resolution appends to # the right; a deque keeps both ends O(1). @@ -634,6 +639,11 @@ def store_source_text(self, uri: str, text: str) -> None: if self.retain_text: self.source_texts[uri] = text + def record_syntax_alias(self, entity: Identified, key: Node, location: str) -> None: + """Where a decoder accepted a compatibility keyword spelling.""" + use = KeywordUse(self.location_of(key, location), key.position, key.value) + self.syntax_aliases.setdefault(entity.id, []).append(use) + def put_source_info(self, entity_id: int, key: Node | None, value: Node) -> None: """Index an entity's authored nodes when source retention is on.""" if self.source_info is not None: diff --git a/fastraml/service/datahover.py b/fastraml/service/datahover.py index 036dccb..371c8e9 100644 --- a/fastraml/service/datahover.py +++ b/fastraml/service/datahover.py @@ -11,9 +11,8 @@ from dataclasses import dataclass from typing import TYPE_CHECKING -from fastraml.errors import RamlError from fastraml.types.base import Property -from fastraml.types.complex_ import ArrayShape, ObjectShape, RecursiveShape, UnionShape +from fastraml.types.navigation import children if TYPE_CHECKING: from collections.abc import Iterable, Iterator @@ -21,6 +20,7 @@ from fastraml.datanode import DataNode, ValueNode from fastraml.positions import Position from fastraml.types.base import BaseShape + from fastraml.types.navigation import TypedChild @dataclass(frozen=True, slots=True, eq=False) @@ -97,36 +97,41 @@ def in_range(self, uri: str, span: Position) -> list[DataTarget]: def _targets(root: DataRoot) -> Iterator[DataTarget]: if root.data.key_location is not None and root.data.key_pos.is_known: yield DataTarget(root.data.key_location, root.data.key_pos, root.base, root.name, root.role, is_key=True) - pending: list[tuple[ValueNode, BaseShape, str, str, bool | None]] = [ - (root.data.value, root.base, root.name, root.role, None) - ] + value, base, name, role = root.data.value, root.base, root.name, root.role + required: bool | None = None + pending: list[tuple[Iterator[TypedChild], str]] = [] seen: set[tuple[ValueNode, BaseShape]] = set() - while pending: - value, base, name, role, required = pending.pop() - if (value, base) in seen: - continue - seen.add((value, base)) - uri = value.location or root.data.location - # Collections have child key/value spans. Indexing their whole extent - # would make whitespace and unknown fields inherit the parent hover. - # An encoded JSON scalar has no nested spans: its root alone is known. - if value.is_scalar or (value.position.is_known and not _has_child_spans(value)): - yield DataTarget(uri, value.position, base, name, role, required) - for branch in _branches(base, value): - shape = branch.shape - if isinstance(shape, ObjectShape) and value.mapping is not None: - for entry in value.mapping.entries: - prop = shape.property_for(entry.key) - if prop is None: - continue - presence = prop.required if isinstance(prop, Property) else None - child_role = 'object property' if isinstance(prop, Property) else 'pattern-matched property' - yield DataTarget(uri, entry.key_pos, prop.base, entry.key, child_role, presence, is_key=True) - pending.append((entry.value, prop.base, entry.key, child_role, presence)) - elif isinstance(shape, ArrayShape) and shape.items is not None and value.sequence is not None: - pending.extend( - (item.value, shape.items, '', 'array item', None) for item in reversed(value.sequence.items) - ) + while True: + if (value, base) not in seen: + seen.add((value, base)) + uri = value.location or root.data.location + # Collections have child key/value spans. Indexing their whole extent + # would make whitespace and unknown fields inherit the parent hover. + # An encoded JSON scalar has no nested spans: its root alone is known. + if value.is_scalar or (value.position.is_known and not _has_child_spans(value)): + yield DataTarget(uri, value.position, base, name, role, required) + if not value.is_scalar: + pending.append((children(base, value), uri)) + # Resume one sibling at a time; frames borrow the model's child iterator + # instead of buffering every typed child and copying it into a worklist. + while pending: + siblings, uri = pending[-1] + child = next(siblings, None) + if child is None: + pending.pop() + continue + prop = child.declaration + required = prop.required if isinstance(prop, Property) else None + if prop is None: + name, role = '', 'array item' + else: + name = str(child.name) + role = 'object property' if isinstance(prop, Property) else 'pattern-matched property' + yield DataTarget(uri, child.key_pos, child.base, name, role, required, is_key=True) + value, base = child.value, child.base + break + else: + return def _has_child_spans(value: ValueNode) -> bool: @@ -135,26 +140,3 @@ def _has_child_spans(value: ValueNode) -> bool: if value.sequence is not None: return any(item.value_pos.is_known for item in value.sequence.items) return False - - -def _branches(base: BaseShape, value: ValueNode) -> Iterator[BaseShape]: - pending = [base] - seen: set[BaseShape] = set() - while pending: - current = pending.pop() - if current in seen: - continue - seen.add(current) - shape = current.shape - if isinstance(shape, RecursiveShape): - pending.append(shape.head) - elif current.alias is not None: - pending.append(current.alias) - elif isinstance(shape, UnionShape): - try: - selected = shape.select(value.raw) - except RamlError: - continue # a discriminator that names no alternative selects none - pending.extend(reversed(shape.any_of or ()) if selected is None else (selected,)) - else: - yield current diff --git a/fastraml/service/hover.py b/fastraml/service/hover.py index e1e61ec..ea82472 100644 --- a/fastraml/service/hover.py +++ b/fastraml/service/hover.py @@ -32,6 +32,7 @@ type_label, underlying_type, ) +from fastraml.service.source import original_tree from fastraml.service.text import Lines from fastraml.types.base import BaseShape, Parameter, facets_of from fastraml.types.complex_ import ObjectShape, UnknownShape @@ -44,14 +45,16 @@ from fastraml.views import authored from fastraml.views.occurrences import DECLARATION_KINDS, Kind, Link, Occurrence, Occurrences, Role from fastraml.views.render import type_name -from fastraml.yamlnode import TAG_INCLUDE, TAG_STR, Node, NodeKind, compose, pairs +from fastraml.yamlnode import TAG_INCLUDE, TAG_STR, Node, NodeKind, backend_name, compose, pairs if TYPE_CHECKING: from collections.abc import Iterable, Iterator + from weakref import ReferenceType from fastraml.parser.endpoints import Body, EndPoint, Operation, Response from fastraml.positions import Position from fastraml.registry import Raml + from fastraml.service.source import Sources from fastraml.types.base import ScalarFacet from fastraml.types.expressions import ExprCache, RdtNode @@ -106,14 +109,29 @@ class Hover: '_responses', '_root', '_sites', + '_source_backend', '_source_builtins', + '_source_generation', + '_sources', '_subjects', ) - def __init__(self, raml: Raml, root: str, occurrences: Occurrences) -> None: + def __init__( # noqa: PLR0913 - the snapshot's borrowed source owner and composition policy + self, + raml: Raml, + root: str, + occurrences: Occurrences, + *, + sources: ReferenceType[Sources] | None = None, + source_generation: int = 0, + source_backend: str = '', + ) -> None: self._raml = raml self._root = root.rpartition('/')[0] + '/' self._occurrences = occurrences + self._sources = sources + self._source_generation = source_generation + self._source_backend = source_backend self._subjects: dict[int, _Subject] = {} self._sites: dict[tuple[str, int, int], _Subject] = {} self._mappings: dict[tuple[str, int, int], _Subject] = {} @@ -359,11 +377,22 @@ def _node(self, uri: str) -> Node | None: if uri in self._nodes: return self._nodes[uri] node = self._raml.source_nodes.get(uri) or self._raml.include_nodes.get(uri) - if node is None: - text = self._raml.source_texts.get(uri) - if text is not None: - with suppress(RamlError): - node = compose(text, uri=uri, max_depth=self._raml.max_depth) + text = self._raml.source_texts.get(uri) + shared = None if self._sources is None else self._sources() + if node is not None and original_tree(uri, node) is None: + self._nodes[uri] = node + return node + if text is not None and shared is not None: + node = shared.node( + uri, + text, + max_depth=self._raml.max_depth, + generation=self._source_generation, + original=node if self._source_backend == backend_name() else None, + ) + elif node is None and text is not None: + with suppress(RamlError): + node = compose(text, uri=uri, max_depth=self._raml.max_depth) self._nodes[uri] = node return node diff --git a/fastraml/service/lsp.py b/fastraml/service/lsp.py index c6db186..521d81d 100644 --- a/fastraml/service/lsp.py +++ b/fastraml/service/lsp.py @@ -497,19 +497,21 @@ def links(params: types.DocumentLinkParams) -> list[types.DocumentLink] | None: def folding(params: types.FoldingRangeParams) -> list[types.FoldingRange] | None: if (uri := _file(params.text_document.uri)) is None: return None - text = self.service.text(uri) or '' - return [types.FoldingRange(start - 1, end - 1) for start, end in queries.folding_ranges(text, uri)] + return [ + types.FoldingRange(start - 1, end - 1) + for start, end in queries.folding_ranges_of(self.service.source(uri)) + ] @feature(types.TEXT_DOCUMENT_SELECTION_RANGE) def selection(params: types.SelectionRangeParams) -> list[types.SelectionRange] | None: if (at := self._in(params.text_document.uri)) is None: return None uri, positions = at - text = self.service.text(uri) or '' + root = self.service.source(uri) found: list[types.SelectionRange] = [] for position in params.positions: parent: types.SelectionRange | None = None - for span in reversed(queries.selection_ranges(text, uri, *positions.to_server(uri, position))): + for span in reversed(queries.selection_ranges_of(root, *positions.to_server(uri, position))): parent = types.SelectionRange(positions.range(uri, span), parent) found.append(parent or types.SelectionRange(types.Range(position, position))) return found diff --git a/fastraml/service/queries.py b/fastraml/service/queries.py index a013f76..6dfb1a4 100644 --- a/fastraml/service/queries.py +++ b/fastraml/service/queries.py @@ -47,17 +47,20 @@ 'detail_line', 'diagnostics', 'folding_ranges', + 'folding_ranges_of', 'highlights', 'hover', 'links', 'references', 'selection_ranges', + 'selection_ranges_of', 'subtypes', 'supertypes', 'suppression', 'symbol', 'tree', 'type_at', + 'visible_names', 'workspace_symbols', ] @@ -374,6 +377,19 @@ def workspace_symbols(snapshots: Iterable[Snapshot], query: str) -> list[Symbol] return found +def visible_names(snapshot: Snapshot, uri: str, kind: str) -> list[Symbol]: + """Completion candidates from a fragment's own resolver, never all declarations.""" + raml = snapshot.raml + anchor = None if raml is None else raml.resolver_at(uri) + if anchor is None: + return [] + return [ + each + for name, entity in anchor.visible_names(kind) + if (each := symbol(name, DECLARATION_KINDS[kind], entity)) is not None + ] + + # -- links, folding and selection ------------------------------------------------------- @@ -395,8 +411,27 @@ def links(snapshot: Snapshot, uri: str) -> list[Site]: def folding_ranges(text: str, uri: str) -> list[tuple[int, int]]: """Each mapping and sequence value spanning lines, from its key's line to its last leaf's, 1-based, from the text alone. + + The composition happens per call; repeated requests for one file should + use `Workspace.source`. + """ + return folding_ranges_of(_compose(text, uri)) + + +def selection_ranges(text: str, uri: str, line: int, column: int) -> list[Position]: + """The spans holding `line:column`, innermost first: a token, its pair, + the pair's mapping, and outward. + + The composition happens per call; repeated requests for one file should + use `Workspace.source`. + """ + return selection_ranges_of(_compose(text, uri), line, column) + + +def folding_ranges_of(root: Node | None) -> list[tuple[int, int]]: + """Each mapping and sequence value of `root` spanning lines, from its key's + line to its last leaf's, 1-based, from the tree alone. """ - root = _compose(text, uri) if root is None: return [] found: set[tuple[int, int]] = set() @@ -407,11 +442,10 @@ def folding_ranges(text: str, uri: str) -> list[tuple[int, int]]: return sorted(found) -def selection_ranges(text: str, uri: str, line: int, column: int) -> list[Position]: - """The spans holding `line:column`, innermost first: a token, its pair, - the pair's mapping, and outward. +def selection_ranges_of(root: Node | None, line: int, column: int) -> list[Position]: + """The spans of `root` holding `line:column`, innermost first: a token, its + pair, the pair's mapping, and outward, from the tree alone. """ - root = _compose(text, uri) if root is None: return [] found: list[Position] = [] diff --git a/fastraml/service/source.py b/fastraml/service/source.py new file mode 100644 index 0000000..24d23e5 --- /dev/null +++ b/fastraml/service/source.py @@ -0,0 +1,107 @@ +"""On-demand composed trees shared by current-text and snapshot consumers.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from fastraml.errors import RamlError +from fastraml.yamlnode import NodeKind, backend_name, compose + +if TYPE_CHECKING: + from fastraml.yamlnode import Node + +_BOM = chr(0xFEFF) + + +def original_tree(uri: str, root: Node | None) -> Node | None: + """Only unchanged YAML/RAML containers have the standalone composition policy. + + JSON includes can normalize whitespace; non-YAML includes and external + schemas can use synthetic scalars. Neither is a generic composed tree. + """ + path = uri.split('#', 1)[0].split('?', 1)[0] + if root is not None and root.kind is not NodeKind.SCALAR and path.lower().endswith(('.raml', '.yaml', '.yml')): + return root + return None + + +@dataclass(slots=True, eq=False) +class _Entry: + text: str + max_depth: int + backend: str + root: Node | None + + +class Sources: + """Workspace-owned trees; snapshots borrow this owner rather than retain it.""" + + __slots__ = ('__weakref__', '_changed', '_entries', '_policy_generation', 'backend', 'generation', 'max_depth') + + def __init__(self, max_depth: int) -> None: + self._entries: dict[str, _Entry] = {} + self._changed: dict[str, int] = {} + self._policy_generation = 0 + self.max_depth = max_depth + self.backend = backend_name() + self.generation = 0 + + def configure(self, max_depth: int) -> None: + """The current owner's composition policy invalidates older publications.""" + backend = backend_name() + if (max_depth, backend) != (self.max_depth, self.backend): + self._entries.clear() + self.generation += 1 + self._policy_generation = self.generation + self.max_depth, self.backend = max_depth, backend + + def discard(self, uri: str, *, read: bool) -> None: + cached = self._entries.pop(uri, None) + self.generation += 1 + # Watchers see unrelated files too. Remember changes only for source + # a snapshot/cache has used, including any already tracked old reader. + if read or cached is not None or uri in self._changed: + self._changed[uri] = self.generation + + def lookup(self, uri: str, text: str, max_depth: int) -> _Entry | None: + """An equal parsed input/policy can share a successful or failed result.""" + entry = self._entries.get(uri) + if ( + entry is not None + and entry.text == text.removeprefix(_BOM) + and entry.max_depth == max_depth + and entry.backend == backend_name() + ): + return entry + return None + + def node( + self, + uri: str, + text: str, + *, + max_depth: int, + generation: int | None = None, + original: Node | None = None, + ) -> Node | None: + """Compose on a miss; an old snapshot cannot replace the current owner.""" + entry = self.lookup(uri, text, max_depth) + text = text.removeprefix(_BOM) + backend = backend_name() + current = ( + (generation is None or generation >= max(self._changed.get(uri, 0), self._policy_generation)) + and max_depth == self.max_depth + and backend == self.backend + ) + if entry is not None and not (original is not None and current): + return entry.root + root = original + if root is None: + try: + root = compose(text, uri=uri, max_depth=max_depth) + except RamlError: + root = None + if current: + self._entries[uri] = _Entry(text, max_depth, backend, root) + return root diff --git a/fastraml/service/workspace.py b/fastraml/service/workspace.py index 7094749..7755947 100644 --- a/fastraml/service/workspace.py +++ b/fastraml/service/workspace.py @@ -24,6 +24,7 @@ from dataclasses import dataclass, field from fnmatch import fnmatch from typing import TYPE_CHECKING, Final +from weakref import ref from fastraml.errors import ErrorKind, RamlError from fastraml.loaders import SafeFileLoader @@ -31,19 +32,22 @@ from fastraml.parser.fragments import FragmentKind, identify_fragment from fastraml.service.hover import Hover from fastraml.service.lenses import EffectiveViews +from fastraml.service.source import Sources, original_tree from fastraml.service.text import Lines from fastraml.uris import file_uri_to_path, path_to_file_uri, relative_to from fastraml.views.lint import configured_linter from fastraml.views.occurrences import build_occurrences -from fastraml.yamlnode import read_head +from fastraml.yamlnode import backend_name, read_head if TYPE_CHECKING: from collections.abc import Iterable, Iterator, Mapping, Sequence + from weakref import ReferenceType from fastraml.config import FastRamlConfig from fastraml.registry import Raml from fastraml.views.lint import Finding, Linter from fastraml.views.occurrences import Occurrences + from fastraml.yamlnode import Node __all__ = ['Buffer', 'Snapshot', 'Workspace', 'canonical'] @@ -102,6 +106,9 @@ class Snapshot: _findings: list[Finding] | None = field(default=None, repr=False) _hover: Hover | None = field(default=None, repr=False) _effective_views: EffectiveViews | None = field(default=None, repr=False) + sources: ReferenceType[Sources] | None = field(default=None, repr=False) + source_generation: int = field(default=0, repr=False) + source_backend: str = field(default='', repr=False) @property def effective_views(self) -> EffectiveViews | None: @@ -114,7 +121,14 @@ def hover(self) -> Hover | None: """The source and model indices used by author-facing hover.""" occurrences = self.occurrences if self._hover is None and self.raml is not None and occurrences is not None: - self._hover = Hover(self.raml, self.root, occurrences) + self._hover = Hover( + self.raml, + self.root, + occurrences, + sources=self.sources, + source_generation=self.source_generation, + source_backend=self.source_backend, + ) return self._hover @property @@ -180,6 +194,7 @@ def __init__( self.buffers: dict[str, Buffer] = {} self._roots: list[str] | None = None self._snapshots: dict[str, Snapshot] = {} + self._sources = Sources(self.config.parser.max_depth) #: What each dropped snapshot read: the order `serving` tries roots in. self._last_read: dict[str, frozenset[str]] = {} #: Whether a snapshot was dropped since the last collection. @@ -232,6 +247,30 @@ def lines(self, uri: str) -> Lines: buffer = self.buffers.get(canonical(uri)) return buffer.lines if buffer is not None else Lines(self.text(uri) or '') + def source(self, uri: str) -> Node | None: + """The composed tree of `uri`'s current text, composed once per text (docs/21 § 4).""" + uri = canonical(uri) + text = self.text(uri) + if text is None: + return None + depth = self.config.parser.max_depth + self._sources.configure(depth) + original = None + normalized = text.removeprefix(BOM) + for snapshot in self._snapshots.values(): + raml = snapshot.raml + if ( + raml is not None + and raml.retain_source + and raml.max_depth == depth + and snapshot.source_backend == self._sources.backend + and raml.source_texts.get(uri) == normalized + ): + original = original_tree(uri, raml.source_nodes.get(uri)) + if original is not None: + break + return self._sources.node(uri, text, max_depth=depth, original=original) + def _put(self, uri: str, buffer: Buffer) -> None: """Hold `buffer`, and drop what read other text for `uri`. @@ -264,11 +303,14 @@ def _touched(self, uri: str, *, appeared: bool) -> None: A file that appeared may be one a failed include was looking for, so a snapshot that ended in an error is dropped then too. """ + read = False for root, snapshot in list(self._snapshots.items()): + read |= uri in snapshot.read if uri in snapshot.read or (appeared and snapshot.error is not None): del self._snapshots[root] self._last_read[root] = snapshot.read self._garbage = True + self._sources.discard(uri, read=read) # -- roots and snapshots -------------------------------------------------- @@ -347,6 +389,8 @@ def readers(self) -> dict[str, list[str]]: return found def _parse(self, root: str) -> Snapshot: + self._sources.configure(self.config.parser.max_depth) + source_backend = backend_name() disk = self._disk(root) if disk is None: refused = RamlError.new('path is outside the workspace root', root, info={'path': root}) @@ -374,7 +418,16 @@ def _parse(self, root: str) -> Snapshot: else RamlError.wrap('load resource', err, root, kind=ErrorKind.READING) ) return Snapshot(root, None, failure, frozenset({root})) - return Snapshot(root, raml, error, _read(raml, root), linter=self.linter) + return Snapshot( + root, + raml, + error, + _read(raml, root), + linter=self.linter, + sources=ref(self._sources), + source_generation=self._sources.generation, + source_backend=source_backend, + ) @property def linter(self) -> Linter: diff --git a/fastraml/sourceinfo.py b/fastraml/sourceinfo.py new file mode 100644 index 0000000..2e566be --- /dev/null +++ b/fastraml/sourceinfo.py @@ -0,0 +1,23 @@ +"""Facts a decoder records when it accepts a compatibility spelling. + +A `KeywordUse` names where a deprecated keyword was written. It is a model +fact the registry stores and a view may report on: not a diagnostic, and it +keeps no YAML node (docs/04 § 5, docs/05 § 3). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from fastraml.positions import Position + + +@dataclass(frozen=True, slots=True, eq=False) +class KeywordUse: + """An accepted compatibility spelling, not a lint finding or retained tree.""" + + location: str + position: Position + name: str diff --git a/fastraml/types/navigation.py b/fastraml/types/navigation.py new file mode 100644 index 0000000..2448f74 --- /dev/null +++ b/fastraml/types/navigation.py @@ -0,0 +1,74 @@ +"""Typed-value descent without using validation to guess a union alternative. + +Matching uses the same property and discriminator operations as validation +(docs/05 § 4 and § 6). Callers decide how to present ambiguous candidates. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from fastraml.errors import RamlError +from fastraml.positions import UNKNOWN, Position +from fastraml.types.complex_ import ArrayShape, ObjectShape, RecursiveShape, UnionShape + +if TYPE_CHECKING: + from collections.abc import Iterator + + from fastraml.datanode import ValueNode + from fastraml.types.base import BaseShape, PatternProperty, Property + + +@dataclass(frozen=True, slots=True, eq=False) +class TypedChild: + name: str | int + value: ValueNode + base: BaseShape + declaration: Property | PatternProperty | None = None + key_pos: Position = UNKNOWN + + +def branches(base: BaseShape, value: ValueNode) -> Iterator[BaseShape]: + """All structural candidates, or none for an unknown discriminator tag.""" + pending = [iter((base,))] + seen: set[BaseShape] = set() + while pending: + current = next(pending[-1], None) + if current is None: + pending.pop() + continue + if current in seen: + continue + seen.add(current) + shape = current.shape + if isinstance(shape, RecursiveShape): + pending.append(iter((shape.head,))) + elif current.alias is not None: + pending.append(iter((current.alias,))) + elif isinstance(shape, UnionShape): + try: + selected = shape.select(value.raw) + except RamlError: + continue + pending.append(iter(shape.any_of or ()) if selected is None else iter((selected,))) + else: + yield current + + +def children(base: BaseShape, value: ValueNode) -> Iterator[TypedChild]: + """The declarations governing each immediate child, in branch/data order. + + Invalid values retain known field types; additional fields have none. Array + items are identified by integer indices, not invented property declarations. + """ + for branch in branches(base, value): + shape = branch.shape + if isinstance(shape, ObjectShape) and value.mapping is not None: + for entry in value.mapping.entries: + prop = shape.property_for(entry.key) + if prop is not None: + yield TypedChild(entry.key, entry.value, prop.base, prop, entry.key_pos) + elif isinstance(shape, ArrayShape) and shape.items is not None and value.sequence is not None: + for index, item in enumerate(value.sequence.items): + yield TypedChild(index, item.value, shape.items) diff --git a/fastraml/types/shape.py b/fastraml/types/shape.py index 962316f..1fc69aa 100644 --- a/fastraml/types/shape.py +++ b/fastraml/types/shape.py @@ -374,6 +374,8 @@ def _decode( # noqa: PLR0912 - one pass over the common-facet vocabulary (docs/ if type_node is not None: raise node_error('type and schema are mutually exclusive', location, key) type_node = value + if key.value == fn.FACET_SCHEMA: + raml.record_syntax_alias(base, key, location) case fn.FACET_DISPLAY_NAME: base.display_name = make_string_facet(raml, key, value, location) case fn.FACET_DESCRIPTION: diff --git a/fastraml/views/lint/rules/schema.py b/fastraml/views/lint/rules/schema.py index 873fd1e..25a6bd8 100644 --- a/fastraml/views/lint/rules/schema.py +++ b/fastraml/views/lint/rules/schema.py @@ -19,7 +19,6 @@ from fastraml.views.graph import is_declaration from fastraml.views.lint.engine import Category, Finding, RuleMeta, Severity from fastraml.views.lint.mediatypes import media_essence -from fastraml.yamlnode import NodeKind, pairs if TYPE_CHECKING: from collections.abc import Iterable @@ -45,8 +44,6 @@ class DeprecatedSchemas: """The deprecated `schemas:` alias for `types:`, and `schema:` for `type:`.""" - requires_source: ClassVar = True - meta: ClassVar = RuleMeta( id='deprecated-schemas', category=Category.SPEC, @@ -67,46 +64,32 @@ class DeprecatedSchemas: ) def type_(self, ctx: Context, iri: str, base: BaseShape, shape_kind: str) -> Iterable[Finding]: # noqa: ARG002 - authored = None if ctx.raml.source_info is None else ctx.raml.source_info.get(base.id) - if authored is None or authored[1].kind is not NodeKind.MAPPING: - return () - # Keys only, sliced rather than paired: this runs once per type node, - # and most declarations have nothing to report. return [ ctx.at( self.meta, 'schema is deprecated; use type', - location=base.location, - position=key.position, + location=use.location, + position=use.position, iri=iri, field='schema', ) - for key in authored[1].content[::2] - if key.value == 'schema' + for use in ctx.raml.syntax_aliases.get(base.id, ()) + if use.name == 'schema' ] def unit(self, ctx: Context, iri: str, fragment: Fragment) -> Iterable[Finding]: - root = ctx.raml.source_node(fragment.location) - if root is None: - # A `type: !include schema.json` is represented by a data-type - # fragment and a graph unit, but JSON input never had a RAML root - # node to retain. Only composed RAML sources can spell `schemas:`. - return () - if root.kind is not NodeKind.MAPPING: - return () - for key, _ in pairs(root): - if key.value == 'schemas': - return ( - ctx.at( - self.meta, - 'schemas is deprecated; use types', - location=fragment.location, - position=key.position, - iri=iri, - field='schemas', - ), - ) - return () + return [ + ctx.at( + self.meta, + 'schemas is deprecated; use types', + location=use.location, + position=use.position, + iri=iri, + field='schemas', + ) + for use in ctx.raml.syntax_aliases.get(fragment.id, ()) + if use.name == 'schemas' + ] class JsonRefSiblings: diff --git a/tests/bench/test_corpus.py b/tests/bench/test_corpus.py index 15bd7fc..912843c 100644 --- a/tests/bench/test_corpus.py +++ b/tests/bench/test_corpus.py @@ -52,6 +52,7 @@ 'hover': lambda root: corpus.write_hover(root, family_count=3), 'effective-types': lambda root: corpus.write_hover(root, family_count=3), 'inlays': lambda root: corpus.write_hover(root, family_count=3), + 'source-structure': lambda root: corpus.write_hover(root, family_count=3), 'service-session': lambda root: corpus.write_hover(root, family_count=3), 'service-source-first': lambda root: corpus.write_hover(root, family_count=3), } @@ -128,6 +129,45 @@ def parse(entry, options): run_one('lenient-recovery', 'unwrap+lint', entry, repeat=1) assert len(results) == 4, 'timing and allocation both reach recovery and lint' + @pytest.mark.parametrize('count', [2, 4]) + def test_source_structure_reaches_folding_and_selection_on_the_cached_tree(self, tmp_path, monkeypatch, count): + from bench.__main__ import run_one + from fastraml.service import queries + from fastraml.service import source as source_module + + folded = [] + selected = [] + compositions = 0 + original_folding = queries.folding_ranges_of + original_selection = queries.selection_ranges_of + original_compose = source_module.compose + + def folding(root): + result = original_folding(root) + assert result + folded.append(len(result)) + return result + + def selection(root, line, column): + result = original_selection(root, line, column) + assert len(result) > 1, 'a position inside a nested structure spans at least root, pair and token' + selected.append(len(result)) + return result + + def compose(text, **kwargs): + nonlocal compositions + compositions += 1 + return original_compose(text, **kwargs) + + monkeypatch.setattr(queries, 'folding_ranges_of', folding) + monkeypatch.setattr(queries, 'selection_ranges_of', selection) + monkeypatch.setattr(source_module, 'compose', compose) + entry = corpus.write_hover(tmp_path, family_count=count) + run_one('source-structure', 'unwrap', entry, repeat=1) + assert len(folded) == 2 * 3, 'timing and allocation both reach folding' + assert len(selected) == 2 * 12, 'timing and allocation both reach selection' + assert compositions == 2, 'each measured pass composes the text once, shared by fifteen requests' + @pytest.mark.parametrize('count', [2, 4]) def test_inlays_reaches_inferred_types_data_types_and_inherited_facets(self, tmp_path, monkeypatch, count): from bench.__main__ import run_one @@ -865,6 +905,23 @@ def test_large_reaches_common_by_two_spellings(tmp_path): assert len(common) == 1 +def test_general_corpora_exercise_the_deprecated_spellings(tmp_path): + """The parse-side compatibility recording only runs on `schemas:` and `schema:`. + + The general corpora carry both spellings (docs/18 § 6), so the parse + workloads measure a document that records them. If a writer ever drops a + spelling, the recording goes unmeasured and this test says so. + """ + corpora = ( + ('large', corpus.write_large(tmp_path / 'large', type_count=24, library_count=4)), + ('endpoints', corpus.write_endpoints(tmp_path / 'endpoints', resource_count=3)), + ) + for name, entry in corpora: + raml = parse_from_path(entry) + spelled = {use.name for uses in raml.syntax_aliases.values() for use in uses} + assert spelled == {'schemas', 'schema'}, f'{name} corpus records both deprecated spellings' + + class TestBaselinesMerge: """`baseline --bench small` must not delete the other four benches' rows. diff --git a/tests/bench/test_service_session.py b/tests/bench/test_service_session.py index 3b3fb67..fb79827 100644 --- a/tests/bench/test_service_session.py +++ b/tests/bench/test_service_session.py @@ -58,6 +58,8 @@ def source(workspace, uri): snapshot = workspace.snapshot(prepared.root) if getattr(snapshot.raml, 'projection', None) is not None: assert len(compositions) == (6 if source_first else 3) + elif getattr(snapshot, 'sources', None) is not None: + assert len(compositions) == 6, 'each version shares one query composition in both request orders' else: assert len(compositions) >= 3, 'historical source readers may compose for each request' if hasattr(snapshot, 'outlines'): @@ -71,7 +73,6 @@ def source(workspace, uri): assert all(hints[index] == hints[index + 1] for index in (0, 2, 4)) assert counts['hovers'] == 9 assert counts['selections'] == 24 - assert len(workspace._snapshots) == 1 def test_session_can_query_an_unchanged_library_after_root_edits(tmp_path): diff --git a/tests/unit/test_lint.py b/tests/unit/test_lint.py index 6a2131d..951e287 100644 --- a/tests/unit/test_lint.py +++ b/tests/unit/test_lint.py @@ -795,12 +795,13 @@ def test_suppression_uses_an_included_files_location(self, workspace): def run_rule(rule_id: str, source: str, tmp_path, **options): + retain_source = options.pop('retain_source', True) config = Config(extends=(), rules=(RuleSetting(id=rule_id),)) raml = parse_from_string( source, file_name='api.raml', base_dir=tmp_path, - options=ParseOptions(unwrap=True, validate=False, retain_source=True, **options), + options=ParseOptions(unwrap=True, validate=False, retain_source=retain_source, **options), ) return Linter(builtin_registry(), config).run(raml) @@ -1236,6 +1237,24 @@ def test_deprecated_schemas_ignores_a_property_named_schema(self, tmp_path): source = '#%RAML 1.0\ntitle: t\ntypes:\n A:\n properties:\n schema: string\n' assert not run_rule('deprecated-schemas', source, tmp_path) + def test_deprecated_schemas_reports_without_retained_source(self, tmp_path): + # The decoder records each accepted spelling (docs/04 § 5, docs/05 § 3), + # so the rule runs with any retention choice (docs/18 § 6). + table = run_rule( + 'deprecated-schemas', + '#%RAML 1.0\ntitle: t\nschemas:\n U: string\n', + tmp_path, + retain_source=False, + ) + assert [(finding.position.line, finding.info) for finding in table] == [(3, {'field': 'schemas'})] + facet = run_rule( + 'deprecated-schemas', + '#%RAML 1.0\ntitle: t\ntypes:\n A:\n schema: string\n', + tmp_path, + retain_source=False, + ) + assert [(finding.position.line, finding.info) for finding in facet] == [(5, {'field': 'schema'})] + @pytest.mark.parametrize( ('resources', 'expected'), [ diff --git a/tests/unit/test_navigation.py b/tests/unit/test_navigation.py new file mode 100644 index 0000000..eb090b4 --- /dev/null +++ b/tests/unit/test_navigation.py @@ -0,0 +1,20 @@ +"""Typed-value descent (docs/05 § 6).""" + +from __future__ import annotations + +from fastraml.parser.entry import ParseOptions +from fastraml.types.navigation import children + + +def test_typed_navigation_keeps_ambiguous_and_invalid_field_candidates(workspace): + text = ( + '#%RAML 1.0\ntitle: T\ntypes:\n' + ' A: {properties: {name: string}}\n' + ' B: {properties: {name: integer}}\n' + ' Both:\n type: A | B\n example: {name: false, extra: 1}\n' + ) + raml = workspace.document(text, ParseOptions(unwrap=True)) + base = raml.types_in(raml.location)['Both'] + found = list(children(base, base.example.data.value)) + assert [(each.name, each.base.type) for each in found] == [('name', 'string'), ('name', 'integer')] + assert all(each.key_pos.is_known for each in found) diff --git a/tests/unit/test_service_queries.py b/tests/unit/test_service_queries.py index 2a32515..11e0bd8 100644 --- a/tests/unit/test_service_queries.py +++ b/tests/unit/test_service_queries.py @@ -137,6 +137,32 @@ def test_nothing_is_under_a_cursor_on_no_name(self, parsed): snapshot, folder = parsed assert queries.definition(snapshot, f'{folder}/api.raml', *_where(API, 'title')) == [] + def test_visible_names_obey_local_precedence_and_one_library_hop(self, memory_workspace): + api = '#%RAML 1.0\ntitle: T\nuses:\n lib: lib.raml\ntypes:\n lib.Person: string\n' + library = '#%RAML 1.0 Library\nuses:\n nested: other.raml\ntypes:\n Person: string\n Other: integer\n' + workspace, folder = _buffered( + memory_workspace, + { + 'api.raml': api, + 'lib.raml': library, + 'other.raml': '#%RAML 1.0 Library\ntypes:\n Hidden: boolean\n', + }, + ) + uri = f'{folder}/api.raml' + found = queries.visible_names(workspace.snapshot(uri), uri, 'types') + assert [(each.name, each.uri.rsplit('/', 1)[-1]) for each in found] == [ + ('lib.Person', 'api.raml'), + ('lib.Other', 'lib.raml'), + ] + + def test_extension_visible_names_exclude_declarations_from_later_chain_positions(self, memory_workspace): + api = '#%RAML 1.0\ntitle: T\ntypes:\n A: string\n' + extension = '#%RAML 1.0 Extension\nextends: api.raml\ntypes:\n B: string\n' + workspace, folder = _buffered(memory_workspace, {'api.raml': api, 'ext.raml': extension}) + snapshot = workspace.snapshot(f'{folder}/ext.raml') + assert [each.name for each in queries.visible_names(snapshot, f'{folder}/api.raml', 'types')] == ['A'] + assert [each.name for each in queries.visible_names(snapshot, f'{folder}/ext.raml', 'types')] == ['A', 'B'] + class TestHover: def test_a_type_describes_its_identity_without_an_effective_dump(self, parsed): diff --git a/tests/unit/test_service_source_cache.py b/tests/unit/test_service_source_cache.py new file mode 100644 index 0000000..4761da1 --- /dev/null +++ b/tests/unit/test_service_source_cache.py @@ -0,0 +1,179 @@ +"""One source input serves all consumers without retaining its workspace.""" + +import gc +from weakref import ref + +import pytest + +from fastraml.config import FastRamlConfig, ParserConfig +from fastraml.service import queries, source +from fastraml.service.workspace import BOM +from tests.unit.test_service_workspace import API, _workspace + +DOCUMENT = API + 'types:\n Item:\n type: string\n minLength: 2\n' + + +def _compositions(monkeypatch): + calls = [] + original = source.compose + + def compose(text, **kwargs): + calls.append((text, kwargs['max_depth'])) + return original(text, **kwargs) + + monkeypatch.setattr(source, 'compose', compose) + return calls + + +@pytest.mark.parametrize('source_first', [False, True]) +def test_hover_and_structure_share_one_tree_in_either_request_order(tmp_path, monkeypatch, source_first): + workspace, folder = _workspace(tmp_path, {'api.raml': DOCUMENT}) + uri = f'{folder}/api.raml' + workspace.open(uri, BOM + DOCUMENT, 1) + calls = _compositions(monkeypatch) + first = workspace.source(uri) if source_first else None + snapshot = workspace.snapshot(uri) + assert snapshot.error is None + assert queries.hover(snapshot, uri, 6, 5) is not None + root = workspace.source(uri) + assert root is not None + assert snapshot.hover._node(uri) is root + assert not source_first or first is root + assert len(calls) == 1 + assert calls[0][0] == DOCUMENT, 'single-BOM normalization agrees with the parser input' + + +def test_failed_composition_is_cached_until_the_input_changes(tmp_path, monkeypatch): + workspace, folder = _workspace(tmp_path, {'api.raml': DOCUMENT}) + uri = f'{folder}/api.raml' + workspace.open(uri, API + 'broken: [\n', 1) + calls = _compositions(monkeypatch) + assert workspace.source(uri) is None + assert workspace.source(uri) is None + assert len(calls) == 1 + workspace.change(uri, DOCUMENT, 2) + assert workspace.source(uri) is not None + assert len(calls) == 2 + + +def test_closed_file_equal_text_reuses_the_tree_even_after_another_disk_decode(tmp_path, monkeypatch): + workspace, folder = _workspace(tmp_path, {'api.raml': DOCUMENT}) + calls = _compositions(monkeypatch) + first = workspace.source(f'{folder}/api.raml') + assert workspace.source(f'{folder}/api.raml') is first + assert len(calls) == 1 + + +def test_source_composition_obeys_depth_and_does_not_reuse_a_different_limit(tmp_path, monkeypatch): + config = FastRamlConfig(parser=ParserConfig(max_depth=1)) + workspace, folder = _workspace(tmp_path, {'api.raml': DOCUMENT}, config=config) + calls = _compositions(monkeypatch) + assert workspace.source(f'{folder}/api.raml') is None + assert workspace.source(f'{folder}/api.raml') is None + workspace.config = FastRamlConfig(parser=ParserConfig(max_depth=32)) + assert workspace.source(f'{folder}/api.raml') is not None + assert [depth for _, depth in calls] == [1, 32] + + +def test_source_cache_key_includes_the_yaml_backend(tmp_path, monkeypatch): + workspace, folder = _workspace(tmp_path, {'api.raml': DOCUMENT}) + calls = _compositions(monkeypatch) + first = workspace.source(f'{folder}/api.raml') + current = source.backend_name() + monkeypatch.setattr(source, 'backend_name', lambda: 'python' if current == 'libyaml' else 'libyaml') + assert workspace.source(f'{folder}/api.raml') is not first + assert len(calls) == 2 + + +def test_old_snapshot_reads_do_not_evict_the_current_text_tree(tmp_path, monkeypatch): + workspace, folder = _workspace(tmp_path, {'api.raml': DOCUMENT}) + uri = f'{folder}/api.raml' + workspace.open(uri, DOCUMENT, 1) + old = workspace.snapshot(uri) + workspace.change(uri, DOCUMENT.replace('minLength: 2', 'minLength: 3'), 2) + current = workspace.source(uri) + calls = _compositions(monkeypatch) + assert old.hover._node(uri) is not current + assert workspace.source(uri) is current + assert len(calls) == 1, 'only the stale snapshot composes its old input' + + +def test_an_old_snapshot_cache_miss_cannot_publish_into_a_new_input_generation(tmp_path, monkeypatch): + workspace, folder = _workspace(tmp_path, {'api.raml': DOCUMENT}) + uri = f'{folder}/api.raml' + workspace.open(uri, DOCUMENT, 1) + old = workspace.snapshot(uri) + workspace.change(uri, DOCUMENT + '# intermediate\n', 2) + workspace.change(uri, DOCUMENT, 3) + calls = _compositions(monkeypatch) + stale = old.hover._node(uri) + assert workspace.source(uri) is not stale + assert len(calls) == 2, 'the old miss is private even when current text returns to the same value' + + +def test_unchanged_dependency_structure_survives_a_root_snapshot_rebuild(tmp_path, monkeypatch): + library = '#%RAML 1.0 Library\ntypes:\n Item: string\n' + api = API + 'uses:\n lib: lib.raml\n' + workspace, folder = _workspace(tmp_path, {'api.raml': api, 'lib.raml': library}) + uri = f'{folder}/lib.raml' + workspace.open(uri, library, 1) + calls = _compositions(monkeypatch) + first = workspace.source(uri) + assert workspace.snapshot(f'{folder}/api.raml').hover._node(uri) is first + workspace.change(f'{folder}/api.raml', api + '# edit\n', 2) + assert workspace.snapshot(f'{folder}/api.raml').hover._node(uri) is first + assert len(calls) == 1 + + +def test_an_unrelated_change_does_not_prevent_a_current_snapshot_publishing_source(tmp_path, monkeypatch): + workspace, folder = _workspace(tmp_path, {'api.raml': DOCUMENT, 'other.raml': API}) + uri = f'{folder}/api.raml' + snapshot = workspace.snapshot(uri) + workspace.change(f'{folder}/other.raml', API + '# changed\n', 1) + calls = _compositions(monkeypatch) + first = snapshot.hover._node(uri) + assert workspace.source(uri) is first + assert len(calls) == 1 + + +@pytest.mark.parametrize('source_first', [False, True]) +def test_full_source_retention_offers_its_original_tree_on_demand(tmp_path, monkeypatch, source_first): + config = FastRamlConfig(lint={'rules': [{'id': 'prefer-array-expression'}]}) + workspace, folder = _workspace(tmp_path, {'api.raml': DOCUMENT}, config=config) + uri = f'{folder}/api.raml' + calls = _compositions(monkeypatch) + standalone = workspace.source(uri) if source_first else None + snapshot = workspace.snapshot(uri) + original = snapshot.raml.source_nodes[uri] + assert workspace.source(uri) is original + assert snapshot.hover._node(uri) is original + assert len(calls) == int(source_first) + if source_first: + assert standalone is not original + assert queries.folding_ranges_of(standalone) == queries.folding_ranges_of(original) + + +def test_holding_a_snapshot_does_not_retain_the_workspace_source_cache(tmp_path): + workspace, folder = _workspace(tmp_path, {'api.raml': DOCUMENT, 'other.yaml': 'a:\n b: c\n'}) + snapshot = workspace.snapshot(f'{folder}/api.raml') + workspace.source(f'{folder}/other.yaml') + cache = ref(workspace._sources) + del workspace + gc.collect() + assert cache() is None + assert queries.hover(snapshot, f'{folder}/api.raml', 6, 5) is not None + + +def test_normalized_json_include_is_not_substituted_for_generic_current_source(tmp_path): + document = API + 'types:\n Item:\n properties: {a: integer}\n example: !include example.json\n' + config = FastRamlConfig(lint={'rules': [{'id': 'prefer-array-expression'}]}) + workspace, folder = _workspace( + tmp_path, {'api.raml': document, 'example.json': '\t{\n\t"a": 1\n}\n'}, config=config + ) + snapshot = workspace.snapshot(f'{folder}/api.raml') + assert snapshot.error is None + uri = f'{folder}/example.json' + original = snapshot.raml.include_nodes[uri] + assert workspace.source(uri) is None, 'generic YAML composition does not perform JSON whitespace normalization' + assert snapshot.hover._node(uri) is original + assert workspace.source(uri) is None diff --git a/tests/unit/test_service_workspace.py b/tests/unit/test_service_workspace.py index 742c305..3fdcef6 100644 --- a/tests/unit/test_service_workspace.py +++ b/tests/unit/test_service_workspace.py @@ -17,6 +17,7 @@ from fastraml.errors import ErrorKind, RamlError from fastraml.gctuning import tuned_gc from fastraml.registry import Raml +from fastraml.service.queries import folding_ranges_of from fastraml.service.workspace import _HEAD_BYTES, BOM, Workspace, _head, canonical from fastraml.uris import path_to_file_uri from fastraml.yamlnode import decode_source, read_head @@ -176,6 +177,38 @@ def load(self, uri, *, max_bytes=None): assert workspace._is_root(canonical(uri)) +class TestSourceTrees: + """The composed tree of a file's current text, once per text (docs/21 § 4).""" + + STRUCTURE = API + 'types:\n Thing:\n properties:\n name: string\n' + + def test_a_unchanged_text_composes_once(self, tmp_path): + workspace, folder = _workspace(tmp_path, {'api.raml': self.STRUCTURE}) + uri = f'{folder}/api.raml' + workspace.open(uri, self.STRUCTURE, 1) + first = workspace.source(uri) + assert first is not None + assert workspace.source(uri) is first + + def test_a_changed_text_composes_again(self, tmp_path): + workspace, folder = _workspace(tmp_path, {'api.raml': self.STRUCTURE}) + uri = f'{folder}/api.raml' + workspace.open(uri, self.STRUCTURE, 1) + first = workspace.source(uri) + workspace.change(uri, self.STRUCTURE + ' extra:\n properties:\n age: integer\n', 2) + second = workspace.source(uri) + assert second is not first + assert len(folding_ranges_of(second)) > len(folding_ranges_of(first)) + + def test_a_closed_buffer_composes_again(self, tmp_path): + workspace, folder = _workspace(tmp_path, {'api.raml': self.STRUCTURE}) + uri = f'{folder}/api.raml' + workspace.open(uri, self.STRUCTURE, 1) + first = workspace.source(uri) + workspace.close(uri) + assert workspace.source(uri) is not first + + class TestSnapshots: FILES = { # noqa: RUF012 - read once per test 'api.raml': API + 'uses:\n lib: lib.raml\ntypes:\n Admin: lib.User\n', @@ -301,11 +334,12 @@ def test_collect_frees_the_dropped_snapshots(self, tmp_path): def test_the_yaml_trees_are_kept_only_for_a_lint_rule_that_reads_them(self, tmp_path): files = {'api.raml': API} reading, folder = _workspace(tmp_path, files) - assert reading.snapshot(f'{folder}/api.raml').raml.retain_source - config = FastRamlConfig(lint={'rules': [{'id': 'deprecated-schemas', 'disabled': True}]}) - plain = Workspace([folder], config=config) - raml = plain.snapshot(f'{folder}/api.raml').raml + raml = reading.snapshot(f'{folder}/api.raml').raml assert (raml.retain_source, raml.retain_text) == (False, True) + config = FastRamlConfig(lint={'rules': [{'id': 'prefer-array-expression'}]}) + syntax = Workspace([folder], config=config) + raml = syntax.snapshot(f'{folder}/api.raml').raml + assert (raml.retain_source, raml.retain_text) == (True, True) def test_the_first_snapshot_serving_a_file_parses_one_root(self, tmp_path, monkeypatch): # After an edit to a library, the outline read one snapshot but brought